diff --git a/.gitignore b/.gitignore index 52b8212b139460f02ecbce157ad6818206c71693..5c90b0f39c11771629f97608e916077b46999ffe 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ *.d *.x *.exe +*.sif *.dll *.pyc __pycache__ diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 4dd079eaae52944d69e8225b4030eca2d3af94ae..192491e0df4cde3ef2d1f22584da6e4cbeb18fb2 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -21,17 +21,12 @@ if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) set(CMAKE_INSTALL_PREFIX "$ENV{HOME}/.local" CACHE PATH "default install path" FORCE ) endif() -# To avoid conflicts with the conventional Makefile build system, we build everything here -file(GLOB LIB_SOURCES ${LAMMPS_SOURCE_DIR}/[^.]*.cpp) -file(GLOB LMP_SOURCES ${LAMMPS_SOURCE_DIR}/main.cpp) -list(REMOVE_ITEM LIB_SOURCES ${LMP_SOURCES}) - # Cmake modules/macros are in a subdirectory to keep this file cleaner set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/Modules) include(LAMMPSUtils) -get_lammps_version(${LAMMPS_SOURCE_DIR}/version.h LAMMPS_VERSION) +get_lammps_version(${LAMMPS_SOURCE_DIR}/version.h PROJECT_VERSION) include(PreventInSourceBuilds) @@ -69,9 +64,10 @@ if(${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang") set(CMAKE_TUNE_DEFAULT "-ffast-math -march=native") endif() -# we require C++11 +# we require C++11 without extensions set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) # GNU compiler specific features for testing if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU") @@ -85,43 +81,33 @@ endif() ######################################################################## # User input options # ######################################################################## -option(BUILD_EXE "Build lmp binary" ON) -if(BUILD_EXE) - set(LAMMPS_MACHINE "" CACHE STRING "Suffix to append to lmp binary (WON'T enable any features automatically") - mark_as_advanced(LAMMPS_MACHINE) - if(LAMMPS_MACHINE) - set(LAMMPS_MACHINE "_${LAMMPS_MACHINE}") - endif() - set(LAMMPS_BINARY lmp${LAMMPS_MACHINE}) +set(LAMMPS_MACHINE "" CACHE STRING "Suffix to append to lmp binary (WON'T enable any features automatically") +mark_as_advanced(LAMMPS_MACHINE) +if(LAMMPS_MACHINE) + set(LAMMPS_MACHINE "_${LAMMPS_MACHINE}") endif() +set(LAMMPS_BINARY lmp${LAMMPS_MACHINE}) -option(BUILD_LIB "Build LAMMPS library" OFF) -if(BUILD_LIB) - option(BUILD_SHARED_LIBS "Build shared library" OFF) - if(BUILD_SHARED_LIBS) # for all pkg libs, mpi_stubs and linalg - set(CMAKE_POSITION_INDEPENDENT_CODE ON) - endif() - set(LAMMPS_LIB_SUFFIX "" CACHE STRING "Suffix to append to liblammps and pkg-config file") - mark_as_advanced(LAMMPS_LIB_SUFFIX) - if(LAMMPS_LIB_SUFFIX) - set(LAMMPS_LIB_SUFFIX "_${LAMMPS_LIB_SUFFIX}") - endif() +option(BUILD_SHARED_LIBS "Build shared library" OFF) +if(BUILD_SHARED_LIBS) # for all pkg libs, mpi_stubs and linalg + set(CMAKE_POSITION_INDEPENDENT_CODE ON) endif() option(BUILD_TOOLS "Build and install LAMMPS tools (msi2lmp, binary2txt, chain)" OFF) -if(NOT BUILD_EXE AND NOT BUILD_LIB) - message(FATAL_ERROR "You need to at least enable one of two following options: BUILD_LIB or BUILD_EXE") -endif() - -option(CMAKE_VERBOSE_MAKEFILE "Generate verbose Makefiles" OFF) include(GNUInstallDirs) +file(GLOB ALL_SOURCES ${LAMMPS_SOURCE_DIR}/[^.]*.cpp) +file(GLOB MAIN_SOURCES ${LAMMPS_SOURCE_DIR}/main.cpp) +list(REMOVE_ITEM ALL_SOURCES ${MAIN_SOURCES}) +add_library(lammps ${ALL_SOURCES}) +add_executable(lmp ${MAIN_SOURCES}) +target_link_libraries(lmp PRIVATE lammps) +set_target_properties(lmp PROPERTIES OUTPUT_NAME ${LAMMPS_BINARY}) +install(TARGETS lmp EXPORT LAMMPS_Targets DESTINATION ${CMAKE_INSTALL_BINDIR}) -set(LAMMPS_LINK_LIBS) -set(LAMMPS_DEPS) -set(LAMMPS_API_DEFINES) +option(CMAKE_VERBOSE_MAKEFILE "Generate verbose Makefiles" OFF) -set(DEFAULT_PACKAGES ASPHERE BODY CLASS2 COLLOID COMPRESS CORESHELL DIPOLE +set(STANDARD_PACKAGES ASPHERE BODY CLASS2 COLLOID COMPRESS DIPOLE GRANULAR KSPACE LATTE MANYBODY MC MESSAGE MISC MOLECULE PERI POEMS QEQ REPLICA RIGID SHOCK SPIN SNAP SRD KIM PYTHON MSCG MPIIO VORONOI USER-ATC USER-AWPMD USER-BOCS USER-CGDNA USER-MESODPD USER-CGSDK USER-COLVARS @@ -130,33 +116,15 @@ set(DEFAULT_PACKAGES ASPHERE BODY CLASS2 COLLOID COMPRESS CORESHELL DIPOLE USER-NETCDF USER-PHONON USER-PLUMED USER-PTM USER-QTB USER-REACTION USER-REAXC USER-SCAFACOS USER-SDPD USER-SMD USER-SMTBQ USER-SPH USER-TALLY USER-UEF USER-VTK USER-QUIP USER-QMMM USER-YAFF USER-ADIOS) -set(ACCEL_PACKAGES USER-OMP KOKKOS OPT USER-INTEL GPU) -foreach(PKG ${DEFAULT_PACKAGES} ${ACCEL_PACKAGES}) +set(SUFFIX_PACKAGES CORESHELL USER-OMP KOKKOS OPT USER-INTEL GPU) +foreach(PKG ${STANDARD_PACKAGES} ${SUFFIX_PACKAGES}) option(PKG_${PKG} "Build ${PKG} Package" OFF) endforeach() -###################################################### -# download and unpack support binaries for compilation -# of windows binaries. -###################################################### -if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows") - set(LAMMPS_THIRDPARTY_URL "http://download.lammps.org/thirdparty") - file(DOWNLOAD "${LAMMPS_THIRDPARTY_URL}/opencl-win-devel.tar.gz" "${CMAKE_CURRENT_BINARY_DIR}/opencl-win-devel.tar.gz" - EXPECTED_MD5 2c00364888d5671195598b44c2e0d44d) - execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf opencl-win-devel.tar.gz WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) - if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86") - set(OpenCL_LIBRARY "${CMAKE_CURRENT_BINARY_DIR}/OpenCL/lib_win32/libOpenCL.dll") - elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86_64") - set(OpenCL_LIBRARY "${CMAKE_CURRENT_BINARY_DIR}/OpenCL/lib_win64/libOpenCL.dll") - endif() - set(OpenCL_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/OpenCL/include") -endif() - -###################################################### ###################################################### # packages with special compiler needs or external libs ###################################################### -include_directories(${LAMMPS_SOURCE_DIR}) +target_include_directories(lammps PUBLIC $) if(PKG_USER-ADIOS) # The search for ADIOS2 must come before MPI because @@ -164,35 +132,50 @@ if(PKG_USER-ADIOS) # script that defines the MPI::MPI_C target enable_language(C) find_package(ADIOS2 REQUIRED) - list(APPEND LAMMPS_LINK_LIBS adios2::adios2) + target_link_libraries(lammps PRIVATE adios2::adios2) endif() -# do MPI detection after language activation, -# in case MPI for these languages is required -set(MPI_CXX_SKIP_MPICXX TRUE) -find_package(MPI QUIET) -option(BUILD_MPI "Build MPI version" ${MPI_FOUND}) +if (CMAKE_SYSTEM_NAME STREQUAL Windows) + option(BUILD_MPI "Build MPI version" OFF) +else() + # do MPI detection after language activation, + # in case MPI for these languages is required + set(MPI_CXX_SKIP_MPICXX TRUE) + find_package(MPI QUIET) + option(BUILD_MPI "Build MPI version" ${MPI_FOUND}) +endif() if(BUILD_MPI) # We use a non-standard procedure to compile with MPI on windows if (CMAKE_SYSTEM_NAME STREQUAL Windows) include(MPI4WIN) + target_link_libraries(lammps PUBLIC MPI::MPI_CXX) else() + set(MPI_CXX_SKIP_MPICXX ON) find_package(MPI REQUIRED) - include_directories(${MPI_CXX_INCLUDE_PATH}) - add_definitions(-DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX=1) - list(APPEND LAMMPS_LINK_LIBS ${MPI_CXX_LIBRARIES}) + target_link_libraries(lammps PUBLIC MPI::MPI_CXX) option(LAMMPS_LONGLONG_TO_LONG "Workaround if your system or MPI version does not recognize 'long long' data types" OFF) if(LAMMPS_LONGLONG_TO_LONG) - add_definitions(-DLAMMPS_LONGLONG_TO_LONG) + target_compile_definitions(lammps PRIVATE -DLAMMPS_LONGLONG_TO_LONG) endif() endif() else() enable_language(C) file(GLOB MPI_SOURCES ${LAMMPS_SOURCE_DIR}/STUBS/mpi.c) add_library(mpi_stubs STATIC ${MPI_SOURCES}) - include_directories(${LAMMPS_SOURCE_DIR}/STUBS) - list(APPEND LAMMPS_LINK_LIBS mpi_stubs) + if(NOT BUILD_SHARED_LIBS) + install(TARGETS mpi_stubs EXPORT LAMMPS_Targets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) + endif() + set_target_properties(mpi_stubs PROPERTIES OUTPUT_NAME lammps_mpi_stubs${LAMMPS_MACHINE}) + target_include_directories(mpi_stubs PUBLIC $ $) + install(FILES ${LAMMPS_SOURCE_DIR}/STUBS/mpi.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/lammps/mpi) + if(BUILD_SHARED_LIBS) + target_link_libraries(lammps PRIVATE mpi_stubs) + target_include_directories(lammps INTERFACE $ $) + else() + target_link_libraries(lammps PUBLIC mpi_stubs) + endif() + add_library(MPI::MPI_CXX ALIAS mpi_stubs) endif() set(LAMMPS_SIZES "smallbig" CACHE STRING "LAMMPS integer sizes (smallsmall: all 32-bit, smallbig: 64-bit #atoms #timesteps, bigbig: also 64-bit imageint, 64-bit atom ids)") @@ -200,8 +183,7 @@ set(LAMMPS_SIZES_VALUES smallbig bigbig smallsmall) set_property(CACHE LAMMPS_SIZES PROPERTY STRINGS ${LAMMPS_SIZES_VALUES}) validate_option(LAMMPS_SIZES LAMMPS_SIZES_VALUES) string(TOUPPER ${LAMMPS_SIZES} LAMMPS_SIZES) -add_definitions(-DLAMMPS_${LAMMPS_SIZES}) -set(LAMMPS_API_DEFINES "${LAMMPS_API_DEFINES} -DLAMMPS_${LAMMPS_SIZES}") +target_compile_definitions(lammps PUBLIC -DLAMMPS_${LAMMPS_SIZES}) # posix_memalign is not available on Windows if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows") @@ -210,13 +192,12 @@ else() set(LAMMPS_MEMALIGN "64" CACHE STRING "enables the use of the posix_memalign() call instead of malloc() when large chunks or memory are allocated by LAMMPS. Set to 0 to disable") endif() if(NOT ${LAMMPS_MEMALIGN} STREQUAL "0") - add_definitions(-DLAMMPS_MEMALIGN=${LAMMPS_MEMALIGN}) + target_compile_definitions(lammps PRIVATE -DLAMMPS_MEMALIGN=${LAMMPS_MEMALIGN}) endif() option(LAMMPS_EXCEPTIONS "enable the use of C++ exceptions for error messages (useful for library interface)" OFF) if(LAMMPS_EXCEPTIONS) - add_definitions(-DLAMMPS_EXCEPTIONS) - set(LAMMPS_API_DEFINES "${LAMMPS_API_DEFINES} -DLAMMPS_EXCEPTIONS") + target_compile_definitions(lammps PUBLIC -DLAMMPS_EXCEPTIONS) endif() # "hard" dependencies between packages resulting @@ -227,27 +208,36 @@ pkg_depends(USER-LB MPI) pkg_depends(USER-PHONON KSPACE) pkg_depends(USER-SCAFACOS MPI) +# detect if we may enable OpenMP support by default +set(BUILD_OMP_DEFAULT OFF) find_package(OpenMP QUIET) - -# TODO: this is a temporary workaround until a better solution is found. AK 2019-05-30 -# GNU GCC 9.x uses settings incompatible with our use of 'default(none)' in OpenMP pragmas -# where we assume older GCC semantics. For the time being, we disable OpenMP by default -# for GCC 9.x and beyond. People may manually turn it on, but need to run the script -# src/USER-OMP/hack_openmp_for_pgi_gcc9.sh on all sources to make it compatible with gcc 9. -if ((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 8.99.9)) - option(BUILD_OMP "Build with OpenMP support" OFF) -else() - option(BUILD_OMP "Build with OpenMP support" ${OpenMP_FOUND}) +if(OpenMP_FOUND) + check_include_file_cxx(omp.h HAVE_OMP_H_INCLUDE) + if(HAVE_OMP_H_INCLUDE) + set(BUILD_OMP_DEFAULT ON) + endif() endif() +option(BUILD_OMP "Build with OpenMP support" ${BUILD_OMP_DEFAULT}) + if(BUILD_OMP) find_package(OpenMP REQUIRED) check_include_file_cxx(omp.h HAVE_OMP_H_INCLUDE) if(NOT HAVE_OMP_H_INCLUDE) - message(FATAL_ERROR "Cannot find required 'omp.h' header file") + message(FATAL_ERROR "Cannot find the 'omp.h' header file required for full OpenMP support") + endif() + + if (((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 8.99.9)) OR + ((CMAKE_CXX_COMPILER_ID STREQUAL "Clang") AND (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 9.99.9)) OR + ((CMAKE_CXX_COMPILER_ID STREQUAL "Intel") AND (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 18.99.9)) + ) + # GCC 9.x and later plus Clang 10.x and later implement strict OpenMP 4.0 semantics for consts. + # Intel 18.0 was tested to support both, so we switch to OpenMP 4+ from 19.x onward to be safe. + target_compile_definitions(lammps PRIVATE -DLAMMPS_OMP_COMPAT=4) + else() + target_compile_definitions(lammps PRIVATE -DLAMMPS_OMP_COMPAT=3) endif() - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") + target_link_libraries(lammps PRIVATE OpenMP::OpenMP_CXX) endif() if(PKG_MSCG OR PKG_USER-ATC OR PKG_USER-AWPMD OR PKG_USER-QUIP OR PKG_LATTE) @@ -262,6 +252,10 @@ if(PKG_MSCG OR PKG_USER-ATC OR PKG_USER-AWPMD OR PKG_USER-QUIP OR PKG_LATTE) enable_language(Fortran) file(GLOB LAPACK_SOURCES ${LAMMPS_LIB_SOURCE_DIR}/linalg/[^.]*.[fF]) add_library(linalg STATIC ${LAPACK_SOURCES}) + if(NOT BUILD_SHARED_LIBS) + install(TARGETS linalg EXPORT LAMMPS_Targets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) + endif() + set_target_properties(linalg PROPERTIES OUTPUT_NAME lammps_linalg${LAMMPS_MACHINE}) set(BLAS_LIBRARIES "$") set(LAPACK_LIBRARIES "$") else() @@ -274,9 +268,13 @@ find_package(JPEG QUIET) option(WITH_JPEG "Enable JPEG support" ${JPEG_FOUND}) if(WITH_JPEG) find_package(JPEG REQUIRED) - add_definitions(-DLAMMPS_JPEG) - include_directories(${JPEG_INCLUDE_DIR}) - list(APPEND LAMMPS_LINK_LIBS ${JPEG_LIBRARIES}) + target_compile_definitions(lammps PRIVATE -DLAMMPS_JPEG) + if(CMAKE_VERSION VERSION_LESS 3.12) + target_include_directories(lammps PRIVATE ${JPEG_INCLUDE_DIR}) + target_link_libraries(lammps PRIVATE ${JPEG_LIBRARIES}) + else() + target_link_libraries(lammps PRIVATE JPEG::JPEG) + endif() endif() find_package(PNG QUIET) @@ -289,9 +287,8 @@ endif() if(WITH_PNG) find_package(PNG REQUIRED) find_package(ZLIB REQUIRED) - include_directories(${PNG_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS}) - list(APPEND LAMMPS_LINK_LIBS ${PNG_LIBRARIES} ${ZLIB_LIBRARIES}) - add_definitions(-DLAMMPS_PNG) + target_link_libraries(lammps PRIVATE PNG::PNG ZLIB::ZLIB) + target_compile_definitions(lammps PRIVATE -DLAMMPS_PNG) endif() find_program(GZIP_EXECUTABLE gzip) @@ -301,7 +298,7 @@ if(WITH_GZIP) if(NOT GZIP_FOUND) message(FATAL_ERROR "gzip executable not found") endif() - add_definitions(-DLAMMPS_GZIP) + target_compile_definitions(lammps PRIVATE -DLAMMPS_GZIP) endif() find_program(FFMPEG_EXECUTABLE ffmpeg) @@ -311,7 +308,7 @@ if(WITH_FFMPEG) if(NOT FFMPEG_FOUND) message(FATAL_ERROR "ffmpeg executable not found") endif() - add_definitions(-DLAMMPS_FFMPEG) + target_compile_definitions(lammps PRIVATE -DLAMMPS_FFMPEG) endif() if(BUILD_SHARED_LIBS) @@ -324,29 +321,24 @@ else() set(CUDA_REQUEST_PIC) endif() -include(Packages/KSPACE) -include(Packages/PYTHON) -include(Packages/VORONOI) -include(Packages/USER-COLVARS) -include(Packages/USER-MOLFILE) -include(Packages/USER-NETCDF) -include(Packages/USER-PLUMED) -include(Packages/USER-QMMM) -include(Packages/USER-QUIP) -include(Packages/USER-SCAFACOS) -include(Packages/USER-SMD) -include(Packages/USER-VTK) -include(Packages/KIM) -include(Packages/LATTE) -include(Packages/MESSAGE) -include(Packages/MSCG) -include(Packages/COMPRESS) +foreach(PKG_WITH_INCL KSPACE PYTHON VORONOI USER-COLVARS USER-MOLFILE USER-NETCDF USER-PLUMED USER-QMMM + USER-QUIP USER-SCAFACOS USER-SMD USER-VTK KIM LATTE MESSAGE MSCG COMPRESS) + if(PKG_${PKG_WITH_INCL}) + include(Packages/${PKG_WITH_INCL}) + endif() +endforeach() set(CMAKE_TUNE_FLAGS "${CMAKE_TUNE_DEFAULT}" CACHE STRING "Compiler specific optimization or instrumentation") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_TUNE_FLAGS}") -if(CMAKE_Fortran_COMPILER) - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${CMAKE_TUNE_FLAGS}") -endif() +separate_arguments(CMAKE_TUNE_FLAGS) +include(CheckCXXCompilerFlag) +foreach(_FLAG ${CMAKE_TUNE_FLAGS}) + check_cxx_compiler_flag("${_FLAG}" COMPILER_SUPPORTS${_FLAG}) + if(COMPILER_SUPPORTS${_FLAG}) + target_compile_options(lammps PRIVATE ${_FLAG}) + else() + message(WARNING "${_FLAG} found in CMAKE_TUNE_FLAGS, but not supported by the compiler, skipping") + endif() +endforeach() ######################################################################## # Basic system tests (standard libraries, headers, functions, types) # ######################################################################## @@ -359,15 +351,7 @@ endforeach(HEADER) set(MATH_LIBRARIES "m" CACHE STRING "math library") mark_as_advanced( MATH_LIBRARIES ) -include(CheckLibraryExists) -# RB: disabled this check because it breaks with KOKKOS CUDA enabled -#foreach(FUNC sin cos) -# check_library_exists(${MATH_LIBRARIES} ${FUNC} "" FOUND_${FUNC}_${MATH_LIBRARIES}) -# if(NOT FOUND_${FUNC}_${MATH_LIBRARIES}) -# message(FATAL_ERROR "Could not find needed math function - ${FUNC}") -# endif(NOT FOUND_${FUNC}_${MATH_LIBRARIES}) -#endforeach(FUNC) -list(APPEND LAMMPS_LINK_LIBS ${MATH_LIBRARIES}) +target_link_libraries(lammps PRIVATE ${MATH_LIBRARIES}) ###################################### # Generate Basic Style files @@ -378,7 +362,7 @@ RegisterStyles(${LAMMPS_SOURCE_DIR}) ############################################## # add sources of enabled packages ############################################ -foreach(PKG ${DEFAULT_PACKAGES}) +foreach(PKG ${STANDARD_PACKAGES}) set(${PKG}_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/${PKG}) file(GLOB ${PKG}_SOURCES ${${PKG}_SOURCES_DIR}/[^.]*.cpp) @@ -391,8 +375,8 @@ foreach(PKG ${DEFAULT_PACKAGES}) # detects styles in package and adds them to global list RegisterStyles(${${PKG}_SOURCES_DIR}) - list(APPEND LIB_SOURCES ${${PKG}_SOURCES}) - include_directories(${${PKG}_SOURCES_DIR}) + target_sources(lammps PRIVATE ${${PKG}_SOURCES}) + target_include_directories(lammps PRIVATE ${${PKG}_SOURCES_DIR}) endif() RegisterPackages(${${PKG}_SOURCES_DIR}) @@ -401,12 +385,12 @@ endforeach() # packages that need defines set foreach(PKG MPIIO) if(PKG_${PKG}) - add_definitions(-DLMP_${PKG}) + target_compile_definitions(lammps PRIVATE -DLMP_${PKG}) endif() endforeach() # dedicated check for entire contents of accelerator packages -foreach(PKG ${ACCEL_PACKAGES}) +foreach(PKG ${SUFFIX_PACKAGES}) set(${PKG}_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/${PKG}) file(GLOB ${PKG}_SOURCES ${${PKG}_SOURCES_DIR}/[^.]*.cpp) @@ -426,14 +410,14 @@ foreach(SIMPLE_LIB POEMS USER-ATC USER-AWPMD USER-H5MD) string(REGEX REPLACE "^USER-" "" PKG_LIB "${SIMPLE_LIB}") string(TOLOWER "${PKG_LIB}" PKG_LIB) file(GLOB_RECURSE ${PKG_LIB}_SOURCES - ${LAMMPS_LIB_SOURCE_DIR}/${PKG_LIB}/[^.]*.F ${LAMMPS_LIB_SOURCE_DIR}/${PKG_LIB}/[^.]*.c ${LAMMPS_LIB_SOURCE_DIR}/${PKG_LIB}/[^.]*.cpp) add_library(${PKG_LIB} STATIC ${${PKG_LIB}_SOURCES}) - if(LAMMPS_USE_MPI4WIN) - add_dependencies(${PKG_LIB} mpi4win_build) + if(NOT BUILD_SHARED_LIBS) + install(TARGETS ${PKG_LIB} EXPORT LAMMPS_Targets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) endif() - list(APPEND LAMMPS_LINK_LIBS ${PKG_LIB}) + set_target_properties(${PKG_LIB} PROPERTIES OUTPUT_NAME lammps_${PKG_LIB}${LAMMPS_MACHINE}) + target_link_libraries(lammps PRIVATE ${PKG_LIB}) if(PKG_LIB STREQUAL awpmd) target_include_directories(awpmd PUBLIC ${LAMMPS_LIB_SOURCE_DIR}/awpmd/systems/interact ${LAMMPS_LIB_SOURCE_DIR}/awpmd/ivutils/include) elseif(PKG_LIB STREQUAL h5md) @@ -445,30 +429,31 @@ foreach(SIMPLE_LIB POEMS USER-ATC USER-AWPMD USER-H5MD) endforeach() if(PKG_USER-AWPMD) - target_link_libraries(awpmd ${LAPACK_LIBRARIES}) + target_link_libraries(awpmd PRIVATE ${LAPACK_LIBRARIES}) endif() if(PKG_USER-ATC) if(LAMMPS_SIZES STREQUAL BIGBIG) message(FATAL_ERROR "The USER-ATC Package is not compatible with -DLAMMPS_BIGBIG") endif() - target_link_libraries(atc ${LAPACK_LIBRARIES}) + target_link_libraries(atc PRIVATE ${LAPACK_LIBRARIES} MPI::MPI_CXX) + target_include_directories(atc PRIVATE ${LAMMPS_SOURCE_DIR}) + target_compile_definitions(atc PRIVATE -DLAMMPS_${LAMMPS_SIZES}) endif() -include(Packages/USER-H5MD) +if(PKG_USER-H5MD) + include(Packages/USER-H5MD) +endif() ###################################################################### # packages which selectively include variants based on enabled styles # e.g. accelerator packages ###################################################################### -include(Packages/CORESHELL) -include(Packages/QEQ) -include(Packages/USER-OMP) -include(Packages/USER-SDPD) -include(Packages/KOKKOS) -include(Packages/OPT) -include(Packages/USER-INTEL) -include(Packages/GPU) +foreach(PKG_WITH_INCL CORESHELL QEQ USER-OMP USER-SDPD KOKKOS OPT USER-INTEL GPU) + if(PKG_${PKG_WITH_INCL}) + include(Packages/${PKG_WITH_INCL}) + endif() +endforeach() ###################################################################### # the windows version of LAMMPS requires a couple extra libraries @@ -476,10 +461,7 @@ include(Packages/GPU) # and after everything else that is compiled locally ###################################################################### if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows") - if(LAMMPS_USE_MPI4WIN) - list(APPEND LAMMPS_LINK_LIBS ${MPI4WIN_LIBRARIES}) - endif() - list(APPEND LAMMPS_LINK_LIBS -lwsock32 -lpsapi) + target_link_libraries(lammps PRIVATE -lwsock32 -lpsapi) endif() ###################################################### @@ -492,14 +474,14 @@ set(LAMMPS_STYLE_HEADERS_DIR ${CMAKE_CURRENT_BINARY_DIR}/styles) GenerateStyleHeaders(${LAMMPS_STYLE_HEADERS_DIR}) GeneratePackagesHeaders(${LAMMPS_STYLE_HEADERS_DIR}) -include_directories(${LAMMPS_STYLE_HEADERS_DIR}) +target_include_directories(lammps PRIVATE ${LAMMPS_STYLE_HEADERS_DIR}) ###################################### # Generate lmpinstalledpkgs.h ###################################### set(temp "#ifndef LMP_INSTALLED_PKGS_H\n#define LMP_INSTALLED_PKGS_H\n") set(temp "${temp}const char * LAMMPS_NS::LAMMPS::installed_packages[] = {\n") -set(temp_PKG_LIST ${DEFAULT_PACKAGES} ${ACCEL_PACKAGES}) +set(temp_PKG_LIST ${STANDARD_PACKAGES} ${SUFFIX_PACKAGES}) list(SORT temp_PKG_LIST) foreach(PKG ${temp_PKG_LIST}) if(PKG_${PKG}) @@ -521,7 +503,7 @@ add_custom_target(gitversion COMMAND ${CMAKE_COMMAND} -DLAMMPS_STYLE_HEADERS_DIR="${LAMMPS_STYLE_HEADERS_DIR}" -P ${CMAKE_CURRENT_SOURCE_DIR}/Modules/generate_lmpgitversion.cmake) set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${LAMMPS_STYLE_HEADERS_DIR}/gitversion.h) -list(APPEND LAMMPS_DEPS gitversion) +add_dependencies(lammps gitversion) ########################################### # Actually add executable and lib to build @@ -529,85 +511,54 @@ list(APPEND LAMMPS_DEPS gitversion) get_property(LANGUAGES GLOBAL PROPERTY ENABLED_LANGUAGES) list (FIND LANGUAGES "Fortran" _index) if (${_index} GREATER -1) - list(APPEND LAMMPS_LINK_LIBS ${CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES}) -endif() -list(REMOVE_DUPLICATES LAMMPS_LINK_LIBS) -if(BUILD_LIB) - add_library(lammps ${LIB_SOURCES}) - target_link_libraries(lammps ${LAMMPS_LINK_LIBS}) - if(LAMMPS_DEPS) - add_dependencies(lammps ${LAMMPS_DEPS}) - endif() - set(LAMMPS_CXX_HEADERS - ${LAMMPS_SOURCE_DIR}/angle.h - ${LAMMPS_SOURCE_DIR}/atom.h - ${LAMMPS_SOURCE_DIR}/bond.h - ${LAMMPS_SOURCE_DIR}/citeme.h - ${LAMMPS_SOURCE_DIR}/comm.h - ${LAMMPS_SOURCE_DIR}/compute.h - ${LAMMPS_SOURCE_DIR}/dihedral.h - ${LAMMPS_SOURCE_DIR}/domain.h - ${LAMMPS_SOURCE_DIR}/error.h - ${LAMMPS_SOURCE_DIR}/fix.h - ${LAMMPS_SOURCE_DIR}/force.h - ${LAMMPS_SOURCE_DIR}/group.h - ${LAMMPS_SOURCE_DIR}/improper.h - ${LAMMPS_SOURCE_DIR}/input.h - ${LAMMPS_SOURCE_DIR}/kspace.h - ${LAMMPS_SOURCE_DIR}/lammps.h - ${LAMMPS_SOURCE_DIR}/lattice.h - ${LAMMPS_SOURCE_DIR}/lmppython.h - ${LAMMPS_SOURCE_DIR}/memory.h - ${LAMMPS_SOURCE_DIR}/modify.h - ${LAMMPS_SOURCE_DIR}/neighbor.h - ${LAMMPS_SOURCE_DIR}/neigh_list.h - ${LAMMPS_SOURCE_DIR}/output.h - ${LAMMPS_SOURCE_DIR}/pair.h - ${LAMMPS_SOURCE_DIR}/pointers.h - ${LAMMPS_SOURCE_DIR}/region.h - ${LAMMPS_SOURCE_DIR}/timer.h - ${LAMMPS_SOURCE_DIR}/universe.h - ${LAMMPS_SOURCE_DIR}/update.h - ${LAMMPS_SOURCE_DIR}/variable.h) - - set_target_properties(lammps PROPERTIES OUTPUT_NAME lammps${LAMMPS_LIB_SUFFIX}) - set_target_properties(lammps PROPERTIES SOVERSION ${SOVERSION}) - install(TARGETS lammps LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) - install(FILES ${LAMMPS_SOURCE_DIR}/library.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/lammps) - install(FILES ${LAMMPS_CXX_HEADERS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/lammps) - configure_file(pkgconfig/liblammps.pc.in ${CMAKE_CURRENT_BINARY_DIR}/liblammps${LAMMPS_LIB_SUFFIX}.pc @ONLY) - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liblammps${LAMMPS_LIB_SUFFIX}.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) - configure_file(FindLAMMPS.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/FindLAMMPS${LAMMPS_LIB_SUFFIX}.cmake @ONLY) - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/FindLAMMPS${LAMMPS_LIB_SUFFIX}.cmake DESTINATION ${CMAKE_INSTALL_DATADIR}/cmake/Modules) -else() - list(APPEND LMP_SOURCES ${LIB_SOURCES}) -endif() - -if(BUILD_EXE) - add_executable(lmp ${LMP_SOURCES}) - if(BUILD_LIB) - target_link_libraries(lmp lammps) - else() - target_link_libraries(lmp ${LAMMPS_LINK_LIBS}) - if(LAMMPS_DEPS) - add_dependencies(lmp ${LAMMPS_DEPS}) - endif() - endif() - - set_target_properties(lmp PROPERTIES OUTPUT_NAME ${LAMMPS_BINARY}) - install(TARGETS lmp DESTINATION ${CMAKE_INSTALL_BINDIR}) - install(FILES ${LAMMPS_DOC_DIR}/lammps.1 DESTINATION ${CMAKE_INSTALL_MANDIR}/man1 RENAME ${LAMMPS_BINARY}.1) + target_link_libraries(lammps PRIVATE ${CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES}) endif() +set(LAMMPS_CXX_HEADERS angle.h atom.h bond.h citeme.h comm.h compute.h dihedral.h domain.h error.h fix.h force.h group.h improper.h + input.h info.h kspace.h lammps.h lattice.h library.h lmppython.h lmptype.h memory.h modify.h neighbor.h neigh_list.h output.h + pair.h pointers.h region.h timer.h universe.h update.h variable.h) +if(LAMMPS_EXCEPTIONS) + list(APPEND LAMMPS_CXX_HEADERS exceptions.h) +endif() + +set_target_properties(lammps PROPERTIES OUTPUT_NAME lammps${LAMMPS_MACHINE}) +set_target_properties(lammps PROPERTIES SOVERSION ${SOVERSION}) +install(TARGETS lammps EXPORT LAMMPS_Targets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) +target_include_directories(lammps PUBLIC $) +file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/includes/lammps) +foreach(_HEADER ${LAMMPS_CXX_HEADERS}) + add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/includes/lammps/${_HEADER} COMMAND ${CMAKE_COMMAND} -E copy_if_different ${LAMMPS_SOURCE_DIR}/${_HEADER} ${CMAKE_CURRENT_BINARY_DIR}/includes/lammps/${_HEADER} DEPENDS ${LAMMPS_SOURCE_DIR}/${_HEADER}) + add_custom_target(${_HEADER} DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/includes/lammps/${_HEADER}) + add_dependencies(lammps ${_HEADER}) + install(FILES ${LAMMPS_SOURCE_DIR}/${_HEADER} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/lammps) +endforeach() +target_include_directories(lammps INTERFACE $) +add_library(LAMMPS::lammps ALIAS lammps) +get_target_property(LAMMPS_DEFINES lammps INTERFACE_COMPILE_DEFINITIONS) +set(LAMMPS_API_DEFINES) +foreach(_DEF ${LAMMPS_DEFINES}) + set(LAMMPS_API_DEFINES "${LAMMPS_API_DEFINES} -D${_DEF}") +endforeach() +configure_file(pkgconfig/liblammps.pc.in ${CMAKE_CURRENT_BINARY_DIR}/liblammps${LAMMPS_MACHINE}.pc @ONLY) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liblammps${LAMMPS_MACHINE}.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) +install(EXPORT LAMMPS_Targets FILE LAMMPS_Targets.cmake NAMESPACE LAMMPS:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/LAMMPS) +file(GLOB MODULE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/Modules/Find*.cmake) +install(FILES ${MODULE_FILES} DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/LAMMPS) +include(CMakePackageConfigHelpers) +configure_file(LAMMPSConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/LAMMPSConfig.cmake @ONLY) +write_basic_package_version_file("LAMMPSConfigVersion.cmake" VERSION ${PROJECT_VERSION} COMPATIBILITY ExactVersion) +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/LAMMPSConfig.cmake" "${CMAKE_CURRENT_BINARY_DIR}/LAMMPSConfigVersion.cmake" DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/LAMMPS) +install(FILES ${LAMMPS_DOC_DIR}/lammps.1 DESTINATION ${CMAKE_INSTALL_MANDIR}/man1 RENAME ${LAMMPS_BINARY}.1) if(BUILD_TOOLS) add_executable(binary2txt ${LAMMPS_TOOLS_DIR}/binary2txt.cpp) + target_compile_definitions(binary2txt PRIVATE -DLAMMPS_${LAMMPS_SIZES}) install(TARGETS binary2txt DESTINATION ${CMAKE_INSTALL_BINDIR}) include(CheckGeneratorSupport) if(CMAKE_GENERATOR_SUPPORT_FORTRAN) enable_language(Fortran) add_executable(chain.x ${LAMMPS_TOOLS_DIR}/chain.f) - target_link_libraries(chain.x ${CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES}) + target_link_libraries(chain.x PRIVATE ${CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES}) install(TARGETS chain.x DESTINATION ${CMAKE_INSTALL_BINDIR}) else() message(WARNING "CMake build doesn't support fortran, skipping building 'chain.x'") @@ -617,7 +568,7 @@ if(BUILD_TOOLS) get_filename_component(MSI2LMP_SOURCE_DIR ${LAMMPS_TOOLS_DIR}/msi2lmp/src ABSOLUTE) file(GLOB MSI2LMP_SOURCES ${MSI2LMP_SOURCE_DIR}/[^.]*.c) add_executable(msi2lmp ${MSI2LMP_SOURCES}) - target_link_libraries(msi2lmp m) + target_link_libraries(msi2lmp PRIVATE ${MATH_LIBRARIES}) install(TARGETS msi2lmp DESTINATION ${CMAKE_INSTALL_BINDIR}) install(FILES ${LAMMPS_DOC_DIR}/msi2lmp.1 DESTINATION ${CMAKE_INSTALL_MANDIR}/man1) endif() @@ -647,12 +598,19 @@ install( # conventional build. Only available, if a shared library is built. # This is primarily for people that only want to use the Python wrapper. ############################################################################### -if(BUILD_LIB AND BUILD_SHARED_LIBS) - find_package(PythonInterp) - if (PYTHONINTERP_FOUND) +if(BUILD_SHARED_LIBS) + if(CMAKE_VERSION VERSION_LESS 3.12) + find_package(PythonInterp) # Deprecated since version 3.12 + if(PYTHONINTERP_FOUND) + set(Python_EXECUTABLE ${PYTHON_EXECUTABLE}) + endif() + else() + find_package(Python COMPONENTS Interpreter) + endif() + if (Python_EXECUTABLE) add_custom_target( install-python - ${PYTHON_EXECUTABLE} install.py -v ${LAMMPS_SOURCE_DIR}/version.h + ${Python_EXECUTABLE} install.py -v ${LAMMPS_SOURCE_DIR}/version.h -m ${LAMMPS_PYTHON_DIR}/lammps.py -l ${CMAKE_BINARY_DIR}/liblammps${CMAKE_SHARED_LIBRARY_SUFFIX} WORKING_DIRECTORY ${LAMMPS_PYTHON_DIR} @@ -673,10 +631,17 @@ endif() # LAMMPS for package managers and with different prefix settings. # This requires either a shared library or that the PYTHON package is included. ############################################################################### -if((BUILD_LIB AND BUILD_SHARED_LIBS) OR (PKG_PYTHON)) - find_package(PythonInterp) - if (PYTHONINTERP_FOUND) - execute_process(COMMAND ${PYTHON_EXECUTABLE} +if(BUILD_SHARED_LIBS OR PKG_PYTHON) + if(CMAKE_VERSION VERSION_LESS 3.12) + find_package(PythonInterp) # Deprecated since version 3.12 + if(PYTHONINTERP_FOUND) + set(Python_EXECUTABLE ${PYTHON_EXECUTABLE}) + endif() + else() + find_package(Python COMPONENTS Interpreter) + endif() + if (Python_EXECUTABLE) + execute_process(COMMAND ${Python_EXECUTABLE} -c "import distutils.sysconfig as cg; print(cg.get_python_lib(1,0,prefix='${CMAKE_INSTALL_PREFIX}'))" OUTPUT_VARIABLE PYTHON_DEFAULT_INSTDIR OUTPUT_STRIP_TRAILING_WHITESPACE) set(PYTHON_INSTDIR ${PYTHON_DEFAULT_INSTDIR} CACHE PATH "Installation folder for LAMMPS Python module") @@ -690,67 +655,83 @@ include(CodeCoverage) ############################################################################### # Print package summary ############################################################################### -foreach(PKG ${DEFAULT_PACKAGES} ${ACCEL_PACKAGES}) +foreach(PKG ${STANDARD_PACKAGES} ${SUFFIX_PACKAGES}) if(PKG_${PKG}) message(STATUS "Building package: ${PKG}") endif() endforeach() -get_directory_property(CPPFLAGS DIRECTORY ${CMAKE_SOURCE_DIR} COMPILE_DEFINITIONS) +get_target_property(DEFINES lammps COMPILE_DEFINITIONS) include(FeatureSummary) feature_summary(DESCRIPTION "The following tools and libraries have been found and configured:" WHAT PACKAGES_FOUND) message(STATUS "<<< Build configuration >>> - Build type ${CMAKE_BUILD_TYPE} - Install path ${CMAKE_INSTALL_PREFIX} - Generator ${CMAKE_GENERATOR} using ${CMAKE_MAKE_PROGRAM} - Compilers and Flags: - C++ Compiler ${CMAKE_CXX_COMPILER} - Type ${CMAKE_CXX_COMPILER_ID} - Version ${CMAKE_CXX_COMPILER_VERSION} - C++ Flags ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BTYPE}} - Defines ${CPPFLAGS}") + Build type: ${CMAKE_BUILD_TYPE} + Install path: ${CMAKE_INSTALL_PREFIX} + Generator: ${CMAKE_GENERATOR} using ${CMAKE_MAKE_PROGRAM} +-- <<< Compilers and Flags: >>> +-- C++ Compiler: ${CMAKE_CXX_COMPILER} + Type: ${CMAKE_CXX_COMPILER_ID} + Version: ${CMAKE_CXX_COMPILER_VERSION} + C++ Flags: ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BTYPE}} + Defines: ${DEFINES}") +get_target_property(OPTIONS lammps COMPILE_OPTIONS) +if(OPTIONS) + message(" Options: ${OPTIONS}") +endif() get_property(LANGUAGES GLOBAL PROPERTY ENABLED_LANGUAGES) list (FIND LANGUAGES "Fortran" _index) if (${_index} GREATER -1) - message(STATUS "Fortran Compiler ${CMAKE_Fortran_COMPILER} - Type ${CMAKE_Fortran_COMPILER_ID} - Version ${CMAKE_Fortran_COMPILER_VERSION} - Fortran Flags ${CMAKE_Fortran_FLAGS} ${CMAKE_Fortran_FLAGS_${BTYPE}}") + message(STATUS "Fortran Compiler: ${CMAKE_Fortran_COMPILER} + Type: ${CMAKE_Fortran_COMPILER_ID} + Version: ${CMAKE_Fortran_COMPILER_VERSION} + Fortran Flags:${CMAKE_Fortran_FLAGS} ${CMAKE_Fortran_FLAGS_${BTYPE}}") endif() list (FIND LANGUAGES "C" _index) if (${_index} GREATER -1) - message(STATUS "C compiler ${CMAKE_C_COMPILER} - Type ${CMAKE_C_COMPILER_ID} - Version ${CMAKE_C_COMPILER_VERSION} - C Flags ${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BTYPE}}") + message(STATUS "C compiler: ${CMAKE_C_COMPILER} + Type: ${CMAKE_C_COMPILER_ID} + Version: ${CMAKE_C_COMPILER_VERSION} + C Flags: ${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BTYPE}}") endif() +message(STATUS "<<< Linker flags: >>>") +message(STATUS "Executable name: ${LAMMPS_BINARY}") if(CMAKE_EXE_LINKER_FLAGS) - message(STATUS "Linker flags: - Executable ${CMAKE_EXE_LINKER_FLAGS}") + message(STATUS "Executable linker flags: ${CMAKE_EXE_LINKER_FLAGS}") endif() if(BUILD_SHARED_LIBS) - message(STATUS "Shared library flags: ${CMAKE_SHARED_LINKER_FLAGS}") + message(STATUS "Shared library flags: ${CMAKE_SHARED_LINKER_FLAGS}") else() - message(STATUS "Static library flags: ${CMAKE_STATIC_LINKER_FLAGS}") + message(STATUS "Static library flags: ${CMAKE_STATIC_LINKER_FLAGS}") endif() -message(STATUS "Link libraries: ${LAMMPS_LINK_LIBS}") if(BUILD_MPI) - message(STATUS "Using MPI with headers in ${MPI_CXX_INCLUDE_PATH} and these libraries: ${MPI_CXX_LIBRARIES};${MPI_Fortran_LIBRARIES}") + message(STATUS "<<< MPI flags >>> +-- MPI includes: ${MPI_CXX_INCLUDE_PATH} +-- MPI libraries: ${MPI_CXX_LIBRARIES};${MPI_Fortran_LIBRARIES}") endif() if(PKG_GPU) - message(STATUS "GPU API: ${GPU_API}") + message(STATUS "<<< GPU package settings >>> +-- GPU API: ${GPU_API}") if(GPU_API STREQUAL "CUDA") message(STATUS "GPU architecture: ${GPU_ARCH}") elseif(GPU_API STREQUAL "OPENCL") - message(STATUS "OpenCL parameter tuning: ${OCL_TUNE}") + message(STATUS "OpenCL tuning: ${OCL_TUNE}") + elseif(GPU_API STREQUAL "HIP") + message(STATUS "HIP platform: ${HIP_PLATFORM}") + message(STATUS "HIP architecture: ${HIP_ARCH}") + if(HIP_USE_DEVICE_SORT) + message(STATUS "HIP GPU sorting: on") + else() + message(STATUS "HIP GPU sorting: off") + endif() endif() - message(STATUS "GPU precision: ${GPU_PREC}") + message(STATUS "GPU precision: ${GPU_PREC}") endif() if(PKG_KOKKOS) message(STATUS "Kokkos Arch: ${KOKKOS_ARCH}") endif() if(PKG_KSPACE) - message(STATUS "Using ${FFT} as primary FFT library") + message(STATUS "<<< FFT settings >>> +-- Primary FFT lib: ${FFT}") if(FFT_SINGLE) message(STATUS "Using single precision FFTs") else() @@ -762,7 +743,7 @@ if(PKG_KSPACE) message(STATUS "Using non-threaded FFTs") endif() if(PKG_KOKKOS) - if(KOKKOS_ENABLE_CUDA) + if(Kokkos_ENABLE_CUDA) if (${FFT} STREQUAL "KISS") message(STATUS "Kokkos FFT: KISS") else() diff --git a/cmake/FindLAMMPS.cmake.in b/cmake/FindLAMMPS.cmake.in deleted file mode 100644 index 586df83c2d080197ea0256794febd395ab81cc63..0000000000000000000000000000000000000000 --- a/cmake/FindLAMMPS.cmake.in +++ /dev/null @@ -1,48 +0,0 @@ -# - Find liblammps -# Find the native liblammps headers and libraries. -# -# The following variables will set: -# LAMMPS_INCLUDE_DIRS - where to find lammps/library.h, etc. -# LAMMPS_LIBRARIES - List of libraries when using lammps. -# LAMMPS_API_DEFINES - lammps library api defines -# LAMMPS_VERSION - lammps library version -# LAMMPS_FOUND - True if liblammps found. -# -# In addition a LAMMPS::LAMMPS imported target is getting created. -# -# LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator -# http://lammps.sandia.gov, Sandia National Laboratories -# Steve Plimpton, sjplimp@sandia.gov -# -# Copyright (2003) Sandia Corporation. Under the terms of Contract -# DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains -# certain rights in this software. This software is distributed under -# the GNU General Public License. -# -# See the README file in the top-level LAMMPS directory. -# - -find_package(PkgConfig) - -pkg_check_modules(PC_LAMMPS liblammps@LAMMPS_LIB_SUFFIX@) -find_path(LAMMPS_INCLUDE_DIR lammps/library.h HINTS ${PC_LAMMPS_INCLUDE_DIRS} @CMAKE_INSTALL_FULL_INCLUDEDIR@) - -set(LAMMPS_VERSION @LAMMPS_VERSION@) -set(LAMMPS_API_DEFINES @LAMMPS_API_DEFINES@) - -find_library(LAMMPS_LIBRARY NAMES lammps@LAMMPS_LIB_SUFFIX@ HINTS ${PC_LAMMPS_LIBRARY_DIRS} @CMAKE_INSTALL_FULL_LIBDIR@) - -set(LAMMPS_INCLUDE_DIRS "${LAMMPS_INCLUDE_DIR}") -set(LAMMPS_LIBRARIES "${LAMMPS_LIBRARY}") - -include(FindPackageHandleStandardArgs) -# handle the QUIETLY and REQUIRED arguments and set LAMMPS_FOUND to TRUE -# if all listed variables are TRUE -find_package_handle_standard_args(LAMMPS REQUIRED_VARS LAMMPS_LIBRARY LAMMPS_INCLUDE_DIR VERSION_VAR LAMMPS_VERSION) - -mark_as_advanced(LAMMPS_INCLUDE_DIR LAMMPS_LIBRARY) - -if(LAMMPS_FOUND AND NOT TARGET LAMMPS::LAMMPS) - add_library(LAMMPS::LAMMPS UNKNOWN IMPORTED) - set_target_properties(LAMMPS::LAMMPS PROPERTIES IMPORTED_LOCATION "${LAMMPS_LIBRARY}" INTERFACE_INCLUDE_DIRECTORIES "${LAMMPS_INCLUDE_DIR}" INTERFACE_COMPILE_DEFINITIONS "${LAMMPS_API_DEFINES}") -endif() diff --git a/cmake/LAMMPSConfig.cmake.in b/cmake/LAMMPSConfig.cmake.in new file mode 100644 index 0000000000000000000000000000000000000000..7cd782b3b8eb4585dd3eb4d1c75e852abf64fd29 --- /dev/null +++ b/cmake/LAMMPSConfig.cmake.in @@ -0,0 +1,87 @@ +set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}) +include(CMakeFindDependencyMacro) +if(@BUILD_MPI@) + find_dependency(MPI REQUIRED CXX) +endif() +if(@PKG_KSPACE@) + if(@FFT@ STREQUAL "FFTW3") + find_dependency(@FFTW@ REQUIRED MODULE) + endif() +endif() +if(NOT @BUILD_SHARED_LIBS@) + if(@BUILD_OMP@) + find_dependency(OpenMP REQUIRED) + endif() + if(@WITH_JPEG@) + find_dependency(JPEG REQUIRED) + endif() + if(@WITH_PNG@) + find_dependency(PNG REQUIRED) + find_dependency(ZLIB REQUIRED) + endif() + if(@PKG_KIM@) + find_dependency(PkgConfig REQUIRED) + pkg_check_modules(KIM-API REQUIRED IMPORTED_TARGET libkim-api>=@KIM-API_MIN_VERSION@) + if(@CURL_FOUND@) + find_dependency(CURL REQUIRED) + endif() + endif() + if(@PKG_USER-SMD@) + find_dependency(Eigen3 NO_MODULE REQUIRED) + endif() + if(@PKG_USER-SCAFACOS@) + find_dependency(PkgConfig REQUIRED) + find_ependency(GSL REQUIRED) + find_dependency(MPI REQUIRED C Fortran) + pkg_check_modules(SCAFACOS REQUIRED IMPORTED_TARGET scafacos) + endif() + if(@PKG_PYTHON@ AND NOT CMAKE_VERSION VERSION_LESS 3.12) + find_package(Python REQUIRED COMPONENTS Development) + endif() + if(@PKG_COMPRESS@) + find_dependency(ZLIB REQUIRED) + endif() + if(@PKG_KOKKOS@) + if(@EXTERNAL_KOKKOS@) + find_dependency(Kokkos 3 REQUIRED) + endif() + endif() + if(@PKG_VORONOI@) + find_dependency(VORO REQUIRED) + endif() + if(@PKG_USER-INTEL@) + if(@INTEL_LRT_MODE@ STREQUAL "THREADS") + find_dependency(Threads REQUIRED) + endif() + if(@TBB_MALLOC_FOUND@) + find_ependency(TBB_MALLOC REQUIRED) + endif() + endif() + if(@PKG_USER-ADIOS@) + find_ependency(ADIOS2 REQUIRED) + endif() + if(@PKG_LATTE@) + find_ependency(LATTE REQUIRED) + endif() + if(@PKG_MESSAGE@) + if(@MESSAGE_ZMQ@) + find_ependency(ZMQ REQUIRED) + endif() + endif() + if(@PKG_MSCG@) + find_ependency(GSL REQUIRED) + find_ependency(MSCG REQUIRED) + endif() + if(@USER-NETCDF@) + if(@NETCDF_FOUND@) + find_ependency(NetCDF REQUIRED) + endif() + if(@PNETCDF_FOUND@) + find_ependency(PNetCDF REQUIRED) + endif() + endif() + if(@PKG_QUIP@) + find_ependency(QUIP REQUIRED) + endif() +endif() +include("${CMAKE_CURRENT_LIST_DIR}/LAMMPS_Targets.cmake") diff --git a/cmake/Modules/Documentation.cmake b/cmake/Modules/Documentation.cmake index feff66a9b2520b4f54b214d7873337ae48269064..929d6ebd4bd9d7903494c5a1f4f5e5e82a934221 100644 --- a/cmake/Modules/Documentation.cmake +++ b/cmake/Modules/Documentation.cmake @@ -1,10 +1,8 @@ ############################################################################### # Build documentation ############################################################################### -option(BUILD_DOC "Build LAMMPS documentation" OFF) +option(BUILD_DOC "Build LAMMPS HTML documentation" OFF) if(BUILD_DOC) - include(ProcessorCount) - ProcessorCount(NPROCS) find_package(PythonInterp 3 REQUIRED) set(VIRTUALENV ${PYTHON_EXECUTABLE} -m virtualenv) @@ -26,15 +24,49 @@ if(BUILD_DOC) COMMAND ${DOCENV_BINARY_DIR}/pip install --upgrade ${LAMMPS_DOC_DIR}/utils/converters ) + # download mathjax distribution and unpack to folder "mathjax" + file(DOWNLOAD "https://github.com/mathjax/MathJax/archive/3.0.5.tar.gz" + "${CMAKE_CURRENT_BINARY_DIR}/mathjax.tar.gz" + EXPECTED_MD5 5d9d3799cce77a1a95eee6be04eb68e7) + + if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/mathjax) + execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf mathjax.tar.gz WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + file(GLOB MATHJAX_VERSION_DIR ${CMAKE_CURRENT_BINARY_DIR}/MathJax-*) + execute_process(COMMAND ${CMAKE_COMMAND} -E rename ${MATHJAX_VERSION_DIR} ${CMAKE_CURRENT_BINARY_DIR}/mathjax) + endif() + file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/html/_static/mathjax) + file(COPY ${CMAKE_CURRENT_BINARY_DIR}/mathjax/es5 DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/html/_static/mathjax/) + + # note, this may run in parallel with other tasks, so we must not use multiple processes here add_custom_command( OUTPUT html DEPENDS ${DOC_SOURCES} docenv requirements.txt - COMMAND ${DOCENV_BINARY_DIR}/sphinx-build -j ${NPROCS} -b html -c ${LAMMPS_DOC_DIR}/utils/sphinx-config -d ${CMAKE_BINARY_DIR}/doctrees ${LAMMPS_DOC_DIR}/src html + COMMAND ${DOCENV_BINARY_DIR}/sphinx-build -b html -c ${LAMMPS_DOC_DIR}/utils/sphinx-config -d ${CMAKE_BINARY_DIR}/doctrees ${LAMMPS_DOC_DIR}/src html ) + # copy selected image files to html output tree + file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/html/JPG) + set(HTML_EXTRA_IMAGES balance_nonuniform.jpg balance_rcb.jpg + balance_uniform.jpg bow_tutorial_01.png bow_tutorial_02.png + bow_tutorial_03.png bow_tutorial_04.png bow_tutorial_05.png + dump1.jpg dump2.jpg examples_mdpd.gif gran_funnel.png gran_mixer.png + hop1.jpg hop2.jpg saed_ewald_intersect.jpg saed_mesh.jpg + screenshot_atomeye.jpg screenshot_gl.jpg screenshot_pymol.jpg + screenshot_vmd.jpg sinusoid.jpg xrd_mesh.jpg) + set(HTML_IMAGE_TARGETS "") + foreach(_IMG ${HTML_EXTRA_IMAGES}) + string(PREPEND _IMG JPG/) + list(APPEND HTML_IMAGE_TARGETS "html/${_IMG}") + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/html/${_IMG} + DEPENDS ${LAMMPS_DOC_DIR}/src/${_IMG} html/JPG + COMMAND ${CMAKE_COMMAND} -E copy ${LAMMPS_DOC_DIR}/src/${_IMG} ${CMAKE_BINARY_DIR}/html/${_IMG} + ) + endforeach() + add_custom_target( doc ALL - DEPENDS html + DEPENDS html html/_static/mathjax/es5 ${HTML_IMAGE_TARGETS} SOURCES ${LAMMPS_DOC_DIR}/utils/requirements.txt ${DOC_SOURCES} ) diff --git a/cmake/Modules/FindCUB.cmake b/cmake/Modules/FindCUB.cmake new file mode 100644 index 0000000000000000000000000000000000000000..848e68e81512144ad56124bdf0dcbcce0f6fa50e --- /dev/null +++ b/cmake/Modules/FindCUB.cmake @@ -0,0 +1,16 @@ +# - Find CUB +# Find the CUB header library +# +# CUB_INCLUDE_DIRS - where to find cub/cub.cuh +# CUB_FOUND - True if CUB found. +# + +find_path(CUB_INCLUDE_DIR cub.cuh PATH_SUFFIXES cub) + +include(FindPackageHandleStandardArgs) +# handle the QUIETLY and REQUIRED arguments and set CUB_FOUND to TRUE +# if all listed variables are TRUE + +find_package_handle_standard_args(CUB DEFAULT_MSG CUB_INCLUDE_DIR) + +mark_as_advanced(CUB_INCLUDE_DIR) diff --git a/cmake/Modules/FindFFTW3.cmake b/cmake/Modules/FindFFTW3.cmake index 63752f85dfa889980598d3891e2dbc3c66be5c87..708ec495091352fded80d323b37942bdb17e086f 100644 --- a/cmake/Modules/FindFFTW3.cmake +++ b/cmake/Modules/FindFFTW3.cmake @@ -14,14 +14,34 @@ find_path(FFTW3_INCLUDE_DIR fftw3.h HINTS ${PC_FFTW3_INCLUDE_DIRS}) find_library(FFTW3_LIBRARY NAMES fftw3 HINTS ${PC_FFTW3_LIBRARY_DIRS}) find_library(FFTW3_OMP_LIBRARY NAMES fftw3_omp HINTS ${PC_FFTW3_LIBRARY_DIRS}) -set(FFTW3_INCLUDE_DIRS ${FFTW3_INCLUDE_DIR}) -set(FFTW3_LIBRARIES ${FFTW3_LIBRARY}) -set(FFTW3_OMP_LIBRARIES ${FFTW3_OMP_LIBRARY}) - include(FindPackageHandleStandardArgs) # handle the QUIETLY and REQUIRED arguments and set FFTW3_FOUND to TRUE # if all listed variables are TRUE find_package_handle_standard_args(FFTW3 DEFAULT_MSG FFTW3_LIBRARY FFTW3_INCLUDE_DIR) +# Copy the results to the output variables and target. +if(FFTW3_FOUND) + set(FFTW3_LIBRARIES ${FFTW3_LIBRARY} ) + set(FFTW3_INCLUDE_DIRS ${FFTW3_INCLUDE_DIR} ) + + if(NOT TARGET FFTW3::FFTW3) + add_library(FFTW3::FFTW3 UNKNOWN IMPORTED) + set_target_properties(FFTW3::FFTW3 PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION "${FFTW3_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${FFTW3_INCLUDE_DIRS}") + endif() + if(FFTW3_OMP_LIBRARY) + set(FFTW3_OMP_LIBRARIES ${FFTW3_OMP_LIBRARY}) + if(NOT TARGET FFTW3::FFTW3_OMP) + add_library(FFTW3::FFTW3_OMP UNKNOWN IMPORTED) + set_target_properties(FFTW3::FFTW3_OMP PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION "${FFTW3_OMP_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${FFTW3_INCLUDE_DIRS}") + endif() + endif() +endif() + mark_as_advanced(FFTW3_INCLUDE_DIR FFTW3_LIBRARY FFTW3_OMP_LIBRARY) diff --git a/cmake/Modules/FindFFTW3F.cmake b/cmake/Modules/FindFFTW3F.cmake index c67aa5faf1ca5b7a95eac870fa268f9fa2e26e2c..3dbcdaa04e190c79d80281e9f91fa166755f585a 100644 --- a/cmake/Modules/FindFFTW3F.cmake +++ b/cmake/Modules/FindFFTW3F.cmake @@ -13,14 +13,34 @@ find_path(FFTW3F_INCLUDE_DIR fftw3.h HINTS ${PC_FFTW3F_INCLUDE_DIRS}) find_library(FFTW3F_LIBRARY NAMES fftw3f HINTS ${PC_FFTW3F_LIBRARY_DIRS}) find_library(FFTW3F_OMP_LIBRARY NAMES fftw3f_omp HINTS ${PC_FFTW3F_LIBRARY_DIRS}) -set(FFTW3F_INCLUDE_DIRS ${FFTW3F_INCLUDE_DIR}) -set(FFTW3F_LIBRARIES ${FFTW3F_LIBRARY}) -set(FFTW3F_OMP_LIBRARIES ${FFTW3F_OMP_LIBRARY}) - include(FindPackageHandleStandardArgs) # handle the QUIETLY and REQUIRED arguments and set FFTW3F_FOUND to TRUE # if all listed variables are TRUE find_package_handle_standard_args(FFTW3F DEFAULT_MSG FFTW3F_LIBRARY FFTW3F_INCLUDE_DIR) -mark_as_advanced(FFTW3F_INCLUDE_DIR FFTW3F_LIBRARY) +# Copy the results to the output variables and target. +if(FFTW3F_FOUND) + set(FFTW3F_LIBRARIES ${FFTW3F_LIBRARY} ) + set(FFTW3F_INCLUDE_DIRS ${FFTW3F_INCLUDE_DIR} ) + + if(NOT TARGET FFTW3F::FFTW3F) + add_library(FFTW3F::FFTW3F UNKNOWN IMPORTED) + set_target_properties(FFTW3F::FFTW3F PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION "${FFTW3F_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${FFTW3F_INCLUDE_DIRS}") + endif() + if(FFTW3F_OMP_LIBRARY) + set(FFTW3F_OMP_LIBRARIES ${FFTW3F_OMP_LIBRARY}) + if(NOT TARGET FFTW3F::FFTW3F_OMP) + add_library(FFTW3F::FFTW3F_OMP UNKNOWN IMPORTED) + set_target_properties(FFTW3F::FFTW3F_OMP PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION "${FFTW3F_OMP_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${FFTW3F_INCLUDE_DIRS}") + endif() + endif() +endif() + +mark_as_advanced(FFTW3F_INCLUDE_DIR FFTW3F_LIBRARY FFTW3F_OMP_LIBRARY) diff --git a/cmake/Modules/FindKIM-API.cmake b/cmake/Modules/FindKIM-API.cmake deleted file mode 100644 index 17c195e62f515752d893118cd9a0d08e0ad8ae58..0000000000000000000000000000000000000000 --- a/cmake/Modules/FindKIM-API.cmake +++ /dev/null @@ -1,85 +0,0 @@ -# -# CDDL HEADER START -# -# The contents of this file are subject to the terms of the Common Development -# and Distribution License Version 1.0 (the "License"). -# -# You can obtain a copy of the license at -# http://www.opensource.org/licenses/CDDL-1.0. See the License for the -# specific language governing permissions and limitations under the License. -# -# When distributing Covered Code, include this CDDL HEADER in each file and -# include the License file in a prominent location with the name LICENSE.CDDL. -# If applicable, add the following below this CDDL HEADER, with the fields -# enclosed by brackets "[]" replaced with your own identifying information: -# -# Portions Copyright (c) [yyyy] [name of copyright owner]. All rights reserved. -# -# CDDL HEADER END -# - -# -# Copyright (c) 2013--2019, Regents of the University of Minnesota. -# All rights reserved. -# -# Contributors: -# Richard Berger -# Christoph Junghans -# Ryan S. Elliott -# - -# - Find KIM-API -# -# sets standard pkg_check_modules variables plus: -# -# KIM-API-CMAKE_C_COMPILER -# KIM-API-CMAKE_CXX_COMPILER -# KIM-API-CMAKE_Fortran_COMPILER -# - -function(_KIMAPI_GET_VERSION _OUT_ver _version_hdr) - if(NOT EXISTS ${_version_hdr}) - message(FATAL_ERROR "Header file ${_version_hdr} not found (check value of KIM-API_INCLUDE_DIR)") - endif() - foreach(_var KIM_VERSION_MAJOR KIM_VERSION_MINOR KIM_VERSION_PATCH) - file(STRINGS ${_version_hdr} _contents REGEX "#define ${_var}[ \t]+") - if(_contents) - string(REGEX REPLACE ".*#define ${_var}[ \t]+([0-9]+).*" "\\1" _${_var} "${_contents}") - if(${_${_var}} STREQUAL "") - message(FATAL_ERROR "Version parsing failed for ${_var} in ${_version_hdr}, got empty return!") - elseif(NOT ${_${_var}} MATCHES "^[0-9]+$") - message(FATAL_ERROR "Version parsing failed for ${_var} in ${_version_hdr}, excepted a number but got ${_${_var}}!") - endif() - else() - message(FATAL_ERROR "No ${_var} line found in include file ${_version_hdr}") - endif() - endforeach() - set(${_OUT_ver} ${_KIM_VERSION_MAJOR}.${_KIM_VERSION_MINOR}.${_KIM_VERSION_PATCH} PARENT_SCOPE) -endfunction() - -if(KIM-API_FIND_QUIETLY) - set(REQ_OR_QUI "QUIET") -else() - set(REQ_OR_QUI "REQUIRED") -endif() - -find_package(PkgConfig ${REQ_OR_QUI}) -include(FindPackageHandleStandardArgs) - -pkg_check_modules(KIM-API ${REQ_OR_QUI} libkim-api>=2.0) - -if(KIM-API_FOUND) - pkg_get_variable(KIM-API-CMAKE_C_COMPILER libkim-api CMAKE_C_COMPILER) - pkg_get_variable(KIM-API-CMAKE_CXX_COMPILER libkim-api CMAKE_CXX_COMPILER) - pkg_get_variable(KIM-API_CMAKE_Fortran_COMPILER libkim-api CMAKE_Fortran_COMPILER) -endif() - -if(KIM-API_INCLUDEDIR) - _KIMAPI_GET_VERSION(KIM-API_VERSION ${KIM-API_INCLUDEDIR}/KIM_Version.h) -else() - set(KIM-API_VERSION 0) -endif() - -# handle the QUIETLY and REQUIRED arguments and set KIM-API_FOUND to TRUE -# if all listed variables are TRUE -find_package_handle_standard_args(KIM-API REQUIRED_VARS KIM-API_LIBRARIES VERSION_VAR KIM-API_VERSION) diff --git a/cmake/Modules/FindLATTE.cmake b/cmake/Modules/FindLATTE.cmake index 74d5173bf0725ae74f0eaf08214b8b593a74e473..ac5c639b79313019ba9880f1f178ce1449034ef1 100644 --- a/cmake/Modules/FindLATTE.cmake +++ b/cmake/Modules/FindLATTE.cmake @@ -7,12 +7,21 @@ find_library(LATTE_LIBRARY NAMES latte) -set(LATTE_LIBRARIES ${LATTE_LIBRARY}) - include(FindPackageHandleStandardArgs) # handle the QUIETLY and REQUIRED arguments and set LATTE_FOUND to TRUE # if all listed variables are TRUE find_package_handle_standard_args(LATTE DEFAULT_MSG LATTE_LIBRARY) +# Copy the results to the output variables and target. +if(LATTE_FOUND) + set(LATTE_LIBRARIES ${LATTE_LIBRARY}) + + if(NOT TARGET LATTE::latte) + add_library(LATTE::latte UNKNOWN IMPORTED) + set_target_properties(LATTE::latte PROPERTIES + IMPORTED_LOCATION "${LATTE_LIBRARY}") + endif() +endif() + mark_as_advanced(LATTE_LIBRARY) diff --git a/cmake/Modules/FindMKL.cmake b/cmake/Modules/FindMKL.cmake index 4246062103f943c2c54347c2cbc796db96cff54e..e3dc60ad9749a2c0d5ddf6e1c30132af03a6c00e 100644 --- a/cmake/Modules/FindMKL.cmake +++ b/cmake/Modules/FindMKL.cmake @@ -10,13 +10,22 @@ find_path(MKL_INCLUDE_DIR mkl_dfti.h HINTS $ENV{MKLROOT}/include) find_library(MKL_LIBRARY NAMES mkl_rt HINTS $ENV{MKLROOT}/lib $ENV{MKLROOT}/lib/intel64) -set(MKL_LIBRARIES ${MKL_LIBRARY}) -set(MKL_INCLUDE_DIRS ${MKL_INCLUDE_DIR}) - include(FindPackageHandleStandardArgs) # handle the QUIETLY and REQUIRED arguments and set MKL_FOUND to TRUE # if all listed variables are TRUE find_package_handle_standard_args(MKL DEFAULT_MSG MKL_LIBRARY MKL_INCLUDE_DIR) +if(MKL_FOUND) + set(MKL_LIBRARIES ${MKL_LIBRARY}) + set(MKL_INCLUDE_DIRS ${MKL_INCLUDE_DIR}) + + if(NOT TARGET MKL::MKL) + add_library(MKL::MKL UNKNOWN IMPORTED) + set_target_properties(MKL::MKL PROPERTIES + IMPORTED_LOCATION "${MKL_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${MKL_INCLUDE_DIR}") + endif() +endif() + mark_as_advanced(MKL_INCLUDE_DIR MKL_LIBRARY ) diff --git a/cmake/Modules/FindMSCG.cmake b/cmake/Modules/FindMSCG.cmake index 311ff7803836aa43c2c48bd4040a60686b1803db..b301303dbec9dbdb186d44b39f806d1c60e2bc45 100644 --- a/cmake/Modules/FindMSCG.cmake +++ b/cmake/Modules/FindMSCG.cmake @@ -10,13 +10,23 @@ find_path(MSCG_INCLUDE_DIR mscg.h PATH_SUFFIXES mscg) find_library(MSCG_LIBRARY NAMES mscg) -set(MSCG_LIBRARIES ${MSCG_LIBRARY}) -set(MSCG_INCLUDE_DIRS ${MSCG_INCLUDE_DIR}) - include(FindPackageHandleStandardArgs) # handle the QUIETLY and REQUIRED arguments and set MSCG_FOUND to TRUE # if all listed variables are TRUE find_package_handle_standard_args(MSCG DEFAULT_MSG MSCG_LIBRARY MSCG_INCLUDE_DIR) +# Copy the results to the output variables and target. +if(MSCG_FOUND) + set(MSCG_LIBRARIES ${MSCG_LIBRARY}) + set(MSCG_INCLUDE_DIRS ${MSCG_INCLUDE_DIR}) + + if(NOT TARGET MSCG::MSCG) + add_library(MSCG::MSCG UNKNOWN IMPORTED) + set_target_properties(MSCG::MSCG PROPERTIES + IMPORTED_LOCATION "${MSCG_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${MSCG_INCLUDE_DIR}") + endif() +endif() + mark_as_advanced(MSCG_INCLUDE_DIR MSCG_LIBRARY ) diff --git a/cmake/Modules/FindNetCDF.cmake b/cmake/Modules/FindNetCDF.cmake index 2a992b6b3b532513d1f2886cd5e76aeaba32df20..5c8caa920820da14855e51ab6032b3d7a4e775b7 100644 --- a/cmake/Modules/FindNetCDF.cmake +++ b/cmake/Modules/FindNetCDF.cmake @@ -120,3 +120,14 @@ set (NETCDF_INCLUDE_DIRS ${NetCDF_includes}) include (FindPackageHandleStandardArgs) find_package_handle_standard_args (NetCDF DEFAULT_MSG NETCDF_LIBRARIES NETCDF_INCLUDE_DIRS NETCDF_HAS_INTERFACES) + +# Copy the results to the output variables and target. +if(NetCDF_FOUND) + if(NOT TARGET NetCDF::NetCDF) + add_library(NetCDF::NetCDF UNKNOWN IMPORTED) + set_target_properties(NetCDF::NetCDF PROPERTIES + IMPORTED_LOCATION "${NETCDF_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${NETCDF_INCLUDE_DIRS}" + INTERFACE_LINK_LIBRARIES "${NETCDF_LIBRARIES}") + endif() +endif() diff --git a/cmake/Modules/FindPNetCDF.cmake b/cmake/Modules/FindPNetCDF.cmake index bc3a5f9538b17c82201437f3ee378e9be0bf49fa..e501e9af4920553f638fb12132f84e495dc9e959 100644 --- a/cmake/Modules/FindPNetCDF.cmake +++ b/cmake/Modules/FindPNetCDF.cmake @@ -53,3 +53,12 @@ include (FindPackageHandleStandardArgs) find_package_handle_standard_args (PNetCDF DEFAULT_MSG PNETCDF_LIBRARIES PNETCDF_INCLUDES) mark_as_advanced (PNETCDF_LIBRARIES PNETCDF_INCLUDES) + +if(PNetCDF_FOUND) + if(NOT TARGET PNetCDF::PNetCDF) + add_library(PNetCDF::PNetCDF UNKNOWN IMPORTED) + set_target_properties(PNetCDF::PNetCDF PROPERTIES + IMPORTED_LOCATION "${PNETCDF_LIBRARIES}" + INTERFACE_INCLUDE_DIRECTORIES "${PNETCDF_INCLUDES}") + endif() +endif() diff --git a/cmake/Modules/FindQUIP.cmake b/cmake/Modules/FindQUIP.cmake index b6d87d11fa1cc82fd65452074a48e7469f5d39b4..277cfae49ed2a31b03a094c79a04a6d980e9a19d 100644 --- a/cmake/Modules/FindQUIP.cmake +++ b/cmake/Modules/FindQUIP.cmake @@ -7,12 +7,21 @@ find_library(QUIP_LIBRARY NAMES quip) -set(QUIP_LIBRARIES ${QUIP_LIBRARY}) - include(FindPackageHandleStandardArgs) # handle the QUIETLY and REQUIRED arguments and set QUIP_FOUND to TRUE # if all listed variables are TRUE find_package_handle_standard_args(QUIP DEFAULT_MSG QUIP_LIBRARY) +# Copy the results to the output variables and target. +if(QUIP_FOUND) + set(QUIP_LIBRARIES ${QUIP_LIBRARY}) + + if(NOT TARGET QUIP::QUIP) + add_library(QUIP::QUIP UNKNOWN IMPORTED) + set_target_properties(QUIP::QUIP PROPERTIES + IMPORTED_LOCATION "${QUIP_LIBRARY}") + endif() +endif() + mark_as_advanced(QUIP_LIBRARY) diff --git a/cmake/Modules/FindTBB.cmake b/cmake/Modules/FindTBB.cmake deleted file mode 100644 index 8cc050817e6b919d3d77935afcd26f1fd4d1cce0..0000000000000000000000000000000000000000 --- a/cmake/Modules/FindTBB.cmake +++ /dev/null @@ -1,46 +0,0 @@ -# - Find parts of TBB -# Find the native TBB headers and libraries. -# -# TBB_INCLUDE_DIRS - where to find tbb.h, etc. -# TBB_LIBRARIES - List of libraries when using tbb. -# TBB_FOUND - True if tbb found. -# - -######################################################## -# TBB - -# TODO use more generic FindTBB - -find_path(TBB_INCLUDE_DIR NAMES tbb/tbb.h PATHS $ENV{TBBROOT}/include) -find_library(TBB_LIBRARY NAMES tbb PATHS $ENV{TBBROOT}/lib/intel64/gcc4.7 - $ENV{TBBROOT}/lib/intel64/gcc4.4 - $ENV{TBBROOT}/lib/intel64/gcc4.1) -set(TBB_LIBRARIES ${TBB_LIBRARY}) -set(TBB_INCLUDE_DIRS ${TBB_INCLUDE_DIR}) - -include(FindPackageHandleStandardArgs) -# handle the QUIETLY and REQUIRED arguments and set TBB_FOUND to TRUE -# if all listed variables are TRUE - -find_package_handle_standard_args(TBB DEFAULT_MSG TBB_LIBRARY TBB_INCLUDE_DIR) - -mark_as_advanced(TBB_INCLUDE_DIR TBB_LIBRARY ) - -######################################################## -# TBB Malloc - -find_path(TBB_MALLOC_INCLUDE_DIR NAMES tbb/tbb.h PATHS $ENV{TBBROOT}/include) -find_library(TBB_MALLOC_LIBRARY NAMES tbbmalloc PATHS $ENV{TBBROOT}/lib/intel64/gcc4.7 - $ENV{TBBROOT}/lib/intel64/gcc4.4 - $ENV{TBBROOT}/lib/intel64/gcc4.1) - -set(TBB_MALLOC_LIBRARIES ${TBB_MALLOC_LIBRARY}) -set(TBB_MALLOC_INCLUDE_DIRS ${TBB_MALLOC_INCLUDE_DIR}) - -include(FindPackageHandleStandardArgs) -# handle the QUIETLY and REQUIRED arguments and set TBB_MALLOC_FOUND to TRUE -# if all listed variables are TRUE - -find_package_handle_standard_args(TBB_MALLOC DEFAULT_MSG TBB_MALLOC_LIBRARY TBB_MALLOC_INCLUDE_DIR) - -mark_as_advanced(TBB_MALLOC_INCLUDE_DIR TBB_MALLOC_LIBRARY ) diff --git a/cmake/Modules/FindTBB_MALLOC.cmake b/cmake/Modules/FindTBB_MALLOC.cmake new file mode 100644 index 0000000000000000000000000000000000000000..896db649af75456c0f4bee348b572091b74e66f1 --- /dev/null +++ b/cmake/Modules/FindTBB_MALLOC.cmake @@ -0,0 +1,36 @@ +# - Find parts of TBB_MALLOC +# Find the native TBB_MALLOC headers and libraries. +# +# TBB_MALLOC_INCLUDE_DIRS - where to find tbb.h, etc. +# TBB_MALLOC_LIBRARIES - List of libraries when using tbb. +# TBB_MALLOC_FOUND - True if tbb found. +# + + +######################################################## +# TBB Malloc + +find_path(TBB_MALLOC_INCLUDE_DIR NAMES tbb/tbb.h PATHS $ENV{TBBROOT}/include) +find_library(TBB_MALLOC_LIBRARY NAMES tbbmalloc PATHS $ENV{TBBROOT}/lib/intel64/gcc4.7 + $ENV{TBBROOT}/lib/intel64/gcc4.4 + $ENV{TBBROOT}/lib/intel64/gcc4.1) + +include(FindPackageHandleStandardArgs) +# handle the QUIETLY and REQUIRED arguments and set TBB_MALLOC_FOUND to TRUE +# if all listed variables are TRUE + +find_package_handle_standard_args(TBB_MALLOC DEFAULT_MSG TBB_MALLOC_LIBRARY TBB_MALLOC_INCLUDE_DIR) + +if(TBB_MALLOC_FOUND) + set(TBB_MALLOC_LIBRARIES ${TBB_MALLOC_LIBRARY}) + set(TBB_MALLOC_INCLUDE_DIRS ${TBB_MALLOC_INCLUDE_DIR}) + + if(NOT TARGET TBB::TBB_MALLOC) + add_library(TBB::TBB_MALLOC UNKNOWN IMPORTED) + set_target_properties(TBB::TBB_MALLOC PROPERTIES + IMPORTED_LOCATION "${TBB_MALLOC_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${TBB_MALLOC_INCLUDE_DIR}") + endif() +endif() + +mark_as_advanced(TBB_MALLOC_INCLUDE_DIR TBB_MALLOC_LIBRARY ) diff --git a/cmake/Modules/FindVORO.cmake b/cmake/Modules/FindVORO.cmake index b0cccbcd1de79e30c9a18065654d141d9954844a..3f0fe98ff19bf67f684041d832a797d12adaab28 100644 --- a/cmake/Modules/FindVORO.cmake +++ b/cmake/Modules/FindVORO.cmake @@ -10,13 +10,23 @@ find_path(VORO_INCLUDE_DIR voro++.hh PATH_SUFFIXES voro++) find_library(VORO_LIBRARY NAMES voro++) -set(VORO_LIBRARIES ${VORO_LIBRARY}) -set(VORO_INCLUDE_DIRS ${VORO_INCLUDE_DIR}) - include(FindPackageHandleStandardArgs) # handle the QUIETLY and REQUIRED arguments and set VORO_FOUND to TRUE # if all listed variables are TRUE find_package_handle_standard_args(VORO DEFAULT_MSG VORO_LIBRARY VORO_INCLUDE_DIR) +# Copy the results to the output variables and target. +if(VORO_FOUND) + set(VORO_LIBRARIES ${VORO_LIBRARY}) + set(VORO_INCLUDE_DIRS ${VORO_INCLUDE_DIR}) + + if(NOT TARGET VORO::VORO) + add_library(VORO::VORO UNKNOWN IMPORTED) + set_target_properties(VORO::VORO PROPERTIES + IMPORTED_LOCATION "${VORO_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${VORO_INCLUDE_DIR}") + endif() +endif() + mark_as_advanced(VORO_INCLUDE_DIR VORO_LIBRARY ) diff --git a/cmake/Modules/FindZMQ.cmake b/cmake/Modules/FindZMQ.cmake index 608ccda777336f9d84dfabd40a9c6bdba1bd12be..7d612c2eb38d519c61cc95d0eb123309332e26a9 100644 --- a/cmake/Modules/FindZMQ.cmake +++ b/cmake/Modules/FindZMQ.cmake @@ -1,8 +1,19 @@ find_path(ZMQ_INCLUDE_DIR zmq.h) find_library(ZMQ_LIBRARY NAMES zmq) -set(ZMQ_LIBRARIES ${ZMQ_LIBRARY}) -set(ZMQ_INCLUDE_DIRS ${ZMQ_INCLUDE_DIR}) - include(FindPackageHandleStandardArgs) find_package_handle_standard_args(ZMQ DEFAULT_MSG ZMQ_LIBRARY ZMQ_INCLUDE_DIR) + +# Copy the results to the output variables and target. +if(ZMQ_FOUND) + set(ZMQ_LIBRARIES ${ZMQ_LIBRARY}) + set(ZMQ_INCLUDE_DIRS ${ZMQ_INCLUDE_DIR}) + + if(NOT TARGET ZMQ::ZMQ) + add_library(ZMQ::ZMQ UNKNOWN IMPORTED) + set_target_properties(ZMQ::ZMQ PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION "${ZMQ_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${ZMQ_INCLUDE_DIR}") + endif() +endif() diff --git a/cmake/Modules/GenerateBinaryHeader.cmake b/cmake/Modules/GenerateBinaryHeader.cmake new file mode 100644 index 0000000000000000000000000000000000000000..512f69e09de985b5ae70cb9bd8bfcb1b4bae0ca7 --- /dev/null +++ b/cmake/Modules/GenerateBinaryHeader.cmake @@ -0,0 +1,3 @@ +# utility script to call GenerateBinaryHeader function +include(${SOURCE_DIR}/Modules/LAMMPSUtils.cmake) +GenerateBinaryHeader(${VARNAME} ${HEADER_FILE} ${SOURCE_FILES}) diff --git a/cmake/Modules/LAMMPSUtils.cmake b/cmake/Modules/LAMMPSUtils.cmake index 3ea2b3cb7ec977175d026672c1a03f386fc86e61..02de630c82d320b67d144ea39eb30c1c44ed2c77 100644 --- a/cmake/Modules/LAMMPSUtils.cmake +++ b/cmake/Modules/LAMMPSUtils.cmake @@ -69,3 +69,19 @@ macro(pkg_depends PKG1 PKG2) message(FATAL_ERROR "${PKG1} package needs LAMMPS to be build with ${PKG2}") endif() endmacro() + +# CMake-only replacement for bin2c and xxd +function(GenerateBinaryHeader varname outfile files) + message("Creating ${outfile}...") + file(WRITE ${outfile} "// CMake generated file\n") + math(EXPR ARG_END "${ARGC}-1") + + foreach(IDX RANGE 2 ${ARG_END}) + list(GET ARGV ${IDX} filename) + file(READ ${filename} content HEX) + string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\1," content "${content}") + string(REGEX REPLACE ",$" "" content "${content}") + file(APPEND ${outfile} "const unsigned char ${varname}[] = { ${content} };\n") + file(APPEND ${outfile} "const unsigned int ${varname}_size = sizeof(${varname});\n") + endforeach() +endfunction(GenerateBinaryHeader) diff --git a/cmake/Modules/MPI4WIN.cmake b/cmake/Modules/MPI4WIN.cmake index 1f3195041aa7cb26bae48eb671d55b05b6a7dec3..19d2d2a13e3e94b66bb24ec4be280b1006535bba 100644 --- a/cmake/Modules/MPI4WIN.cmake +++ b/cmake/Modules/MPI4WIN.cmake @@ -16,8 +16,10 @@ else() endif() ExternalProject_get_property(mpi4win_build SOURCE_DIR) -add_definitions(-DMPICH_SKIP_MPICXX) -include_directories("${SOURCE_DIR}/include") -set(MPI4WIN_LIBRARIES "${SOURCE_DIR}/lib/libmpi.a") -list(APPEND LAMMPS_DEPS mpi4win_build) -set(LAMMPS_USE_MPI4WIN ON) +file(MAKE_DIRECTORY "${SOURCE_DIR}/include") +add_library(MPI::MPI_CXX UNKNOWN IMPORTED) +set_target_properties(MPI::MPI_CXX PROPERTIES + IMPORTED_LOCATION "${SOURCE_DIR}/lib/libmpi.a" + INTERFACE_INCLUDE_DIRECTORIES "${SOURCE_DIR}/include" + INTERFACE_COMPILE_DEFINITIONS "MPICH_SKIP_MPICXX") +add_dependencies(MPI::MPI_CXX mpi4win_build) diff --git a/cmake/Modules/Packages/COMPRESS.cmake b/cmake/Modules/Packages/COMPRESS.cmake index 864b868865f531ebcce5374bceed2612fb37d391..801a38a21561bde85fb2c031a96786043bd858b4 100644 --- a/cmake/Modules/Packages/COMPRESS.cmake +++ b/cmake/Modules/Packages/COMPRESS.cmake @@ -1,5 +1,2 @@ -if(PKG_COMPRESS) - find_package(ZLIB REQUIRED) - include_directories(${ZLIB_INCLUDE_DIRS}) - list(APPEND LAMMPS_LINK_LIBS ${ZLIB_LIBRARIES}) -endif() +find_package(ZLIB REQUIRED) +target_link_libraries(lammps PRIVATE ZLIB::ZLIB) diff --git a/cmake/Modules/Packages/CORESHELL.cmake b/cmake/Modules/Packages/CORESHELL.cmake index 591477c89948c3d34f975e8cdf65181b96af9171..4e12d2ee363e80377fb7e8eb44a2f073c1317027 100644 --- a/cmake/Modules/Packages/CORESHELL.cmake +++ b/cmake/Modules/Packages/CORESHELL.cmake @@ -1,13 +1,11 @@ -if(PKG_CORESHELL) - set(CORESHELL_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/CORESHELL) - set(CORESHELL_SOURCES) - set_property(GLOBAL PROPERTY "CORESHELL_SOURCES" "${CORESHELL_SOURCES}") + set(CORESHELL_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/CORESHELL) + set(CORESHELL_SOURCES) + set_property(GLOBAL PROPERTY "CORESHELL_SOURCES" "${CORESHELL_SOURCES}") - # detects styles which have a CORESHELL version - RegisterStylesExt(${CORESHELL_SOURCES_DIR} cs CORESHELL_SOURCES) + # detects styles which have a CORESHELL version + RegisterStylesExt(${CORESHELL_SOURCES_DIR} cs CORESHELL_SOURCES) - get_property(CORESHELL_SOURCES GLOBAL PROPERTY CORESHELL_SOURCES) + get_property(CORESHELL_SOURCES GLOBAL PROPERTY CORESHELL_SOURCES) - list(APPEND LIB_SOURCES ${CORESHELL_SOURCES}) - include_directories(${CORESHELL_SOURCES_DIR}) -endif() + target_sources(lammps PRIVATE ${CORESHELL_SOURCES}) + target_include_directories(lammps PRIVATE ${CORESHELL_SOURCES_DIR}) diff --git a/cmake/Modules/Packages/GPU.cmake b/cmake/Modules/Packages/GPU.cmake index abbcb1f495f1f1e3fe22db53e046ec543e5f40cd..574be941f71df26ee6dbf2f8e8e1990031e4318d 100644 --- a/cmake/Modules/Packages/GPU.cmake +++ b/cmake/Modules/Packages/GPU.cmake @@ -1,197 +1,363 @@ -if(PKG_GPU) - set(GPU_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/GPU) - set(GPU_SOURCES ${GPU_SOURCES_DIR}/gpu_extra.h - ${GPU_SOURCES_DIR}/fix_gpu.h - ${GPU_SOURCES_DIR}/fix_gpu.cpp) - - set(GPU_API "opencl" CACHE STRING "API used by GPU package") - set(GPU_API_VALUES opencl cuda) - set_property(CACHE GPU_API PROPERTY STRINGS ${GPU_API_VALUES}) - validate_option(GPU_API GPU_API_VALUES) - string(TOUPPER ${GPU_API} GPU_API) - - set(GPU_PREC "mixed" CACHE STRING "LAMMPS GPU precision") - set(GPU_PREC_VALUES double mixed single) - set_property(CACHE GPU_PREC PROPERTY STRINGS ${GPU_PREC_VALUES}) - validate_option(GPU_PREC GPU_PREC_VALUES) - string(TOUPPER ${GPU_PREC} GPU_PREC) - - if(GPU_PREC STREQUAL "DOUBLE") - set(GPU_PREC_SETTING "DOUBLE_DOUBLE") - elseif(GPU_PREC STREQUAL "MIXED") - set(GPU_PREC_SETTING "SINGLE_DOUBLE") - elseif(GPU_PREC STREQUAL "SINGLE") - set(GPU_PREC_SETTING "SINGLE_SINGLE") - endif() - - file(GLOB GPU_LIB_SOURCES ${LAMMPS_LIB_SOURCE_DIR}/gpu/[^.]*.cpp) - file(MAKE_DIRECTORY ${LAMMPS_LIB_BINARY_DIR}/gpu) +set(GPU_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/GPU) +set(GPU_SOURCES ${GPU_SOURCES_DIR}/gpu_extra.h + ${GPU_SOURCES_DIR}/fix_gpu.h + ${GPU_SOURCES_DIR}/fix_gpu.cpp) + +set(GPU_API "opencl" CACHE STRING "API used by GPU package") +set(GPU_API_VALUES opencl cuda hip) +set_property(CACHE GPU_API PROPERTY STRINGS ${GPU_API_VALUES}) +validate_option(GPU_API GPU_API_VALUES) +string(TOUPPER ${GPU_API} GPU_API) + +set(GPU_PREC "mixed" CACHE STRING "LAMMPS GPU precision") +set(GPU_PREC_VALUES double mixed single) +set_property(CACHE GPU_PREC PROPERTY STRINGS ${GPU_PREC_VALUES}) +validate_option(GPU_PREC GPU_PREC_VALUES) +string(TOUPPER ${GPU_PREC} GPU_PREC) + +if(GPU_PREC STREQUAL "DOUBLE") + set(GPU_PREC_SETTING "DOUBLE_DOUBLE") +elseif(GPU_PREC STREQUAL "MIXED") + set(GPU_PREC_SETTING "SINGLE_DOUBLE") +elseif(GPU_PREC STREQUAL "SINGLE") + set(GPU_PREC_SETTING "SINGLE_SINGLE") +endif() - if(GPU_API STREQUAL "CUDA") - find_package(CUDA REQUIRED) - find_program(BIN2C bin2c) - if(NOT BIN2C) - message(FATAL_ERROR "Could not find bin2c, use -DBIN2C=/path/to/bin2c to help cmake finding it.") +file(GLOB GPU_LIB_SOURCES ${LAMMPS_LIB_SOURCE_DIR}/gpu/[^.]*.cpp) +file(MAKE_DIRECTORY ${LAMMPS_LIB_BINARY_DIR}/gpu) + +if(GPU_API STREQUAL "CUDA") + find_package(CUDA REQUIRED) + find_program(BIN2C bin2c) + if(NOT BIN2C) + message(FATAL_ERROR "Could not find bin2c, use -DBIN2C=/path/to/bin2c to help cmake finding it.") + endif() + option(CUDPP_OPT "Enable CUDPP_OPT" ON) + option(CUDA_MPS_SUPPORT "Enable tweaks to support CUDA Multi-process service (MPS)" OFF) + if(CUDA_MPS_SUPPORT) + set(GPU_CUDA_MPS_FLAGS "-DCUDA_PROXY") + endif() + + set(GPU_ARCH "sm_50" CACHE STRING "LAMMPS GPU CUDA SM primary architecture (e.g. sm_60)") + + file(GLOB GPU_LIB_CU ${LAMMPS_LIB_SOURCE_DIR}/gpu/[^.]*.cu ${CMAKE_CURRENT_SOURCE_DIR}/gpu/[^.]*.cu) + list(REMOVE_ITEM GPU_LIB_CU ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_pppm.cu) + + cuda_include_directories(${LAMMPS_LIB_SOURCE_DIR}/gpu ${LAMMPS_LIB_BINARY_DIR}/gpu) + + if(CUDPP_OPT) + cuda_include_directories(${LAMMPS_LIB_SOURCE_DIR}/gpu/cudpp_mini) + file(GLOB GPU_LIB_CUDPP_SOURCES ${LAMMPS_LIB_SOURCE_DIR}/gpu/cudpp_mini/[^.]*.cpp) + file(GLOB GPU_LIB_CUDPP_CU ${LAMMPS_LIB_SOURCE_DIR}/gpu/cudpp_mini/[^.]*.cu) + endif() + + # build arch/gencode commands for nvcc based on CUDA toolkit version and use choice + # --arch translates directly instead of JIT, so this should be for the preferred or most common architecture + set(GPU_CUDA_GENCODE "-arch=${GPU_ARCH} ") + # Fermi (GPU Arch 2.x) is supported by CUDA 3.2 to CUDA 8.0 + if((CUDA_VERSION VERSION_GREATER "3.1") AND (CUDA_VERSION VERSION_LESS "9.0")) + string(APPEND GPU_CUDA_GENCODE "-gencode arch=compute_20,code=[sm_20,compute_20] ") + endif() + # Kepler (GPU Arch 3.x) is supported by CUDA 5 and later + if(CUDA_VERSION VERSION_GREATER "4.9") + string(APPEND GPU_CUDA_GENCODE "-gencode arch=compute_30,code=[sm_30,compute_30] -gencode arch=compute_35,code=[sm_35,compute_35] ") + endif() + # Maxwell (GPU Arch 5.x) is supported by CUDA 6 and later + if(CUDA_VERSION VERSION_GREATER "5.9") + string(APPEND GPU_CUDA_GENCODE "-gencode arch=compute_50,code=[sm_50,compute_50] -gencode arch=compute_52,code=[sm_52,compute_52] ") + endif() + # Pascal (GPU Arch 6.x) is supported by CUDA 8 and later + if(CUDA_VERSION VERSION_GREATER "7.9") + string(APPEND GPU_CUDA_GENCODE "-gencode arch=compute_60,code=[sm_60,compute_60] -gencode arch=compute_61,code=[sm_61,compute_61] ") + endif() + # Volta (GPU Arch 7.0) is supported by CUDA 9 and later + if(CUDA_VERSION VERSION_GREATER "8.9") + string(APPEND GPU_CUDA_GENCODE "-gencode arch=compute_70,code=[sm_70,compute_70] ") + endif() + # Turing (GPU Arch 7.5) is supported by CUDA 10 and later + if(CUDA_VERSION VERSION_GREATER "9.9") + string(APPEND GPU_CUDA_GENCODE "-gencode arch=compute_75,code=[sm_75,compute_75] ") + endif() + + cuda_compile_fatbin(GPU_GEN_OBJS ${GPU_LIB_CU} OPTIONS + -DUNIX -O3 --use_fast_math -Wno-deprecated-gpu-targets -DNV_KERNEL -DUCL_CUDADR ${GPU_CUDA_GENCODE} -D_${GPU_PREC_SETTING}) + + cuda_compile(GPU_OBJS ${GPU_LIB_CUDPP_CU} OPTIONS ${CUDA_REQUEST_PIC} + -DUNIX -O3 --use_fast_math -Wno-deprecated-gpu-targets -DUCL_CUDADR ${GPU_CUDA_GENCODE} -D_${GPU_PREC_SETTING}) + + foreach(CU_OBJ ${GPU_GEN_OBJS}) + get_filename_component(CU_NAME ${CU_OBJ} NAME_WE) + string(REGEX REPLACE "^.*_lal_" "" CU_NAME "${CU_NAME}") + add_custom_command(OUTPUT ${LAMMPS_LIB_BINARY_DIR}/gpu/${CU_NAME}_cubin.h + COMMAND ${BIN2C} -c -n ${CU_NAME} ${CU_OBJ} > ${LAMMPS_LIB_BINARY_DIR}/gpu/${CU_NAME}_cubin.h + DEPENDS ${CU_OBJ} + COMMENT "Generating ${CU_NAME}_cubin.h") + list(APPEND GPU_LIB_SOURCES ${LAMMPS_LIB_BINARY_DIR}/gpu/${CU_NAME}_cubin.h) + endforeach() + set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${LAMMPS_LIB_BINARY_DIR}/gpu/*_cubin.h") + + add_library(gpu STATIC ${GPU_LIB_SOURCES} ${GPU_LIB_CUDPP_SOURCES} ${GPU_OBJS}) + target_link_libraries(gpu PRIVATE ${CUDA_LIBRARIES} ${CUDA_CUDA_LIBRARY}) + target_include_directories(gpu PRIVATE ${LAMMPS_LIB_BINARY_DIR}/gpu ${CUDA_INCLUDE_DIRS}) + target_compile_definitions(gpu PRIVATE -D_${GPU_PREC_SETTING} -DMPI_GERYON -DUCL_NO_EXIT ${GPU_CUDA_MPS_FLAGS}) + if(CUDPP_OPT) + target_include_directories(gpu PRIVATE ${LAMMPS_LIB_SOURCE_DIR}/gpu/cudpp_mini) + target_compile_definitions(gpu PRIVATE -DUSE_CUDPP) + endif() + + target_link_libraries(lammps PRIVATE gpu) + + add_executable(nvc_get_devices ${LAMMPS_LIB_SOURCE_DIR}/gpu/geryon/ucl_get_devices.cpp) + target_compile_definitions(nvc_get_devices PRIVATE -DUCL_CUDADR) + target_link_libraries(nvc_get_devices PRIVATE ${CUDA_LIBRARIES} ${CUDA_CUDA_LIBRARY}) + target_include_directories(nvc_get_devices PRIVATE ${CUDA_INCLUDE_DIRS}) + +elseif(GPU_API STREQUAL "OPENCL") + if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows") + # download and unpack support binaries for compilation of windows binaries. + set(LAMMPS_THIRDPARTY_URL "http://download.lammps.org/thirdparty") + file(DOWNLOAD "${LAMMPS_THIRDPARTY_URL}/opencl-win-devel.tar.gz" "${CMAKE_CURRENT_BINARY_DIR}/opencl-win-devel.tar.gz" + EXPECTED_MD5 2c00364888d5671195598b44c2e0d44d) + execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf opencl-win-devel.tar.gz WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + add_library(OpenCL::OpenCL UNKNOWN IMPORTED) + if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86") + set_target_properties(OpenCL::OpenCL PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/OpenCL/lib_win32/libOpenCL.dll") + elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86_64") + set_target_properties(OpenCL::OpenCL PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/OpenCL/lib_win64/libOpenCL.dll") + endif() + set_target_properties(OpenCL::OpenCL PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_BINARY_DIR}/OpenCL/include") + else() + find_package(OpenCL REQUIRED) + endif() + set(OCL_TUNE "generic" CACHE STRING "OpenCL Device Tuning") + set(OCL_TUNE_VALUES intel fermi kepler cypress generic) + set_property(CACHE OCL_TUNE PROPERTY STRINGS ${OCL_TUNE_VALUES}) + validate_option(OCL_TUNE OCL_TUNE_VALUES) + string(TOUPPER ${OCL_TUNE} OCL_TUNE) + + include(OpenCLUtils) + set(OCL_COMMON_HEADERS ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_preprocessor.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_aux_fun1.h) + + file(GLOB GPU_LIB_CU ${LAMMPS_LIB_SOURCE_DIR}/gpu/[^.]*.cu) + list(REMOVE_ITEM GPU_LIB_CU + ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_gayberne.cu + ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_gayberne_lj.cu + ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_re_squared.cu + ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_re_squared_lj.cu + ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff.cu + ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_zbl.cu + ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_mod.cu + ) + + foreach(GPU_KERNEL ${GPU_LIB_CU}) + get_filename_component(basename ${GPU_KERNEL} NAME_WE) + string(SUBSTRING ${basename} 4 -1 KERNEL_NAME) + GenerateOpenCLHeader(${KERNEL_NAME} ${CMAKE_CURRENT_BINARY_DIR}/gpu/${KERNEL_NAME}_cl.h ${OCL_COMMON_HEADERS} ${GPU_KERNEL}) + list(APPEND GPU_LIB_SOURCES ${CMAKE_CURRENT_BINARY_DIR}/gpu/${KERNEL_NAME}_cl.h) + endforeach() + + GenerateOpenCLHeader(gayberne ${CMAKE_CURRENT_BINARY_DIR}/gpu/gayberne_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_ellipsoid_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_gayberne.cu) + GenerateOpenCLHeader(gayberne_lj ${CMAKE_CURRENT_BINARY_DIR}/gpu/gayberne_lj_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_ellipsoid_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_gayberne_lj.cu) + GenerateOpenCLHeader(re_squared ${CMAKE_CURRENT_BINARY_DIR}/gpu/re_squared_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_ellipsoid_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_re_squared.cu) + GenerateOpenCLHeader(re_squared_lj ${CMAKE_CURRENT_BINARY_DIR}/gpu/re_squared_lj_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_ellipsoid_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_re_squared_lj.cu) + GenerateOpenCLHeader(tersoff ${CMAKE_CURRENT_BINARY_DIR}/gpu/tersoff_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff.cu) + GenerateOpenCLHeader(tersoff_zbl ${CMAKE_CURRENT_BINARY_DIR}/gpu/tersoff_zbl_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_zbl_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_zbl.cu) + GenerateOpenCLHeader(tersoff_mod ${CMAKE_CURRENT_BINARY_DIR}/gpu/tersoff_mod_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_mod_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_mod.cu) + + list(APPEND GPU_LIB_SOURCES + ${CMAKE_CURRENT_BINARY_DIR}/gpu/gayberne_cl.h + ${CMAKE_CURRENT_BINARY_DIR}/gpu/gayberne_lj_cl.h + ${CMAKE_CURRENT_BINARY_DIR}/gpu/re_squared_cl.h + ${CMAKE_CURRENT_BINARY_DIR}/gpu/re_squared_lj_cl.h + ${CMAKE_CURRENT_BINARY_DIR}/gpu/tersoff_cl.h + ${CMAKE_CURRENT_BINARY_DIR}/gpu/tersoff_zbl_cl.h + ${CMAKE_CURRENT_BINARY_DIR}/gpu/tersoff_mod_cl.h + ) + + add_library(gpu STATIC ${GPU_LIB_SOURCES}) + target_link_libraries(gpu PRIVATE OpenCL::OpenCL) + target_include_directories(gpu PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/gpu) + target_compile_definitions(gpu PRIVATE -D_${GPU_PREC_SETTING} -D${OCL_TUNE}_OCL -DMPI_GERYON -DUCL_NO_EXIT) + target_compile_definitions(gpu PRIVATE -DUSE_OPENCL) + + target_link_libraries(lammps PRIVATE gpu) + + add_executable(ocl_get_devices ${LAMMPS_LIB_SOURCE_DIR}/gpu/geryon/ucl_get_devices.cpp) + target_compile_definitions(ocl_get_devices PRIVATE -DUCL_OPENCL) + target_link_libraries(ocl_get_devices PRIVATE OpenCL::OpenCL) +elseif(GPU_API STREQUAL "HIP") + if(NOT DEFINED HIP_PATH) + if(NOT DEFINED ENV{HIP_PATH}) + set(HIP_PATH "/opt/rocm/hip" CACHE PATH "Path to which HIP has been installed") + else() + set(HIP_PATH $ENV{HIP_PATH} CACHE PATH "Path to which HIP has been installed") endif() - option(CUDPP_OPT "Enable CUDPP_OPT" ON) - option(CUDA_MPS_SUPPORT "Enable tweaks to support CUDA Multi-process service (MPS)" OFF) - if(CUDA_MPS_SUPPORT) - set(GPU_CUDA_MPS_FLAGS "-DCUDA_PROXY") + endif() + set(CMAKE_MODULE_PATH "${HIP_PATH}/cmake" ${CMAKE_MODULE_PATH}) + find_package(HIP REQUIRED) + option(HIP_USE_DEVICE_SORT "Use GPU sorting" ON) + + if(NOT DEFINED HIP_PLATFORM) + if(NOT DEFINED ENV{HIP_PLATFORM}) + set(HIP_PLATFORM "hcc" CACHE PATH "HIP Platform to be used during compilation") + else() + set(HIP_PLATFORM $ENV{HIP_PLATFORM} CACHE PATH "HIP Platform used during compilation") endif() + endif() + + set(ENV{HIP_PLATFORM} ${HIP_PLATFORM}) + + if(HIP_PLATFORM STREQUAL "hcc") + set(HIP_ARCH "gfx906" CACHE STRING "HIP target architecture") + elseif(HIP_PLATFORM STREQUAL "nvcc") + find_package(CUDA REQUIRED) + set(HIP_ARCH "sm_50" CACHE STRING "HIP primary CUDA architecture (e.g. sm_60)") + + # build arch/gencode commands for nvcc based on CUDA toolkit version and use choice + # --arch translates directly instead of JIT, so this should be for the preferred or most common architecture + set(HIP_CUDA_GENCODE "-arch=${HIP_ARCH} ") + # Fermi (GPU Arch 2.x) is supported by CUDA 3.2 to CUDA 8.0 + if((CUDA_VERSION VERSION_GREATER "3.1") AND (CUDA_VERSION VERSION_LESS "9.0")) + string(APPEND HIP_CUDA_GENCODE "-gencode arch=compute_20,code=[sm_20,compute_20] ") + endif() + # Kepler (GPU Arch 3.x) is supported by CUDA 5 and later + if(CUDA_VERSION VERSION_GREATER "4.9") + string(APPEND HIP_CUDA_GENCODE "-gencode arch=compute_30,code=[sm_30,compute_30] -gencode arch=compute_35,code=[sm_35,compute_35] ") + endif() + # Maxwell (GPU Arch 5.x) is supported by CUDA 6 and later + if(CUDA_VERSION VERSION_GREATER "5.9") + string(APPEND HIP_CUDA_GENCODE "-gencode arch=compute_50,code=[sm_50,compute_50] -gencode arch=compute_52,code=[sm_52,compute_52] ") + endif() + # Pascal (GPU Arch 6.x) is supported by CUDA 8 and later + if(CUDA_VERSION VERSION_GREATER "7.9") + string(APPEND HIP_CUDA_GENCODE "-gencode arch=compute_60,code=[sm_60,compute_60] -gencode arch=compute_61,code=[sm_61,compute_61] ") + endif() + # Volta (GPU Arch 7.0) is supported by CUDA 9 and later + if(CUDA_VERSION VERSION_GREATER "8.9") + string(APPEND HIP_CUDA_GENCODE "-gencode arch=compute_70,code=[sm_70,compute_70] ") + endif() + # Turing (GPU Arch 7.5) is supported by CUDA 10 and later + if(CUDA_VERSION VERSION_GREATER "9.9") + string(APPEND HIP_CUDA_GENCODE "-gencode arch=compute_75,code=[sm_75,compute_75] ") + endif() + endif() + + file(GLOB GPU_LIB_CU ${LAMMPS_LIB_SOURCE_DIR}/gpu/[^.]*.cu ${CMAKE_CURRENT_SOURCE_DIR}/gpu/[^.]*.cu) + list(REMOVE_ITEM GPU_LIB_CU ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_pppm.cu) + + set(GPU_LIB_CU_HIP "") + foreach(CU_FILE ${GPU_LIB_CU}) + get_filename_component(CU_NAME ${CU_FILE} NAME_WE) + string(REGEX REPLACE "^.*lal_" "" CU_NAME "${CU_NAME}") + + set(CU_CPP_FILE "${LAMMPS_LIB_BINARY_DIR}/gpu/${CU_NAME}.cu.cpp") + set(CUBIN_FILE "${LAMMPS_LIB_BINARY_DIR}/gpu/${CU_NAME}.cubin") + set(CUBIN_H_FILE "${LAMMPS_LIB_BINARY_DIR}/gpu/${CU_NAME}_cubin.h") + + if(HIP_PLATFORM STREQUAL "hcc") + configure_file(${CU_FILE} ${CU_CPP_FILE} COPYONLY) + + add_custom_command(OUTPUT ${CUBIN_FILE} + VERBATIM COMMAND ${HIP_HIPCC_EXECUTABLE} --genco -t="${HIP_ARCH}" -f=\"-O3 -ffast-math -DUSE_HIP -D_${GPU_PREC_SETTING} -I${LAMMPS_LIB_SOURCE_DIR}/gpu\" -o ${CUBIN_FILE} ${CU_CPP_FILE} + DEPENDS ${CU_CPP_FILE} + COMMENT "Generating ${CU_NAME}.cubin") + elseif(HIP_PLATFORM STREQUAL "nvcc") + add_custom_command(OUTPUT ${CUBIN_FILE} + VERBATIM COMMAND ${HIP_HIPCC_EXECUTABLE} --fatbin --use_fast_math -DUSE_HIP -D_${GPU_PREC_SETTING} ${HIP_CUDA_GENCODE} -I${LAMMPS_LIB_SOURCE_DIR}/gpu -o ${CUBIN_FILE} ${CU_FILE} + DEPENDS ${CU_FILE} + COMMENT "Generating ${CU_NAME}.cubin") + endif() - set(GPU_ARCH "sm_30" CACHE STRING "LAMMPS GPU CUDA SM primary architecture (e.g. sm_60)") + add_custom_command(OUTPUT ${CUBIN_H_FILE} + COMMAND ${CMAKE_COMMAND} -D SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} -D VARNAME=${CU_NAME} -D HEADER_FILE=${CUBIN_H_FILE} -D SOURCE_FILES=${CUBIN_FILE} -P ${CMAKE_CURRENT_SOURCE_DIR}/Modules/GenerateBinaryHeader.cmake + DEPENDS ${CUBIN_FILE} + COMMENT "Generating ${CU_NAME}_cubin.h") - file(GLOB GPU_LIB_CU ${LAMMPS_LIB_SOURCE_DIR}/gpu/[^.]*.cu ${CMAKE_CURRENT_SOURCE_DIR}/gpu/[^.]*.cu) - list(REMOVE_ITEM GPU_LIB_CU ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_pppm.cu) + list(APPEND GPU_LIB_SOURCES ${CUBIN_H_FILE}) + endforeach() - cuda_include_directories(${LAMMPS_LIB_SOURCE_DIR}/gpu ${LAMMPS_LIB_BINARY_DIR}/gpu) + set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${LAMMPS_LIB_BINARY_DIR}/gpu/*_cubin.h ${LAMMPS_LIB_BINARY_DIR}/gpu/*.cu.cpp") - if(CUDPP_OPT) - cuda_include_directories(${LAMMPS_LIB_SOURCE_DIR}/gpu/cudpp_mini) - file(GLOB GPU_LIB_CUDPP_SOURCES ${LAMMPS_LIB_SOURCE_DIR}/gpu/cudpp_mini/[^.]*.cpp) - file(GLOB GPU_LIB_CUDPP_CU ${LAMMPS_LIB_SOURCE_DIR}/gpu/cudpp_mini/[^.]*.cu) - endif() + hip_add_library(gpu STATIC ${GPU_LIB_SOURCES}) + target_include_directories(gpu PRIVATE ${LAMMPS_LIB_BINARY_DIR}/gpu) + target_compile_definitions(gpu PRIVATE -D_${GPU_PREC_SETTING} -DMPI_GERYON -DUCL_NO_EXIT) + target_compile_definitions(gpu PRIVATE -DUSE_HIP) - # build arch/gencode commands for nvcc based on CUDA toolkit version and use choice - # --arch translates directly instead of JIT, so this should be for the preferred or most common architecture - set(GPU_CUDA_GENCODE "-arch=${GPU_ARCH} ") - # Fermi (GPU Arch 2.x) is supported by CUDA 3.2 to CUDA 8.0 - if((CUDA_VERSION VERSION_GREATER "3.1") AND (CUDA_VERSION VERSION_LESS "9.0")) - string(APPEND GPU_CUDA_GENCODE "-gencode arch=compute_20,code=[sm_20,compute_20] ") - endif() - # Kepler (GPU Arch 3.x) is supported by CUDA 5 and later - if(CUDA_VERSION VERSION_GREATER "4.9") - string(APPEND GPU_CUDA_GENCODE "-gencode arch=compute_30,code=[sm_30,compute_30] -gencode arch=compute_35,code=[sm_35,compute_35] ") - endif() - # Maxwell (GPU Arch 5.x) is supported by CUDA 6 and later - if(CUDA_VERSION VERSION_GREATER "5.9") - string(APPEND GPU_CUDA_GENCODE "-gencode arch=compute_50,code=[sm_50,compute_50] -gencode arch=compute_52,code=[sm_52,compute_52] ") - endif() - # Pascal (GPU Arch 6.x) is supported by CUDA 8 and later - if(CUDA_VERSION VERSION_GREATER "7.9") - string(APPEND GPU_CUDA_GENCODE "-gencode arch=compute_60,code=[sm_60,compute_60] -gencode arch=compute_61,code=[sm_61,compute_61] ") - endif() - # Volta (GPU Arch 7.0) is supported by CUDA 9 and later - if(CUDA_VERSION VERSION_GREATER "8.9") - string(APPEND GPU_CUDA_GENCODE "-gencode arch=compute_70,code=[sm_70,compute_70] ") - endif() - # Turing (GPU Arch 7.5) is supported by CUDA 10 and later - if(CUDA_VERSION VERSION_GREATER "9.9") - string(APPEND GPU_CUDA_GENCODE "-gencode arch=compute_75,code=[sm_75,compute_75] ") - endif() + if(HIP_USE_DEVICE_SORT) + # add hipCUB + target_include_directories(gpu PRIVATE ${HIP_ROOT_DIR}/../include) + target_compile_definitions(gpu PRIVATE -DUSE_HIP_DEVICE_SORT) - cuda_compile_fatbin(GPU_GEN_OBJS ${GPU_LIB_CU} OPTIONS - -DUNIX -O3 --use_fast_math -Wno-deprecated-gpu-targets -DNV_KERNEL -DUCL_CUDADR ${GPU_CUDA_GENCODE} -D_${GPU_PREC_SETTING}) - - cuda_compile(GPU_OBJS ${GPU_LIB_CUDPP_CU} OPTIONS ${CUDA_REQUEST_PIC} - -DUNIX -O3 --use_fast_math -Wno-deprecated-gpu-targets -DUCL_CUDADR ${GPU_CUDA_GENCODE} -D_${GPU_PREC_SETTING}) - - foreach(CU_OBJ ${GPU_GEN_OBJS}) - get_filename_component(CU_NAME ${CU_OBJ} NAME_WE) - string(REGEX REPLACE "^.*_lal_" "" CU_NAME "${CU_NAME}") - add_custom_command(OUTPUT ${LAMMPS_LIB_BINARY_DIR}/gpu/${CU_NAME}_cubin.h - COMMAND ${BIN2C} -c -n ${CU_NAME} ${CU_OBJ} > ${LAMMPS_LIB_BINARY_DIR}/gpu/${CU_NAME}_cubin.h - DEPENDS ${CU_OBJ} - COMMENT "Generating ${CU_NAME}_cubin.h") - list(APPEND GPU_LIB_SOURCES ${LAMMPS_LIB_BINARY_DIR}/gpu/${CU_NAME}_cubin.h) - endforeach() - set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${LAMMPS_LIB_BINARY_DIR}/gpu/*_cubin.h") - - - add_library(gpu STATIC ${GPU_LIB_SOURCES} ${GPU_LIB_CUDPP_SOURCES} ${GPU_OBJS}) - target_link_libraries(gpu ${CUDA_LIBRARIES} ${CUDA_CUDA_LIBRARY}) - target_include_directories(gpu PRIVATE ${LAMMPS_LIB_BINARY_DIR}/gpu ${CUDA_INCLUDE_DIRS}) - target_compile_definitions(gpu PRIVATE -D_${GPU_PREC_SETTING} -DMPI_GERYON -DUCL_NO_EXIT ${GPU_CUDA_MPS_FLAGS}) - if(CUDPP_OPT) - target_include_directories(gpu PRIVATE ${LAMMPS_LIB_SOURCE_DIR}/gpu/cudpp_mini) - target_compile_definitions(gpu PRIVATE -DUSE_CUDPP) - endif() + if(HIP_PLATFORM STREQUAL "nvcc") + find_package(CUB) - list(APPEND LAMMPS_LINK_LIBS gpu) - if(LAMMPS_USE_MPI4WIN) - add_dependencies(gpu mpi4win_build) + if(CUB_FOUND) + set(DOWNLOAD_CUB_DEFAULT OFF) + else() + set(DOWNLOAD_CUB_DEFAULT ON) endif() - add_executable(nvc_get_devices ${LAMMPS_LIB_SOURCE_DIR}/gpu/geryon/ucl_get_devices.cpp) - target_compile_definitions(nvc_get_devices PRIVATE -DUCL_CUDADR) - target_link_libraries(nvc_get_devices PRIVATE ${CUDA_LIBRARIES} ${CUDA_CUDA_LIBRARY}) - target_include_directories(nvc_get_devices PRIVATE ${CUDA_INCLUDE_DIRS}) - - - elseif(GPU_API STREQUAL "OPENCL") - find_package(OpenCL REQUIRED) - set(OCL_TUNE "generic" CACHE STRING "OpenCL Device Tuning") - set(OCL_TUNE_VALUES intel fermi kepler cypress generic) - set_property(CACHE OCL_TUNE PROPERTY STRINGS ${OCL_TUNE_VALUES}) - validate_option(OCL_TUNE OCL_TUNE_VALUES) - string(TOUPPER ${OCL_TUNE} OCL_TUNE) - - include(OpenCLUtils) - set(OCL_COMMON_HEADERS ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_preprocessor.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_aux_fun1.h) - - file(GLOB GPU_LIB_CU ${LAMMPS_LIB_SOURCE_DIR}/gpu/[^.]*.cu) - list(REMOVE_ITEM GPU_LIB_CU - ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_gayberne.cu - ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_gayberne_lj.cu - ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_re_squared.cu - ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_re_squared_lj.cu - ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff.cu - ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_zbl.cu - ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_mod.cu - ) - - foreach(GPU_KERNEL ${GPU_LIB_CU}) - get_filename_component(basename ${GPU_KERNEL} NAME_WE) - string(SUBSTRING ${basename} 4 -1 KERNEL_NAME) - GenerateOpenCLHeader(${KERNEL_NAME} ${CMAKE_CURRENT_BINARY_DIR}/gpu/${KERNEL_NAME}_cl.h ${OCL_COMMON_HEADERS} ${GPU_KERNEL}) - list(APPEND GPU_LIB_SOURCES ${CMAKE_CURRENT_BINARY_DIR}/gpu/${KERNEL_NAME}_cl.h) - endforeach() - - GenerateOpenCLHeader(gayberne ${CMAKE_CURRENT_BINARY_DIR}/gpu/gayberne_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_ellipsoid_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_gayberne.cu) - GenerateOpenCLHeader(gayberne_lj ${CMAKE_CURRENT_BINARY_DIR}/gpu/gayberne_lj_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_ellipsoid_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_gayberne_lj.cu) - GenerateOpenCLHeader(re_squared ${CMAKE_CURRENT_BINARY_DIR}/gpu/re_squared_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_ellipsoid_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_re_squared.cu) - GenerateOpenCLHeader(re_squared_lj ${CMAKE_CURRENT_BINARY_DIR}/gpu/re_squared_lj_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_ellipsoid_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_re_squared_lj.cu) - GenerateOpenCLHeader(tersoff ${CMAKE_CURRENT_BINARY_DIR}/gpu/tersoff_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff.cu) - GenerateOpenCLHeader(tersoff_zbl ${CMAKE_CURRENT_BINARY_DIR}/gpu/tersoff_zbl_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_zbl_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_zbl.cu) - GenerateOpenCLHeader(tersoff_mod ${CMAKE_CURRENT_BINARY_DIR}/gpu/tersoff_mod_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_mod_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_mod.cu) - - list(APPEND GPU_LIB_SOURCES - ${CMAKE_CURRENT_BINARY_DIR}/gpu/gayberne_cl.h - ${CMAKE_CURRENT_BINARY_DIR}/gpu/gayberne_lj_cl.h - ${CMAKE_CURRENT_BINARY_DIR}/gpu/re_squared_cl.h - ${CMAKE_CURRENT_BINARY_DIR}/gpu/re_squared_lj_cl.h - ${CMAKE_CURRENT_BINARY_DIR}/gpu/tersoff_cl.h - ${CMAKE_CURRENT_BINARY_DIR}/gpu/tersoff_zbl_cl.h - ${CMAKE_CURRENT_BINARY_DIR}/gpu/tersoff_mod_cl.h - ) - - add_library(gpu STATIC ${GPU_LIB_SOURCES}) - target_link_libraries(gpu ${OpenCL_LIBRARIES}) - target_include_directories(gpu PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/gpu ${OpenCL_INCLUDE_DIRS}) - target_compile_definitions(gpu PRIVATE -D_${GPU_PREC_SETTING} -D${OCL_TUNE}_OCL -DMPI_GERYON -DUCL_NO_EXIT) - target_compile_definitions(gpu PRIVATE -DUSE_OPENCL) - - list(APPEND LAMMPS_LINK_LIBS gpu) - if(LAMMPS_USE_MPI4WIN) - add_dependencies(gpu mpi4win_build) + option(DOWNLOAD_CUB "Download and compile the CUB library instead of using an already installed one" ${DOWNLOAD_CUB_DEFAULT}) + + if(DOWNLOAD_CUB) + message(STATUS "CUB download requested") + include(ExternalProject) + + ExternalProject_Add(CUB + GIT_REPOSITORY https://github.com/NVlabs/cub + TIMEOUT 5 + PREFIX "${CMAKE_CURRENT_BINARY_DIR}" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + UPDATE_COMMAND "" + ) + ExternalProject_get_property(CUB SOURCE_DIR) + set(CUB_INCLUDE_DIR ${SOURCE_DIR}) + else() + find_package(CUB) + if(NOT CUB_FOUND) + message(FATAL_ERROR "CUB library not found. Help CMake to find it by setting CUB_INCLUDE_DIR, or set DOWNLOAD_VORO=ON to download it") + endif() endif() - add_executable(ocl_get_devices ${LAMMPS_LIB_SOURCE_DIR}/gpu/geryon/ucl_get_devices.cpp) - target_compile_definitions(ocl_get_devices PRIVATE -DUCL_OPENCL) - target_link_libraries(ocl_get_devices PRIVATE ${OpenCL_LIBRARIES}) - target_include_directories(ocl_get_devices PRIVATE ${OpenCL_INCLUDE_DIRS}) + target_include_directories(gpu PRIVATE ${CUB_INCLUDE_DIR}) endif() + endif() + + hip_add_executable(hip_get_devices ${LAMMPS_LIB_SOURCE_DIR}/gpu/geryon/ucl_get_devices.cpp) + target_compile_definitions(hip_get_devices PRIVATE -DUCL_HIP) + + if(HIP_PLATFORM STREQUAL "nvcc") + target_compile_definitions(gpu PRIVATE -D__HIP_PLATFORM_NVCC__) + target_include_directories(gpu PRIVATE ${HIP_ROOT_DIR}/include) + target_include_directories(gpu PRIVATE ${CUDA_INCLUDE_DIRS}) + target_link_libraries(gpu PRIVATE ${CUDA_LIBRARIES} ${CUDA_CUDA_LIBRARY}) + + target_compile_definitions(hip_get_devices PRIVATE -D__HIP_PLATFORM_NVCC__) + target_include_directories(hip_get_devices PRIVATE ${HIP_ROOT_DIR}/include) + target_include_directories(hip_get_devices PRIVATE ${CUDA_INCLUDE_DIRS}) + target_link_libraries(hip_get_devices PRIVATE ${CUDA_LIBRARIES} ${CUDA_CUDA_LIBRARY}) + endif() + + target_link_libraries(lammps PRIVATE gpu) +endif() - # GPU package - FindStyleHeaders(${GPU_SOURCES_DIR} FIX_CLASS fix_ FIX) +# GPU package +FindStyleHeaders(${GPU_SOURCES_DIR} FIX_CLASS fix_ FIX) - set_property(GLOBAL PROPERTY "GPU_SOURCES" "${GPU_SOURCES}") +set_property(GLOBAL PROPERTY "GPU_SOURCES" "${GPU_SOURCES}") - # detects styles which have GPU version - RegisterStylesExt(${GPU_SOURCES_DIR} gpu GPU_SOURCES) +# detects styles which have GPU version +RegisterStylesExt(${GPU_SOURCES_DIR} gpu GPU_SOURCES) - get_property(GPU_SOURCES GLOBAL PROPERTY GPU_SOURCES) +get_property(GPU_SOURCES GLOBAL PROPERTY GPU_SOURCES) - list(APPEND LIB_SOURCES ${GPU_SOURCES}) - include_directories(${GPU_SOURCES_DIR}) +target_link_libraries(gpu PRIVATE MPI::MPI_CXX) +if(NOT BUILD_SHARED_LIBS) + install(TARGETS gpu EXPORT LAMMPS_Targets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) endif() +target_compile_definitions(gpu PRIVATE -DLAMMPS_${LAMMPS_SIZES}) +set_target_properties(gpu PROPERTIES OUTPUT_NAME lammps_gpu${LAMMPS_MACHINE}) +target_sources(lammps PRIVATE ${GPU_SOURCES}) +target_include_directories(lammps PRIVATE ${GPU_SOURCES_DIR}) diff --git a/cmake/Modules/Packages/KIM.cmake b/cmake/Modules/Packages/KIM.cmake index 617068ce13710a6b78089c6984ec9383cd1df71b..b4e1fb3502c22f42b84670e8f4d6f5b720143392 100644 --- a/cmake/Modules/Packages/KIM.cmake +++ b/cmake/Modules/Packages/KIM.cmake @@ -1,67 +1,61 @@ -if(PKG_KIM) - set(KIM-API_MIN_VERSION 2.1) - find_package(CURL) - if(CURL_FOUND) - include_directories(${CURL_INCLUDE_DIRS}) - list(APPEND LAMMPS_LINK_LIBS ${CURL_LIBRARIES}) - add_definitions(-DLMP_KIM_CURL) - set(LMP_DEBUG_CURL OFF CACHE STRING "Set libcurl verbose mode on/off. If on, it displays a lot of verbose information about its operations.") - mark_as_advanced(LMP_DEBUG_CURL) - if(LMP_DEBUG_CURL) - add_definitions(-DLMP_DEBUG_CURL) - endif() - set(LMP_NO_SSL_CHECK OFF CACHE STRING "Tell libcurl to not verify the peer. If on, the connection succeeds regardless of the names in the certificate. Insecure - Use with caution!") - mark_as_advanced(LMP_NO_SSL_CHECK) - if(LMP_NO_SSL_CHECK) - add_definitions(-DLMP_NO_SSL_CHECK) - endif() +set(KIM-API_MIN_VERSION 2.1.3) +find_package(CURL) +if(CURL_FOUND) + target_link_libraries(lammps PRIVATE CURL::libcurl) + target_compile_definitions(lammps PRIVATE -DLMP_KIM_CURL) + set(LMP_DEBUG_CURL OFF CACHE STRING "Set libcurl verbose mode on/off. If on, it displays a lot of verbose information about its operations.") + mark_as_advanced(LMP_DEBUG_CURL) + if(LMP_DEBUG_CURL) + target_compile_definitions(lammps PRIVATE -DLMP_DEBUG_CURL) endif() - find_package(KIM-API QUIET) + set(LMP_NO_SSL_CHECK OFF CACHE STRING "Tell libcurl to not verify the peer. If on, the connection succeeds regardless of the names in the certificate. Insecure - Use with caution!") + mark_as_advanced(LMP_NO_SSL_CHECK) + if(LMP_NO_SSL_CHECK) + target_compile_definitions(lammps PRIVATE -DLMP_NO_SSL_CHECK) + endif() +endif() +find_package(PkgConfig QUIET) +set(DOWNLOAD_KIM_DEFAULT ON) +if(PKG_CONFIG_FOUND) + pkg_check_modules(KIM-API QUIET libkim-api>=${KIM-API_MIN_VERSION}) if(KIM-API_FOUND) - if (KIM-API_VERSION VERSION_LESS ${KIM-API_MIN_VERSION}) - if ("${DOWNLOAD_KIM}" STREQUAL "") - message(WARNING "Unsuitable KIM-API version \"${KIM-API_VERSION}\" found, but required is at least \"${KIM-API_MIN_VERSION}\". Default behavior set to download and build our own.") - endif() - set(DOWNLOAD_KIM_DEFAULT ON) - else() - set(DOWNLOAD_KIM_DEFAULT OFF) - endif() - else() - if ("${DOWNLOAD_KIM}" STREQUAL "") - message(WARNING "KIM-API package not found. Default behavior set to download and build our own") - endif() - set(DOWNLOAD_KIM_DEFAULT ON) + set(DOWNLOAD_KIM_DEFAULT OFF) endif() - option(DOWNLOAD_KIM "Download KIM-API from OpenKIM instead of using an already installed one" ${DOWNLOAD_KIM_DEFAULT}) - if(DOWNLOAD_KIM) - message(STATUS "KIM-API download requested - we will build our own") - # Workaround for cross compilation with MinGW where ${CMAKE_INSTALL_LIBDIR} - # is a full path, so we need to remove the prefix - string(REPLACE ${CMAKE_INSTALL_PREFIX} "" _KIM_LIBDIR ${CMAKE_INSTALL_LIBDIR}) - include(ExternalProject) - enable_language(C) - enable_language(Fortran) - ExternalProject_Add(kim_build - URL https://s3.openkim.org/kim-api/kim-api-2.1.3.txz - URL_MD5 6ee829a1bbba5f8b9874c88c4c4ebff8 - BINARY_DIR build - CMAKE_ARGS ${CMAKE_REQUEST_PIC} - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} - -DCMAKE_INSTALL_PREFIX= - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM} - -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} - BUILD_BYPRODUCTS /${_KIM_LIBDIR}/libkim-api${CMAKE_SHARED_LIBRARY_SUFFIX} - ) - ExternalProject_get_property(kim_build INSTALL_DIR) - set(KIM-API_INCLUDE_DIRS ${INSTALL_DIR}/include/kim-api) - set(KIM-API_LDFLAGS ${INSTALL_DIR}/${_KIM_LIBDIR}/libkim-api${CMAKE_SHARED_LIBRARY_SUFFIX}) - list(APPEND LAMMPS_DEPS kim_build) - else() - find_package(KIM-API ${KIM-API_MIN_VERSION} REQUIRED) +endif() +option(DOWNLOAD_KIM "Download KIM-API from OpenKIM instead of using an already installed one" ${DOWNLOAD_KIM_DEFAULT}) +if(DOWNLOAD_KIM) + message(STATUS "KIM-API download requested - we will build our own") + include(ExternalProject) + enable_language(C) + enable_language(Fortran) + ExternalProject_Add(kim_build + URL https://s3.openkim.org/kim-api/kim-api-2.1.3.txz + URL_MD5 6ee829a1bbba5f8b9874c88c4c4ebff8 + BINARY_DIR build + CMAKE_ARGS ${CMAKE_REQUEST_PIC} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} + -DCMAKE_INSTALL_LIBDIR=lib + -DCMAKE_INSTALL_PREFIX= + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM} + -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} + BUILD_BYPRODUCTS /lib/libkim-api${CMAKE_SHARED_LIBRARY_SUFFIX} + ) + ExternalProject_get_property(kim_build INSTALL_DIR) + file(MAKE_DIRECTORY ${INSTALL_DIR}/include/kim-api) + add_library(LAMMPS::KIM UNKNOWN IMPORTED) + set_target_properties(LAMMPS::KIM PROPERTIES + IMPORTED_LOCATION "${INSTALL_DIR}/lib/libkim-api${CMAKE_SHARED_LIBRARY_SUFFIX}" + INTERFACE_INCLUDE_DIRECTORIES "${INSTALL_DIR}/include/kim-api") + target_link_libraries(lammps PRIVATE LAMMPS::KIM) + add_dependencies(LAMMPS::KIM kim_build) + if(NOT BUILD_SHARED_LIBS) + install(CODE "MESSAGE(FATAL_ERROR \"Installing liblammps with downloaded libraries is currently not supported.\")") endif() - list(APPEND LAMMPS_LINK_LIBS "${KIM-API_LDFLAGS}") - include_directories(${KIM-API_INCLUDE_DIRS}) +else() + find_package(PkgConfig REQUIRED) + pkg_check_modules(KIM-API REQUIRED IMPORTED_TARGET libkim-api>=${KIM-API_MIN_VERSION}) + target_link_libraries(lammps PRIVATE PkgConfig::KIM-API) endif() diff --git a/cmake/Modules/Packages/KOKKOS.cmake b/cmake/Modules/Packages/KOKKOS.cmake index 29beaca957885e0886d34c16437467066c06a061..a9a5c01a7200d3f27509fcf783fa4a708ed7db2f 100644 --- a/cmake/Modules/Packages/KOKKOS.cmake +++ b/cmake/Modules/Packages/KOKKOS.cmake @@ -1,74 +1,111 @@ -if(PKG_KOKKOS) - # TODO: this option needs to be documented when this works with a - # regular release version of KOKKOS, and a version compatibility check - # of external KOKKOS lib versus what the KOKKOS package needs is required. - option(EXTERNAL_KOKKOS "Build against external kokkos library") - if(EXTERNAL_KOKKOS) - find_package(Kokkos REQUIRED) - list(APPEND LAMMPS_LINK_LIBS Kokkos::kokkos) - else() - set(LAMMPS_LIB_KOKKOS_SRC_DIR ${LAMMPS_LIB_SOURCE_DIR}/kokkos) - set(LAMMPS_LIB_KOKKOS_BIN_DIR ${LAMMPS_LIB_BINARY_DIR}/kokkos) - add_subdirectory(${LAMMPS_LIB_KOKKOS_SRC_DIR} ${LAMMPS_LIB_KOKKOS_BIN_DIR}) +######################################################################## +# consistency checks and Kokkos options/settings required by LAMMPS +if(Kokkos_ENABLE_CUDA) + message(STATUS "KOKKOS: Enabling CUDA LAMBDA function support") + set(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "" FORCE) +endif() +# Adding OpenMP compiler flags without the checks done for +# BUILD_OMP can result in compile failures. Enforce consistency. +if(Kokkos_ENABLE_OPENMP AND NOT BUILD_OMP) + message(FATAL_ERROR "Must enable BUILD_OMP with Kokkos_ENABLE_OPENMP") +endif() +######################################################################## - set(Kokkos_INCLUDE_DIRS ${LAMMPS_LIB_KOKKOS_SRC_DIR}/core/src - ${LAMMPS_LIB_KOKKOS_SRC_DIR}/containers/src - ${LAMMPS_LIB_KOKKOS_SRC_DIR}/algorithms/src - ${LAMMPS_LIB_KOKKOS_BIN_DIR}) - include_directories(${Kokkos_INCLUDE_DIRS}) - list(APPEND LAMMPS_LINK_LIBS kokkos) +option(EXTERNAL_KOKKOS "Build against external kokkos library" OFF) +option(DOWNLOAD_KOKKOS "Download the KOKKOS library instead of using the bundled one" OFF) +if(DOWNLOAD_KOKKOS) + message(STATUS "KOKKOS download requested - we will build our own") + file(DOWNLOAD https://github.com/kokkos/kokkos/compare/3.0.00...stanmoore1:lammps.diff ${CMAKE_CURRENT_BINARY_DIR}/kokkos-lammps.patch) + include(ExternalProject) + ExternalProject_Add(kokkos_build + URL https://github.com/kokkos/kokkos/archive/3.0.00.tar.gz + URL_MD5 281c7093aa3a603276e93abdf4be23b9 + PATCH_COMMAND patch -p1 < ${CMAKE_CURRENT_BINARY_DIR}/kokkos-lammps.patch + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX= ${CMAKE_REQUEST_PIC} + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_LIBDIR=lib + -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} + BUILD_BYPRODUCTS /lib/libkokkoscore.a + ) + ExternalProject_get_property(kokkos_build INSTALL_DIR) + file(MAKE_DIRECTORY ${INSTALL_DIR}/include) + add_library(LAMMPS::KOKKOS UNKNOWN IMPORTED) + set_target_properties(LAMMPS::KOKKOS PROPERTIES + IMPORTED_LOCATION "${INSTALL_DIR}/lib/libkokkoscore.a" + INTERFACE_INCLUDE_DIRECTORIES "${INSTALL_DIR}/include" + INTERFACE_LINK_LIBRARIES ${CMAKE_DL_LIBS}) + target_link_libraries(lammps PRIVATE LAMMPS::KOKKOS) + add_dependencies(LAMMPS::KOKKOS kokkos_build) + if(NOT BUILD_SHARED_LIBS) + install(CODE "MESSAGE(FATAL_ERROR \"Installing liblammps with downloaded libraries is currently not supported.\")") endif() - add_definitions(-DLMP_KOKKOS) +elseif(EXTERNAL_KOKKOS) + find_package(Kokkos 3) + if(NOT Kokkos_FOUND) + message(FATAL_ERROR "KOKKOS library not found, help CMake to find it by setting KOKKOS_LIBRARY, or set DOWNLOAD_KOKKOS=ON to download it") + endif() + target_link_libraries(lammps PRIVATE Kokkos::kokkos) +else() + set(LAMMPS_LIB_KOKKOS_SRC_DIR ${LAMMPS_LIB_SOURCE_DIR}/kokkos) + set(LAMMPS_LIB_KOKKOS_BIN_DIR ${LAMMPS_LIB_BINARY_DIR}/kokkos) + add_subdirectory(${LAMMPS_LIB_KOKKOS_SRC_DIR} ${LAMMPS_LIB_KOKKOS_BIN_DIR}) - set(KOKKOS_PKG_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/KOKKOS) - set(KOKKOS_PKG_SOURCES ${KOKKOS_PKG_SOURCES_DIR}/kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/atom_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/atom_vec_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/comm_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/comm_tiled_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/min_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/min_linesearch_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/neighbor_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/neigh_list_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/neigh_bond_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/fix_nh_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/nbin_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/npair_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/npair_halffull_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/domain_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/modify_kokkos.cpp) + set(Kokkos_INCLUDE_DIRS ${LAMMPS_LIB_KOKKOS_SRC_DIR}/core/src + ${LAMMPS_LIB_KOKKOS_SRC_DIR}/containers/src + ${LAMMPS_LIB_KOKKOS_SRC_DIR}/algorithms/src + ${LAMMPS_LIB_KOKKOS_BIN_DIR}) + target_include_directories(lammps PRIVATE ${Kokkos_INCLUDE_DIRS}) + target_link_libraries(lammps PRIVATE kokkos) +endif() +target_compile_definitions(lammps PRIVATE -DLMP_KOKKOS) - if(PKG_KSPACE) - list(APPEND KOKKOS_PKG_SOURCES ${KOKKOS_PKG_SOURCES_DIR}/fft3d_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/gridcomm_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/remap_kokkos.cpp) - if(KOKKOS_ENABLE_CUDA) - if(NOT ${FFT} STREQUAL "KISS") - add_definitions(-DFFT_CUFFT) - list(APPEND LAMMPS_LINK_LIBS cufft) - endif() +set(KOKKOS_PKG_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/KOKKOS) +set(KOKKOS_PKG_SOURCES ${KOKKOS_PKG_SOURCES_DIR}/kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/atom_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/atom_vec_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/comm_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/comm_tiled_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/min_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/min_linesearch_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/neighbor_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/neigh_list_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/neigh_bond_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/fix_nh_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/nbin_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/npair_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/npair_halffull_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/domain_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/modify_kokkos.cpp) + +if(PKG_KSPACE) + list(APPEND KOKKOS_PKG_SOURCES ${KOKKOS_PKG_SOURCES_DIR}/fft3d_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/gridcomm_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/remap_kokkos.cpp) + if(Kokkos_ENABLE_CUDA) + if(NOT ${FFT} STREQUAL "KISS") + target_compile_definitions(lammps PRIVATE -DFFT_CUFFT) + target_link_libraries(lammps PRIVATE cufft) endif() endif() +endif() - set_property(GLOBAL PROPERTY "KOKKOS_PKG_SOURCES" "${KOKKOS_PKG_SOURCES}") - - # detects styles which have KOKKOS version - RegisterStylesExt(${KOKKOS_PKG_SOURCES_DIR} kokkos KOKKOS_PKG_SOURCES) +set_property(GLOBAL PROPERTY "KOKKOS_PKG_SOURCES" "${KOKKOS_PKG_SOURCES}") - # register kokkos-only styles - RegisterNBinStyle(${KOKKOS_PKG_SOURCES_DIR}/nbin_kokkos.h) - RegisterNPairStyle(${KOKKOS_PKG_SOURCES_DIR}/npair_kokkos.h) - RegisterNPairStyle(${KOKKOS_PKG_SOURCES_DIR}/npair_halffull_kokkos.h) +# detects styles which have KOKKOS version +RegisterStylesExt(${KOKKOS_PKG_SOURCES_DIR} kokkos KOKKOS_PKG_SOURCES) - if(PKG_USER-DPD) - get_property(KOKKOS_PKG_SOURCES GLOBAL PROPERTY KOKKOS_PKG_SOURCES) - list(APPEND KOKKOS_PKG_SOURCES ${KOKKOS_PKG_SOURCES_DIR}/npair_ssa_kokkos.cpp) - RegisterNPairStyle(${KOKKOS_PKG_SOURCES_DIR}/npair_ssa_kokkos.h) - set_property(GLOBAL PROPERTY "KOKKOS_PKG_SOURCES" "${KOKKOS_PKG_SOURCES}") - endif() +# register kokkos-only styles +RegisterNBinStyle(${KOKKOS_PKG_SOURCES_DIR}/nbin_kokkos.h) +RegisterNPairStyle(${KOKKOS_PKG_SOURCES_DIR}/npair_kokkos.h) +RegisterNPairStyle(${KOKKOS_PKG_SOURCES_DIR}/npair_halffull_kokkos.h) +if(PKG_USER-DPD) get_property(KOKKOS_PKG_SOURCES GLOBAL PROPERTY KOKKOS_PKG_SOURCES) - - list(APPEND LIB_SOURCES ${KOKKOS_PKG_SOURCES}) - include_directories(${KOKKOS_PKG_SOURCES_DIR}) + list(APPEND KOKKOS_PKG_SOURCES ${KOKKOS_PKG_SOURCES_DIR}/npair_ssa_kokkos.cpp) + RegisterNPairStyle(${KOKKOS_PKG_SOURCES_DIR}/npair_ssa_kokkos.h) + set_property(GLOBAL PROPERTY "KOKKOS_PKG_SOURCES" "${KOKKOS_PKG_SOURCES}") endif() + +get_property(KOKKOS_PKG_SOURCES GLOBAL PROPERTY KOKKOS_PKG_SOURCES) + +target_sources(lammps PRIVATE ${KOKKOS_PKG_SOURCES}) +target_include_directories(lammps PRIVATE ${KOKKOS_PKG_SOURCES_DIR}) diff --git a/cmake/Modules/Packages/KSPACE.cmake b/cmake/Modules/Packages/KSPACE.cmake index 07612447f985140dc88fb1c55112814a32ba8c81..dce0c15ef60a0eab19600ee616e87c7ac0a04dac 100644 --- a/cmake/Modules/Packages/KSPACE.cmake +++ b/cmake/Modules/Packages/KSPACE.cmake @@ -1,60 +1,56 @@ -if(PKG_KSPACE) - option(FFT_SINGLE "Use single precision FFTs instead of double precision FFTs" OFF) - set(FFTW "FFTW3") - if(FFT_SINGLE) - set(FFTW "FFTW3F") - add_definitions(-DFFT_SINGLE) - endif() - find_package(${FFTW} QUIET) - if(${FFTW}_FOUND) - set(FFT "FFTW3" CACHE STRING "FFT library for KSPACE package") +option(FFT_SINGLE "Use single precision FFTs instead of double precision FFTs" OFF) +set(FFTW "FFTW3") +if(FFT_SINGLE) + set(FFTW "FFTW3F") + target_compile_definitions(lammps PUBLIC -DFFT_SINGLE) +endif() +find_package(${FFTW} QUIET) +if(${FFTW}_FOUND) + set(FFT "FFTW3" CACHE STRING "FFT library for KSPACE package") +else() + set(FFT "KISS" CACHE STRING "FFT library for KSPACE package") +endif() +set(FFT_VALUES KISS FFTW3 MKL) +set_property(CACHE FFT PROPERTY STRINGS ${FFT_VALUES}) +validate_option(FFT FFT_VALUES) +string(TOUPPER ${FFT} FFT) + +if(FFT STREQUAL "FFTW3") + find_package(${FFTW} REQUIRED) + target_compile_definitions(lammps PUBLIC -DFFT_FFTW3) + target_link_libraries(lammps PUBLIC ${FFTW}::${FFTW}) + if(FFTW3_OMP_LIBRARY OR FFTW3F_OMP_LIBRARY) + option(FFT_FFTW_THREADS "Use threaded FFTW library" ON) else() - set(FFT "KISS" CACHE STRING "FFT library for KSPACE package") + option(FFT_FFTW_THREADS "Use threaded FFT library" OFF) endif() - set(FFT_VALUES KISS FFTW3 MKL) - set_property(CACHE FFT PROPERTY STRINGS ${FFT_VALUES}) - validate_option(FFT FFT_VALUES) - string(TOUPPER ${FFT} FFT) - if(FFT STREQUAL "FFTW3") - find_package(${FFTW} REQUIRED) - add_definitions(-DFFT_FFTW3) - include_directories(${${FFTW}_INCLUDE_DIRS}) - list(APPEND LAMMPS_LINK_LIBS ${${FFTW}_LIBRARIES}) + if(FFT_FFTW_THREADS) if(FFTW3_OMP_LIBRARY OR FFTW3F_OMP_LIBRARY) - option(FFT_FFTW_THREADS "Use threaded FFTW library" ON) + target_compile_definitions(lammps PRIVATE -DFFT_FFTW_THREADS) + target_link_libraries(lammps PRIVATE ${FFTW}::${FFTW}_OMP) else() - option(FFT_FFTW_THREADS "Use threaded FFT library" OFF) - endif() - - if(FFT_FFTW_THREADS) - if(FFTW3_OMP_LIBRARY OR FFTW3F_OMP_LIBRARY) - add_definitions(-DFFT_FFTW_THREADS) - list(APPEND LAMMPS_LINK_LIBS ${${FFTW}_OMP_LIBRARIES}) - else() - message(FATAL_ERROR "Need OpenMP enabled FFTW3 library for FFT_THREADS") - endif() + message(FATAL_ERROR "Need OpenMP enabled FFTW3 library for FFT_THREADS") endif() - elseif(FFT STREQUAL "MKL") - find_package(MKL REQUIRED) - add_definitions(-DFFT_MKL) - option(FFT_MKL_THREADS "Use threaded MKL FFT" ON) - if(FFT_MKL_THREADS) - add_definitions(-DFFT_MKL_THREADS) - endif() - include_directories(${MKL_INCLUDE_DIRS}) - list(APPEND LAMMPS_LINK_LIBS ${MKL_LIBRARIES}) - else() - # last option is KISSFFT - add_definitions(-DFFT_KISS) endif() - - set(FFT_PACK "array" CACHE STRING "Optimization for FFT") - set(FFT_PACK_VALUES array pointer memcpy) - set_property(CACHE FFT_PACK PROPERTY STRINGS ${FFT_PACK_VALUES}) - validate_option(FFT_PACK FFT_PACK_VALUES) - if(NOT FFT_PACK STREQUAL "array") - string(TOUPPER ${FFT_PACK} FFT_PACK) - add_definitions(-DFFT_PACK_${FFT_PACK}) +elseif(FFT STREQUAL "MKL") + find_package(MKL REQUIRED) + target_compile_definitions(lammps PRIVATE -DFFT_MKL) + option(FFT_MKL_THREADS "Use threaded MKL FFT" ON) + if(FFT_MKL_THREADS) + target_compile_definitions(lammps PRIVATE -DFFT_MKL_THREADS) endif() + target_link_libraries(lammps PRIVATE MKL::MKL) +else() + # last option is KISSFFT + target_compile_definitions(lammps PRIVATE -DFFT_KISS) +endif() + +set(FFT_PACK "array" CACHE STRING "Optimization for FFT") +set(FFT_PACK_VALUES array pointer memcpy) +set_property(CACHE FFT_PACK PROPERTY STRINGS ${FFT_PACK_VALUES}) +validate_option(FFT_PACK FFT_PACK_VALUES) +if(NOT FFT_PACK STREQUAL "array") + string(TOUPPER ${FFT_PACK} FFT_PACK) + target_compile_definitions(lammps PRIVATE -DFFT_PACK_${FFT_PACK}) endif() diff --git a/cmake/Modules/Packages/LATTE.cmake b/cmake/Modules/Packages/LATTE.cmake index 9f665d7f0e7299c33fbd81cfe9d03874e56edf7c..4ce8888f37668575024e277c2531aae8e266f7b5 100644 --- a/cmake/Modules/Packages/LATTE.cmake +++ b/cmake/Modules/Packages/LATTE.cmake @@ -1,40 +1,39 @@ -if(PKG_LATTE) - enable_language(Fortran) - find_package(LATTE) - if(LATTE_FOUND) - set(DOWNLOAD_LATTE_DEFAULT OFF) - else() - set(DOWNLOAD_LATTE_DEFAULT ON) - endif() - option(DOWNLOAD_LATTE "Download the LATTE library instead of using an already installed one" ${DOWNLOAD_LATTE_DEFAULT}) - if(DOWNLOAD_LATTE) - message(STATUS "LATTE download requested - we will build our own") - # Workaround for cross compilation with MinGW where ${CMAKE_INSTALL_LIBDIR} - # is a full path, so we need to remove the prefix - string(REPLACE ${CMAKE_INSTALL_PREFIX} "" _LATTE_LIBDIR ${CMAKE_INSTALL_LIBDIR}) - include(ExternalProject) - ExternalProject_Add(latte_build - URL https://github.com/lanl/LATTE/archive/v1.2.1.tar.gz - URL_MD5 85ac414fdada2d04619c8f936344df14 - SOURCE_SUBDIR cmake - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX= ${CMAKE_REQUEST_PIC} - -DBLAS_LIBRARIES=${BLAS_LIBRARIES} -DLAPACK_LIBRARIES=${LAPACK_LIBRARIES} - -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} -DCMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS} - -DCMAKE_Fortran_FLAGS_${BTYPE}=${CMAKE_Fortran_FLAGS_${BTYPE}} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} - BUILD_BYPRODUCTS /${_LATTE_LIBDIR}/liblatte.a - ) - list(APPEND LAMMPS_DEPS latte_build) - ExternalProject_get_property(latte_build INSTALL_DIR) - set(LATTE_LIBRARIES ${INSTALL_DIR}/${_LATTE_LIBDIR}/liblatte.a) - else() - find_package(LATTE) - if(NOT LATTE_FOUND) - message(FATAL_ERROR "LATTE library not found, help CMake to find it by setting LATTE_LIBRARY, or set DOWNLOAD_LATTE=ON to download it") - endif() +enable_language(Fortran) +find_package(LATTE) +if(LATTE_FOUND) + set(DOWNLOAD_LATTE_DEFAULT OFF) +else() + set(DOWNLOAD_LATTE_DEFAULT ON) +endif() +option(DOWNLOAD_LATTE "Download the LATTE library instead of using an already installed one" ${DOWNLOAD_LATTE_DEFAULT}) +if(DOWNLOAD_LATTE) + message(STATUS "LATTE download requested - we will build our own") + include(ExternalProject) + ExternalProject_Add(latte_build + URL https://github.com/lanl/LATTE/archive/v1.2.1.tar.gz + URL_MD5 85ac414fdada2d04619c8f936344df14 + SOURCE_SUBDIR cmake + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX= ${CMAKE_REQUEST_PIC} -DCMAKE_INSTALL_LIBDIR=lib + -DBLAS_LIBRARIES=${BLAS_LIBRARIES} -DLAPACK_LIBRARIES=${LAPACK_LIBRARIES} + -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} -DCMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS} + -DCMAKE_Fortran_FLAGS_${BTYPE}=${CMAKE_Fortran_FLAGS_${BTYPE}} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} + BUILD_BYPRODUCTS /lib/liblatte.a + ) + ExternalProject_get_property(latte_build INSTALL_DIR) + add_library(LAMMPS::LATTE UNKNOWN IMPORTED) + set_target_properties(LAMMPS::LATTE PROPERTIES + IMPORTED_LOCATION "${INSTALL_DIR}/lib/liblatte.a" + INTERFACE_LINK_LIBRARIES "${LAPACK_LIBRARIES}") + target_link_libraries(lammps PRIVATE LAMMPS::LATTE) + add_dependencies(LAMMPS::LATTE latte_build) + if(NOT BUILD_SHARED_LIBS) + install(CODE "MESSAGE(FATAL_ERROR \"Installing liblammps with downloaded libraries is currently not supported.\")") endif() - if(NOT LAPACK_FOUND) - add_dependencies(latte_build linalg) +else() + find_package(LATTE) + if(NOT LATTE_FOUND) + message(FATAL_ERROR "LATTE library not found, help CMake to find it by setting LATTE_LIBRARY, or set DOWNLOAD_LATTE=ON to download it") endif() - list(APPEND LAMMPS_LINK_LIBS ${LATTE_LIBRARIES} ${LAPACK_LIBRARIES}) + target_link_libraries(lammps PRIVATE LATTE::latte) endif() diff --git a/cmake/Modules/Packages/MESSAGE.cmake b/cmake/Modules/Packages/MESSAGE.cmake index aff9c2964adecf9088dcd1e23463f1fc36bb7bc9..e1dc4c107236e63e5a5e4ab8b77af1bae1e4b679 100644 --- a/cmake/Modules/Packages/MESSAGE.cmake +++ b/cmake/Modules/Packages/MESSAGE.cmake @@ -1,32 +1,35 @@ -if(PKG_MESSAGE) - if(LAMMPS_SIZES STREQUAL BIGBIG) - message(FATAL_ERROR "The MESSAGE Package is not compatible with -DLAMMPS_BIGBIG") - endif() - option(MESSAGE_ZMQ "Use ZeroMQ in MESSAGE package" OFF) - file(GLOB_RECURSE cslib_SOURCES ${LAMMPS_LIB_SOURCE_DIR}/message/cslib/[^.]*.F - ${LAMMPS_LIB_SOURCE_DIR}/message/cslib/[^.]*.c - ${LAMMPS_LIB_SOURCE_DIR}/message/cslib/[^.]*.cpp) - - add_library(cslib STATIC ${cslib_SOURCES}) - if(BUILD_MPI) - target_compile_definitions(cslib PRIVATE -DMPI_YES) - set_target_properties(cslib PROPERTIES OUTPUT_NAME "csmpi") - else() - target_compile_definitions(cslib PRIVATE -DMPI_NO) - target_include_directories(cslib PRIVATE ${LAMMPS_LIB_SOURCE_DIR}/message/cslib/src/STUBS_MPI) - set_target_properties(cslib PROPERTIES OUTPUT_NAME "csnompi") - endif() +if(LAMMPS_SIZES STREQUAL BIGBIG) + message(FATAL_ERROR "The MESSAGE Package is not compatible with -DLAMMPS_BIGBIG") +endif() +option(MESSAGE_ZMQ "Use ZeroMQ in MESSAGE package" OFF) +file(GLOB_RECURSE cslib_SOURCES ${LAMMPS_LIB_SOURCE_DIR}/message/cslib/[^.]*.F + ${LAMMPS_LIB_SOURCE_DIR}/message/cslib/[^.]*.c + ${LAMMPS_LIB_SOURCE_DIR}/message/cslib/[^.]*.cpp) - if(MESSAGE_ZMQ) - target_compile_definitions(cslib PRIVATE -DZMQ_YES) - find_package(ZMQ REQUIRED) - target_include_directories(cslib PRIVATE ${ZMQ_INCLUDE_DIRS}) - target_link_libraries(cslib PUBLIC ${ZMQ_LIBRARIES}) - else() - target_compile_definitions(cslib PRIVATE -DZMQ_NO) - target_include_directories(cslib PRIVATE ${LAMMPS_LIB_SOURCE_DIR}/message/cslib/src/STUBS_ZMQ) - endif() +add_library(cslib STATIC ${cslib_SOURCES}) +if(NOT BUILD_SHARED_LIBS) + install(TARGETS cslib EXPORT LAMMPS_Targets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) +endif() +target_compile_definitions(cslib PRIVATE -DLAMMPS_${LAMMPS_SIZES}) +set_target_properties(cslib PROPERTIES OUTPUT_NAME lammps_cslib${LAMMPS_MACHINE}) +if(BUILD_MPI) + target_compile_definitions(cslib PRIVATE -DMPI_YES) + set_target_properties(cslib PROPERTIES OUTPUT_NAME "csmpi") + target_link_libraries(cslib PRIVATE MPI::MPI_CXX) +else() + target_compile_definitions(cslib PRIVATE -DMPI_NO) + target_include_directories(cslib PRIVATE ${LAMMPS_LIB_SOURCE_DIR}/message/cslib/src/STUBS_MPI) + set_target_properties(cslib PROPERTIES OUTPUT_NAME "csnompi") +endif() - list(APPEND LAMMPS_LINK_LIBS cslib) - include_directories(${LAMMPS_LIB_SOURCE_DIR}/message/cslib/src) +if(MESSAGE_ZMQ) + target_compile_definitions(cslib PRIVATE -DZMQ_YES) + find_package(ZMQ REQUIRED) + target_link_libraries(cslib PUBLIC ZMQ::ZMQ) +else() + target_compile_definitions(cslib PRIVATE -DZMQ_NO) + target_include_directories(cslib PRIVATE ${LAMMPS_LIB_SOURCE_DIR}/message/cslib/src/STUBS_ZMQ) endif() + +target_link_libraries(lammps PRIVATE cslib) +target_include_directories(lammps PRIVATE ${LAMMPS_LIB_SOURCE_DIR}/message/cslib/src) diff --git a/cmake/Modules/Packages/MSCG.cmake b/cmake/Modules/Packages/MSCG.cmake index 35f0c57449b28755f1993bf3212c02df27c93787..668d318d689edb039cf0b3f5df0565b052a7e9e2 100644 --- a/cmake/Modules/Packages/MSCG.cmake +++ b/cmake/Modules/Packages/MSCG.cmake @@ -1,48 +1,48 @@ -if(PKG_MSCG) - find_package(GSL REQUIRED) - find_package(MSCG QUIET) - if(MSGC_FOUND) - set(DOWNLOAD_MSCG_DEFAULT OFF) - else() - set(DOWNLOAD_MSCG_DEFAULT ON) +find_package(GSL REQUIRED) +find_package(MSCG QUIET) +if(MSGC_FOUND) + set(DOWNLOAD_MSCG_DEFAULT OFF) +else() + set(DOWNLOAD_MSCG_DEFAULT ON) +endif() +option(DOWNLOAD_MSCG "Download MSCG library instead of using an already installed one)" ${DOWNLOAD_MSCG_DEFAULT}) +if(DOWNLOAD_MSCG) + include(ExternalProject) + ExternalProject_Add(mscg_build + URL https://github.com/uchicago-voth/MSCG-release/archive/1.7.3.1.tar.gz + URL_MD5 8c45e269ee13f60b303edd7823866a91 + SOURCE_SUBDIR src/CMake + CMAKE_ARGS ${CMAKE_REQUEST_PIC} ${EXTRA_MSCG_OPTS} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} + -DBLAS_LIBRARIES=${BLAS_LIBRARIES} -DLAPACK_LIBRARIES=${LAPACK_LIBRARIES} + -DCMAKE_INSTALL_PREFIX= + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM} + -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} + BUILD_COMMAND ${CMAKE_COMMAND} --build . --target mscg + INSTALL_COMMAND "" + BUILD_BYPRODUCTS /libmscg.a + ) + ExternalProject_get_property(mscg_build BINARY_DIR) + ExternalProject_get_property(mscg_build SOURCE_DIR) + file(MAKE_DIRECTORY ${SOURCE_DIR}/src) + add_library(LAMMPS::MSCG UNKNOWN IMPORTED) + set_target_properties(LAMMPS::MSCG PROPERTIES + IMPORTED_LOCATION "${BINARY_DIR}/libmscg.a" + INTERFACE_INCLUDE_DIRECTORIES "${SOURCE_DIR}/src" + INTERFACE_LINK_LIBRARIES "${LAPACK_LIBRARIES}") + target_link_libraries(lammps PRIVATE LAMMPS::MSCG) + add_dependencies(LAMMPS::MSCG mscg_build) + if(NOT BUILD_SHARED_LIBS) + install(CODE "MESSAGE(FATAL_ERROR \"Installing liblammps with downloaded libraries is currently not supported.\")") endif() - option(DOWNLOAD_MSCG "Download MSCG library instead of using an already installed one)" ${DOWNLOAD_MSCG_DEFAULT}) - if(DOWNLOAD_MSCG) - include(ExternalProject) - if(NOT LAPACK_FOUND) - set(EXTRA_MSCG_OPTS "-DLAPACK_LIBRARIES=${CMAKE_CURRENT_BINARY_DIR}/liblinalg.a") - endif() - ExternalProject_Add(mscg_build - URL https://github.com/uchicago-voth/MSCG-release/archive/1.7.3.1.tar.gz - URL_MD5 8c45e269ee13f60b303edd7823866a91 - SOURCE_SUBDIR src/CMake - CMAKE_ARGS ${CMAKE_REQUEST_PIC} ${EXTRA_MSCG_OPTS} - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} - -DCMAKE_INSTALL_PREFIX= - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM} - -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} - BUILD_COMMAND ${CMAKE_COMMAND} --build . --target mscg - INSTALL_COMMAND "" - BUILD_BYPRODUCTS /libmscg.a - ) - ExternalProject_get_property(mscg_build BINARY_DIR) - set(MSCG_LIBRARIES ${BINARY_DIR}/libmscg.a) - ExternalProject_get_property(mscg_build SOURCE_DIR) - set(MSCG_INCLUDE_DIRS ${SOURCE_DIR}/src) - list(APPEND LAMMPS_DEPS mscg_build) - if(NOT LAPACK_FOUND) - file(MAKE_DIRECTORY ${MSCG_INCLUDE_DIRS}) - add_dependencies(mscg_build linalg) - endif() - else() - find_package(MSCG) - if(NOT MSCG_FOUND) - message(FATAL_ERROR "MSCG not found, help CMake to find it by setting MSCG_LIBRARY and MSCG_INCLUDE_DIRS, or set DOWNLOAD_MSCG=ON to download it") - endif() +else() + find_package(MSCG) + if(NOT MSCG_FOUND) + message(FATAL_ERROR "MSCG not found, help CMake to find it by setting MSCG_LIBRARY and MSCG_INCLUDE_DIRS, or set DOWNLOAD_MSCG=ON to download it") endif() - list(APPEND LAMMPS_LINK_LIBS ${MSCG_LIBRARIES} ${GSL_LIBRARIES} ${LAPACK_LIBRARIES}) - include_directories(${MSCG_INCLUDE_DIRS}) + target_link_libraries(lammps PRIVATE MSCG::MSCG) endif() +target_link_libraries(lammps PRIVATE GSL::gsl ${LAPACK_LIBRARIES}) diff --git a/cmake/Modules/Packages/OPT.cmake b/cmake/Modules/Packages/OPT.cmake index f2802c757bdb1d3472395270c9ba1f7a4b652c19..bcf572b087d56f66c512b6d906f987d512d8347a 100644 --- a/cmake/Modules/Packages/OPT.cmake +++ b/cmake/Modules/Packages/OPT.cmake @@ -1,13 +1,11 @@ -if(PKG_OPT) - set(OPT_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/OPT) - set(OPT_SOURCES) - set_property(GLOBAL PROPERTY "OPT_SOURCES" "${OPT_SOURCES}") + set(OPT_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/OPT) + set(OPT_SOURCES) + set_property(GLOBAL PROPERTY "OPT_SOURCES" "${OPT_SOURCES}") - # detects styles which have OPT version - RegisterStylesExt(${OPT_SOURCES_DIR} opt OPT_SOURCES) + # detects styles which have OPT version + RegisterStylesExt(${OPT_SOURCES_DIR} opt OPT_SOURCES) - get_property(OPT_SOURCES GLOBAL PROPERTY OPT_SOURCES) + get_property(OPT_SOURCES GLOBAL PROPERTY OPT_SOURCES) - list(APPEND LIB_SOURCES ${OPT_SOURCES}) - include_directories(${OPT_SOURCES_DIR}) -endif() + target_sources(lammps PRIVATE ${OPT_SOURCES}) + target_include_directories(lammps PRIVATE ${OPT_SOURCES_DIR}) diff --git a/cmake/Modules/Packages/PYTHON.cmake b/cmake/Modules/Packages/PYTHON.cmake index 4f8959ae389ca97ef8774641f08949cf1bff600c..a577f824fe418cefb7d728acea61668151ed0364 100644 --- a/cmake/Modules/Packages/PYTHON.cmake +++ b/cmake/Modules/Packages/PYTHON.cmake @@ -1,6 +1,9 @@ -if(PKG_PYTHON) - find_package(PythonLibs REQUIRED) - add_definitions(-DLMP_PYTHON) - include_directories(${PYTHON_INCLUDE_DIR}) - list(APPEND LAMMPS_LINK_LIBS ${PYTHON_LIBRARY}) +if(CMAKE_VERSION VERSION_LESS 3.12) + find_package(PythonLibs REQUIRED) # Deprecated since version 3.12 + target_include_directories(lammps PRIVATE ${PYTHON_INCLUDE_DIR}) + target_link_libraries(lammps PRIVATE ${PYTHON_LIBRARY}) +else() + find_package(Python REQUIRED COMPONENTS Development) + target_link_libraries(lammps PRIVATE Python::Python) endif() +target_compile_definitions(lammps PRIVATE -DLMP_PYTHON) diff --git a/cmake/Modules/Packages/QEQ.cmake b/cmake/Modules/Packages/QEQ.cmake index 94cca30540dab41dd831254881ed86f788d00653..1bdaa02f431568038af65f7fe0bb49f558ae582b 100644 --- a/cmake/Modules/Packages/QEQ.cmake +++ b/cmake/Modules/Packages/QEQ.cmake @@ -1,20 +1,18 @@ # Fix qeq/fire requires MANYBODY (i.e. COMB and COMB3) to be installed -if(PKG_QEQ) - set(QEQ_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/QEQ) - file(GLOB QEQ_HEADERS ${QEQ_SOURCES_DIR}/fix*.h) - file(GLOB QEQ_SOURCES ${QEQ_SOURCES_DIR}/fix*.cpp) +set(QEQ_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/QEQ) +file(GLOB QEQ_HEADERS ${QEQ_SOURCES_DIR}/fix*.h) +file(GLOB QEQ_SOURCES ${QEQ_SOURCES_DIR}/fix*.cpp) - if(NOT PKG_MANYBODY) - list(REMOVE_ITEM QEQ_HEADERS ${QEQ_SOURCES_DIR}/fix_qeq_fire.h) - list(REMOVE_ITEM QEQ_SOURCES ${QEQ_SOURCES_DIR}/fix_qeq_fire.cpp) - endif() - set_property(GLOBAL PROPERTY "QEQ_SOURCES" "${QEQ_SOURCES}") +if(NOT PKG_MANYBODY) + list(REMOVE_ITEM QEQ_HEADERS ${QEQ_SOURCES_DIR}/fix_qeq_fire.h) + list(REMOVE_ITEM QEQ_SOURCES ${QEQ_SOURCES_DIR}/fix_qeq_fire.cpp) +endif() +set_property(GLOBAL PROPERTY "QEQ_SOURCES" "${QEQ_SOURCES}") - foreach(MY_HEADER ${QEQ_HEADERS}) - AddStyleHeader(${MY_HEADER} FIX) - endforeach() +foreach(MY_HEADER ${QEQ_HEADERS}) + AddStyleHeader(${MY_HEADER} FIX) +endforeach() - get_property(QEQ_SOURCES GLOBAL PROPERTY QEQ_SOURCES) - list(APPEND LIB_SOURCES ${QEQ_SOURCES}) - include_directories(${QEQ_SOURCES_DIR}) -endif() +get_property(QEQ_SOURCES GLOBAL PROPERTY QEQ_SOURCES) +target_sources(lammps PRIVATE ${QEQ_SOURCES}) +target_include_directories(lammps PRIVATE ${QEQ_SOURCES_DIR}) diff --git a/cmake/Modules/Packages/USER-COLVARS.cmake b/cmake/Modules/Packages/USER-COLVARS.cmake index a112fbb6aa1d0c6dac813599405e56843e2a3269..281b497dea6008cb094f61bf8753fabd09c3a88e 100644 --- a/cmake/Modules/Packages/USER-COLVARS.cmake +++ b/cmake/Modules/Packages/USER-COLVARS.cmake @@ -1,27 +1,32 @@ -if(PKG_USER-COLVARS) +set(COLVARS_SOURCE_DIR ${LAMMPS_LIB_SOURCE_DIR}/colvars) - set(COLVARS_SOURCE_DIR ${LAMMPS_LIB_SOURCE_DIR}/colvars) +file(GLOB COLVARS_SOURCES ${COLVARS_SOURCE_DIR}/[^.]*.cpp) - file(GLOB COLVARS_SOURCES ${COLVARS_SOURCE_DIR}/[^.]*.cpp) +# Build Lepton by default +option(COLVARS_LEPTON "Build and link the Lepton library" ON) - # Build Lepton by default - option(COLVARS_LEPTON "Build and link the Lepton library" ON) - - if(COLVARS_LEPTON) - set(LEPTON_DIR ${LAMMPS_LIB_SOURCE_DIR}/colvars/lepton) - file(GLOB LEPTON_SOURCES ${LEPTON_DIR}/src/[^.]*.cpp) - add_library(lepton STATIC ${LEPTON_SOURCES}) - target_include_directories(lepton PRIVATE ${LEPTON_DIR}/include) +if(COLVARS_LEPTON) + set(LEPTON_DIR ${LAMMPS_LIB_SOURCE_DIR}/colvars/lepton) + file(GLOB LEPTON_SOURCES ${LEPTON_DIR}/src/[^.]*.cpp) + add_library(lepton STATIC ${LEPTON_SOURCES}) + if(NOT BUILD_SHARED_LIBS) + install(TARGETS lepton EXPORT LAMMPS_Targets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) endif() + set_target_properties(lepton PROPERTIES OUTPUT_NAME lammps_lepton${LAMMPS_MACHINE}) + target_include_directories(lepton PRIVATE ${LEPTON_DIR}/include) +endif() - add_library(colvars STATIC ${COLVARS_SOURCES}) - target_include_directories(colvars PUBLIC ${LAMMPS_LIB_SOURCE_DIR}/colvars) - list(APPEND LAMMPS_LINK_LIBS colvars) - - if(COLVARS_LEPTON) - list(APPEND LAMMPS_LINK_LIBS lepton) - target_compile_options(colvars PRIVATE -DLEPTON) - target_include_directories(colvars PUBLIC ${LEPTON_DIR}/include) - endif() +add_library(colvars STATIC ${COLVARS_SOURCES}) +if(NOT BUILD_SHARED_LIBS) + install(TARGETS colvars EXPORT LAMMPS_Targets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) +endif() +target_compile_definitions(colvars PRIVATE -DLAMMPS_${LAMMPS_SIZES}) +set_target_properties(colvars PROPERTIES OUTPUT_NAME lammps_colvars${LAMMPS_MACHINE}) +target_include_directories(colvars PUBLIC ${LAMMPS_LIB_SOURCE_DIR}/colvars) +target_link_libraries(lammps PRIVATE colvars) +if(COLVARS_LEPTON) + target_link_libraries(lammps PRIVATE lepton) + target_compile_options(colvars PRIVATE -DLEPTON) + target_include_directories(colvars PUBLIC ${LEPTON_DIR}/include) endif() diff --git a/cmake/Modules/Packages/USER-H5MD.cmake b/cmake/Modules/Packages/USER-H5MD.cmake index 40ea7b7444cd024c762e9708ebc8dd965a95c153..4fcae93027c36204799b5388de90e2f6b9b4795f 100644 --- a/cmake/Modules/Packages/USER-H5MD.cmake +++ b/cmake/Modules/Packages/USER-H5MD.cmake @@ -1,8 +1,5 @@ -if(PKG_USER-H5MD) - enable_language(C) +enable_language(C) - find_package(HDF5 REQUIRED) - target_link_libraries(h5md ${HDF5_LIBRARIES}) - target_include_directories(h5md PRIVATE ${HDF5_INCLUDE_DIRS}) - include_directories(${HDF5_INCLUDE_DIRS}) -endif() +find_package(HDF5 REQUIRED) +target_link_libraries(h5md PRIVATE ${HDF5_LIBRARIES}) +target_include_directories(h5md PUBLIC ${HDF5_INCLUDE_DIRS}) diff --git a/cmake/Modules/Packages/USER-INTEL.cmake b/cmake/Modules/Packages/USER-INTEL.cmake index d0941a0a12d594a75f3ce331c560eecbb080b1ec..0ae1ec018a309be7f2ac4293e04d12019ddd9a8b 100644 --- a/cmake/Modules/Packages/USER-INTEL.cmake +++ b/cmake/Modules/Packages/USER-INTEL.cmake @@ -1,113 +1,109 @@ -if(PKG_USER-INTEL) - check_include_file_cxx(immintrin.h FOUND_IMMINTRIN) - if(NOT FOUND_IMMINTRIN) - message(FATAL_ERROR "immintrin.h header not found, Intel package won't work without it") - endif() +check_include_file_cxx(immintrin.h FOUND_IMMINTRIN) +if(NOT FOUND_IMMINTRIN) + message(FATAL_ERROR "immintrin.h header not found, Intel package won't work without it") +endif() - add_definitions(-DLMP_USER_INTEL) +target_compile_definitions(lammps PRIVATE -DLMP_USER_INTEL) - set(INTEL_ARCH "cpu" CACHE STRING "Architectures used by USER-INTEL (cpu or knl)") - set(INTEL_ARCH_VALUES cpu knl) - set_property(CACHE INTEL_ARCH PROPERTY STRINGS ${INTEL_ARCH_VALUES}) - validate_option(INTEL_ARCH INTEL_ARCH_VALUES) - string(TOUPPER ${INTEL_ARCH} INTEL_ARCH) +set(INTEL_ARCH "cpu" CACHE STRING "Architectures used by USER-INTEL (cpu or knl)") +set(INTEL_ARCH_VALUES cpu knl) +set_property(CACHE INTEL_ARCH PROPERTY STRINGS ${INTEL_ARCH_VALUES}) +validate_option(INTEL_ARCH INTEL_ARCH_VALUES) +string(TOUPPER ${INTEL_ARCH} INTEL_ARCH) - find_package(Threads QUIET) +find_package(Threads QUIET) +if(Threads_FOUND) + set(INTEL_LRT_MODE "threads" CACHE STRING "Long-range threads mode (none, threads, or c++11)") +else() + set(INTEL_LRT_MODE "none" CACHE STRING "Long-range threads mode (none, threads, or c++11)") +endif() +set(INTEL_LRT_VALUES none threads c++11) +set_property(CACHE INTEL_LRT_MODE PROPERTY STRINGS ${INTEL_LRT_VALUES}) +validate_option(INTEL_LRT_MODE INTEL_LRT_VALUES) +string(TOUPPER ${INTEL_LRT_MODE} INTEL_LRT_MODE) +if(INTEL_LRT_MODE STREQUAL "THREADS") if(Threads_FOUND) - set(INTEL_LRT_MODE "threads" CACHE STRING "Long-range threads mode (none, threads, or c++11)") + target_compile_definitions(lammps PRIVATE -DLMP_INTEL_USELRT) + target_link_libraries(lammps PRIVATE Threads::Threads) else() - set(INTEL_LRT_MODE "none" CACHE STRING "Long-range threads mode (none, threads, or c++11)") + message(FATAL_ERROR "Must have working threads library for Long-range thread support") endif() - set(INTEL_LRT_VALUES none threads c++11) - set_property(CACHE INTEL_LRT_MODE PROPERTY STRINGS ${INTEL_LRT_VALUES}) - validate_option(INTEL_LRT_MODE INTEL_LRT_VALUES) - string(TOUPPER ${INTEL_LRT_MODE} INTEL_LRT_MODE) - if(INTEL_LRT_MODE STREQUAL "THREADS") - if(Threads_FOUND) - add_definitions(-DLMP_INTEL_USELRT) - list(APPEND LAMMPS_LINK_LIBS ${CMAKE_THREAD_LIBS_INIT}) - else() - message(FATAL_ERROR "Must have working threads library for Long-range thread support") - endif() - endif() - if(INTEL_LRT_MODE STREQUAL "C++11") - add_definitions(-DLMP_INTEL_USERLRT -DLMP_INTEL_LRT11) +endif() +if(INTEL_LRT_MODE STREQUAL "C++11") + target_compile_definitions(lammps PRIVATE -DLMP_INTEL_USELRT -DLMP_INTEL_LRT11) +endif() + +if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 16) + message(FATAL_ERROR "USER-INTEL needs at least a 2016 Intel compiler, found ${CMAKE_CXX_COMPILER_VERSION}") endif() +else() + message(WARNING "USER-INTEL gives best performance with Intel compilers") +endif() +find_package(TBB_MALLOC QUIET) +if(TBB_MALLOC_FOUND) + target_link_libraries(lammps PRIVATE TBB::TBB_MALLOC) +else() + target_compile_definitions(lammps PRIVATE -DLMP_INTEL_NO_TBB) if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel") - if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 16) - message(FATAL_ERROR "USER-INTEL needs at least a 2016 Intel compiler, found ${CMAKE_CXX_COMPILER_VERSION}") - endif() - else() - message(WARNING "USER-INTEL gives best performance with Intel compilers") + message(WARNING "USER-INTEL with Intel compilers should use TBB malloc libraries") endif() +endif() - find_package(TBB QUIET) - if(TBB_FOUND) - list(APPEND LAMMPS_LINK_LIBS ${TBB_MALLOC_LIBRARIES}) - else() - add_definitions(-DLMP_INTEL_NO_TBB) - if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel") - message(WARNING "USER-INTEL with Intel compilers should use TBB malloc libraries") - endif() - endif() +find_package(MKL QUIET) +if(MKL_FOUND) + target_compile_definitions(lammps PRIVATE -DLMP_USE_MKL_RNG) + target_link_libraries(lammps PRIVATE MKL::MKL) +else() + message(STATUS "Pair style dpd/intel will be faster with MKL libraries") +endif() - find_package(MKL QUIET) - if(MKL_FOUND) - add_definitions(-DLMP_USE_MKL_RNG) - list(APPEND LAMMPS_LINK_LIBS ${MKL_LIBRARIES}) - else() - message(STATUS "Pair style dpd/intel will be faster with MKL libraries") - endif() +if((NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Windows") AND (NOT ${LAMMPS_MEMALIGN} STREQUAL "64") AND (NOT ${LAMMPS_MEMALIGN} STREQUAL "128") AND (NOT ${LAMMPS_MEMALIGN} STREQUAL "256")) + message(FATAL_ERROR "USER-INTEL only supports memory alignment of 64, 128 or 256 on this platform") +endif() - if((NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Windows") AND (NOT ${LAMMPS_MEMALIGN} STREQUAL "64") AND (NOT ${LAMMPS_MEMALIGN} STREQUAL "128") AND (NOT ${LAMMPS_MEMALIGN} STREQUAL "256")) - message(FATAL_ERROR "USER-INTEL only supports memory alignment of 64, 128 or 256 on this platform") +if(INTEL_ARCH STREQUAL "KNL") + if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + message(FATAL_ERROR "Must use Intel compiler with USER-INTEL for KNL architecture") endif() - - if(INTEL_ARCH STREQUAL "KNL") - if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "Intel") - message(FATAL_ERROR "Must use Intel compiler with USER-INTEL for KNL architecture") - endif() - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -xHost -qopenmp -qoffload") - set(MIC_OPTIONS "-qoffload-option,mic,compiler,\"-fp-model fast=2 -mGLOB_default_function_attrs=\\\"gather_scatter_loop_unroll=4\\\"\"") - add_compile_options(-xMIC-AVX512 -qoffload -fno-alias -ansi-alias -restrict -qoverride-limits ${MIC_OPTIONS}) - add_definitions(-DLMP_INTEL_OFFLOAD) - else() - if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel") - include(CheckCXXCompilerFlag) - foreach(_FLAG -O2 -fp-model fast=2 -no-prec-div -qoverride-limits -qopt-zmm-usage=high -qno-offload -fno-alias -ansi-alias -restrict) - check_cxx_compiler_flag("${__FLAG}" COMPILER_SUPPORTS${_FLAG}) - if(COMPILER_SUPPORTS${_FLAG}) - add_compile_options(${_FLAG}) - endif() - endforeach() - else() - add_compile_options(-O3 -ffast-math) - endif() + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -xHost -qopenmp -qoffload") + set(MIC_OPTIONS "-qoffload-option,mic,compiler,\"-fp-model fast=2 -mGLOB_default_function_attrs=\\\"gather_scatter_loop_unroll=4\\\"\"") + target_compile_options(lammps PRIVATE -xMIC-AVX512 -qoffload -fno-alias -ansi-alias -restrict -qoverride-limits ${MIC_OPTIONS}) + target_compile_definitions(lammps PRIVATE -DLMP_INTEL_OFFLOAD) +else() + if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + include(CheckCXXCompilerFlag) + foreach(_FLAG -O2 -fp-model fast=2 -no-prec-div -qoverride-limits -qopt-zmm-usage=high -qno-offload -fno-alias -ansi-alias -restrict) + check_cxx_compiler_flag("${_FLAG}" COMPILER_SUPPORTS${_FLAG}) + if(COMPILER_SUPPORTS${_FLAG}) + target_compile_options(lammps PRIVATE ${_FLAG}) + endif() + endforeach() endif() +endif() - # collect sources - set(USER-INTEL_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/USER-INTEL) - set(USER-INTEL_SOURCES ${USER-INTEL_SOURCES_DIR}/fix_intel.cpp - ${USER-INTEL_SOURCES_DIR}/fix_nh_intel.cpp - ${USER-INTEL_SOURCES_DIR}/intel_buffers.cpp - ${USER-INTEL_SOURCES_DIR}/nbin_intel.cpp - ${USER-INTEL_SOURCES_DIR}/npair_intel.cpp) +# collect sources +set(USER-INTEL_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/USER-INTEL) +set(USER-INTEL_SOURCES ${USER-INTEL_SOURCES_DIR}/fix_intel.cpp + ${USER-INTEL_SOURCES_DIR}/fix_nh_intel.cpp + ${USER-INTEL_SOURCES_DIR}/intel_buffers.cpp + ${USER-INTEL_SOURCES_DIR}/nbin_intel.cpp + ${USER-INTEL_SOURCES_DIR}/npair_intel.cpp) - set_property(GLOBAL PROPERTY "USER-INTEL_SOURCES" "${USER-INTEL_SOURCES}") +set_property(GLOBAL PROPERTY "USER-INTEL_SOURCES" "${USER-INTEL_SOURCES}") - # detect styles which have a USER-INTEL version - RegisterStylesExt(${USER-INTEL_SOURCES_DIR} intel USER-INTEL_SOURCES) - RegisterNBinStyle(${USER-INTEL_SOURCES_DIR}/nbin_intel.h) - RegisterNPairStyle(${USER-INTEL_SOURCES_DIR}/npair_intel.h) - RegisterFixStyle(${USER-INTEL_SOURCES_DIR}/fix_intel.h) +# detect styles which have a USER-INTEL version +RegisterStylesExt(${USER-INTEL_SOURCES_DIR} intel USER-INTEL_SOURCES) +RegisterNBinStyle(${USER-INTEL_SOURCES_DIR}/nbin_intel.h) +RegisterNPairStyle(${USER-INTEL_SOURCES_DIR}/npair_intel.h) +RegisterFixStyle(${USER-INTEL_SOURCES_DIR}/fix_intel.h) - get_property(USER-INTEL_SOURCES GLOBAL PROPERTY USER-INTEL_SOURCES) - if(PKG_KSPACE) - list(APPEND USER-INTEL_SOURCES ${USER-INTEL_SOURCES_DIR}/verlet_lrt_intel.cpp) - RegisterIntegrateStyle(${USER-INTEL_SOURCES_DIR}/verlet_lrt_intel.h) - endif() - - list(APPEND LIB_SOURCES ${USER-INTEL_SOURCES}) - include_directories(${USER-INTEL_SOURCES_DIR}) +get_property(USER-INTEL_SOURCES GLOBAL PROPERTY USER-INTEL_SOURCES) +if(PKG_KSPACE) + list(APPEND USER-INTEL_SOURCES ${USER-INTEL_SOURCES_DIR}/verlet_lrt_intel.cpp) + RegisterIntegrateStyle(${USER-INTEL_SOURCES_DIR}/verlet_lrt_intel.h) endif() + +target_sources(lammps PRIVATE ${USER-INTEL_SOURCES}) +target_include_directories(lammps PRIVATE ${USER-INTEL_SOURCES_DIR}) diff --git a/cmake/Modules/Packages/USER-MOLFILE.cmake b/cmake/Modules/Packages/USER-MOLFILE.cmake index 16ffc349940eb3b5a58b1eab4f73b4154523a075..dd6435e56b2b41d6f8802aabf79878039c787134 100644 --- a/cmake/Modules/Packages/USER-MOLFILE.cmake +++ b/cmake/Modules/Packages/USER-MOLFILE.cmake @@ -1,10 +1,11 @@ -if(PKG_USER-MOLFILE) - set(MOLFILE_INCLUDE_DIRS "${LAMMPS_LIB_SOURCE_DIR}/molfile" CACHE STRING "Path to VMD molfile plugin headers") - add_library(molfile INTERFACE) - target_include_directories(molfile INTERFACE ${MOLFILE_INCLUDE_DIRS}) - # no need to link with -ldl on windows - if(NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Windows") - target_link_libraries(molfile INTERFACE ${CMAKE_DL_LIBS}) - endif() - list(APPEND LAMMPS_LINK_LIBS molfile) +set(MOLFILE_INCLUDE_DIRS "${LAMMPS_LIB_SOURCE_DIR}/molfile" CACHE STRING "Path to VMD molfile plugin headers") +add_library(molfile INTERFACE) +if(NOT BUILD_SHARED_LIBS) + install(TARGETS molfile EXPORT LAMMPS_Targets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) endif() +target_include_directories(molfile INTERFACE ${MOLFILE_INCLUDE_DIRS}) +# no need to link with -ldl on windows +if(NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Windows") + target_link_libraries(molfile INTERFACE ${CMAKE_DL_LIBS}) +endif() +target_link_libraries(lammps PRIVATE molfile) diff --git a/cmake/Modules/Packages/USER-NETCDF.cmake b/cmake/Modules/Packages/USER-NETCDF.cmake index 921156f1e033cd1f2c543b960c385e7ed6048db4..91186c84db736137e02fa155362f595231e4ee95 100644 --- a/cmake/Modules/Packages/USER-NETCDF.cmake +++ b/cmake/Modules/Packages/USER-NETCDF.cmake @@ -1,24 +1,20 @@ -if(PKG_USER-NETCDF) - # USER-NETCDF can use NetCDF, Parallel NetCDF (PNetCDF), or both. At least one necessary. - # NetCDF library enables dump style "netcdf", while PNetCDF enables dump style "netcdf/mpiio" - find_package(NetCDF) - if(NETCDF_FOUND) - find_package(PNetCDF) - else(NETCDF_FOUND) - find_package(PNetCDF REQUIRED) - endif(NETCDF_FOUND) +# USER-NETCDF can use NetCDF, Parallel NetCDF (PNetCDF), or both. At least one necessary. +# NetCDF library enables dump style "netcdf", while PNetCDF enables dump style "netcdf/mpiio" +find_package(NetCDF) +if(NETCDF_FOUND) + find_package(PNetCDF) +else(NETCDF_FOUND) + find_package(PNetCDF REQUIRED) +endif(NETCDF_FOUND) - if(NETCDF_FOUND) - include_directories(${NETCDF_INCLUDE_DIRS}) - list(APPEND LAMMPS_LINK_LIBS ${NETCDF_LIBRARIES}) - add_definitions(-DLMP_HAS_NETCDF) - endif(NETCDF_FOUND) +if(NETCDF_FOUND) + target_link_libraries(lammps PRIVATE NetCDF::NetCDF) + target_compile_definitions(lammps PRIVATE -DLMP_HAS_NETCDF) +endif(NETCDF_FOUND) - if(PNETCDF_FOUND) - include_directories(${PNETCDF_INCLUDES}) - list(APPEND LAMMPS_LINK_LIBS ${PNETCDF_LIBRARIES}) - add_definitions(-DLMP_HAS_PNETCDF) - endif(PNETCDF_FOUND) +if(PNETCDF_FOUND) + target_link_libraries(lammps PRIVATE PNetCDF::PNetCDF) + target_compile_definitions(lammps PRIVATE -DLMP_HAS_PNETCDF) +endif(PNETCDF_FOUND) - add_definitions(-DNC_64BIT_DATA=0x0020) -endif() +target_compile_definitions(lammps PRIVATE -DNC_64BIT_DATA=0x0020) diff --git a/cmake/Modules/Packages/USER-OMP.cmake b/cmake/Modules/Packages/USER-OMP.cmake index 668f42f10a56f382d5a1146193a6b7c69da0a00a..a1053ad355d3667245fadc45f126a8f60fd6ba70 100644 --- a/cmake/Modules/Packages/USER-OMP.cmake +++ b/cmake/Modules/Packages/USER-OMP.cmake @@ -1,42 +1,40 @@ -if(PKG_USER-OMP) - set(USER-OMP_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/USER-OMP) - set(USER-OMP_SOURCES ${USER-OMP_SOURCES_DIR}/thr_data.cpp - ${USER-OMP_SOURCES_DIR}/thr_omp.cpp - ${USER-OMP_SOURCES_DIR}/fix_omp.cpp - ${USER-OMP_SOURCES_DIR}/fix_nh_omp.cpp - ${USER-OMP_SOURCES_DIR}/fix_nh_sphere_omp.cpp - ${USER-OMP_SOURCES_DIR}/domain_omp.cpp) - add_definitions(-DLMP_USER_OMP) - set_property(GLOBAL PROPERTY "OMP_SOURCES" "${USER-OMP_SOURCES}") + set(USER-OMP_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/USER-OMP) + set(USER-OMP_SOURCES ${USER-OMP_SOURCES_DIR}/thr_data.cpp + ${USER-OMP_SOURCES_DIR}/thr_omp.cpp + ${USER-OMP_SOURCES_DIR}/fix_omp.cpp + ${USER-OMP_SOURCES_DIR}/fix_nh_omp.cpp + ${USER-OMP_SOURCES_DIR}/fix_nh_sphere_omp.cpp + ${USER-OMP_SOURCES_DIR}/domain_omp.cpp) + target_compile_definitions(lammps PRIVATE -DLMP_USER_OMP) + set_property(GLOBAL PROPERTY "OMP_SOURCES" "${USER-OMP_SOURCES}") - # detects styles which have USER-OMP version - RegisterStylesExt(${USER-OMP_SOURCES_DIR} omp OMP_SOURCES) - RegisterFixStyle(${USER-OMP_SOURCES_DIR}/fix_omp.h) + # detects styles which have USER-OMP version + RegisterStylesExt(${USER-OMP_SOURCES_DIR} omp OMP_SOURCES) + RegisterFixStyle(${USER-OMP_SOURCES_DIR}/fix_omp.h) - get_property(USER-OMP_SOURCES GLOBAL PROPERTY OMP_SOURCES) + get_property(USER-OMP_SOURCES GLOBAL PROPERTY OMP_SOURCES) - # manually add package dependent source files from USER-OMP that do not provide styles + # manually add package dependent source files from USER-OMP that do not provide styles - if(PKG_ASPHERE) - list(APPEND USER-OMP_SOURCES ${USER-OMP_SOURCES_DIR}/fix_nh_asphere_omp.cpp) - endif() + if(PKG_ASPHERE) + list(APPEND USER-OMP_SOURCES ${USER-OMP_SOURCES_DIR}/fix_nh_asphere_omp.cpp) + endif() - if(PKG_RIGID) - list(APPEND USER-OMP_SOURCES ${USER-OMP_SOURCES_DIR}/fix_rigid_nh_omp.cpp) - endif() + if(PKG_RIGID) + list(APPEND USER-OMP_SOURCES ${USER-OMP_SOURCES_DIR}/fix_rigid_nh_omp.cpp) + endif() - if(PKG_USER-REAXC) - list(APPEND USER-OMP_SOURCES ${USER-OMP_SOURCES_DIR}/reaxc_bond_orders_omp.cpp - ${USER-OMP_SOURCES_DIR}/reaxc_hydrogen_bonds_omp.cpp - ${USER-OMP_SOURCES_DIR}/reaxc_nonbonded_omp.cpp - ${USER-OMP_SOURCES_DIR}/reaxc_bonds_omp.cpp - ${USER-OMP_SOURCES_DIR}/reaxc_init_md_omp.cpp - ${USER-OMP_SOURCES_DIR}/reaxc_torsion_angles_omp.cpp - ${USER-OMP_SOURCES_DIR}/reaxc_forces_omp.cpp - ${USER-OMP_SOURCES_DIR}/reaxc_multi_body_omp.cpp - ${USER-OMP_SOURCES_DIR}/reaxc_valence_angles_omp.cpp) - endif() + if(PKG_USER-REAXC) + list(APPEND USER-OMP_SOURCES ${USER-OMP_SOURCES_DIR}/reaxc_bond_orders_omp.cpp + ${USER-OMP_SOURCES_DIR}/reaxc_hydrogen_bonds_omp.cpp + ${USER-OMP_SOURCES_DIR}/reaxc_nonbonded_omp.cpp + ${USER-OMP_SOURCES_DIR}/reaxc_bonds_omp.cpp + ${USER-OMP_SOURCES_DIR}/reaxc_init_md_omp.cpp + ${USER-OMP_SOURCES_DIR}/reaxc_torsion_angles_omp.cpp + ${USER-OMP_SOURCES_DIR}/reaxc_forces_omp.cpp + ${USER-OMP_SOURCES_DIR}/reaxc_multi_body_omp.cpp + ${USER-OMP_SOURCES_DIR}/reaxc_valence_angles_omp.cpp) + endif() - list(APPEND LIB_SOURCES ${USER-OMP_SOURCES}) - include_directories(${USER-OMP_SOURCES_DIR}) -endif() + target_sources(lammps PRIVATE ${USER-OMP_SOURCES}) + target_include_directories(lammps PRIVATE ${USER-OMP_SOURCES_DIR}) diff --git a/cmake/Modules/Packages/USER-PLUMED.cmake b/cmake/Modules/Packages/USER-PLUMED.cmake index 426ae2df2a8b84fe1c60042d446832fe12a15471..e7605b6bca20eae70c660dad1cd63601e1f2f83d 100644 --- a/cmake/Modules/Packages/USER-PLUMED.cmake +++ b/cmake/Modules/Packages/USER-PLUMED.cmake @@ -1,96 +1,103 @@ -if(PKG_USER-PLUMED) - set(PLUMED_MODE "static" CACHE STRING "Linkage mode for Plumed2 library") - set(PLUMED_MODE_VALUES static shared runtime) - set_property(CACHE PLUMED_MODE PROPERTY STRINGS ${PLUMED_MODE_VALUES}) - validate_option(PLUMED_MODE PLUMED_MODE_VALUES) - string(TOUPPER ${PLUMED_MODE} PLUMED_MODE) +set(PLUMED_MODE "static" CACHE STRING "Linkage mode for Plumed2 library") +set(PLUMED_MODE_VALUES static shared runtime) +set_property(CACHE PLUMED_MODE PROPERTY STRINGS ${PLUMED_MODE_VALUES}) +validate_option(PLUMED_MODE PLUMED_MODE_VALUES) +string(TOUPPER ${PLUMED_MODE} PLUMED_MODE) - set(PLUMED_LINK_LIBS "") - if(PLUMED_MODE STREQUAL "STATIC") - find_package(LAPACK REQUIRED) - find_package(BLAS REQUIRED) - find_package(GSL REQUIRED) - list(APPEND LAPACK_LIBRARIES ${BLAS_LIBRARIES}) - list(APPEND PLUMED_LINK_LIBS ${LAPACK_LIBRARIES} ${GSL_LIBRARIES}) - find_package(ZLIB QUIET) - if(ZLIB_FOUND) - list(APPEND PLUMED_LINK_LIBS ${ZLIB_LIBRARIES}) - endif() +set(PLUMED_LINK_LIBS) +if(PLUMED_MODE STREQUAL "STATIC") + find_package(LAPACK REQUIRED) + find_package(BLAS REQUIRED) + find_package(GSL REQUIRED) + list(APPEND PLUMED_LINK_LIBS ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} GSL::gsl) + find_package(ZLIB QUIET) + if(ZLIB_FOUND) + list(APPEND PLUMED_LINK_LIBS ZLIB::ZLIB) + endif() + find_package(FFTW3 QUIET) + if(FFTW3_FOUND) + list(APPEND PLUMED_LINK_LIBS FFTW3::FFTW3) endif() +endif() - find_package(PkgConfig QUIET) - set(DOWNLOAD_PLUMED_DEFAULT ON) - if(PKG_CONFIG_FOUND) - pkg_check_modules(PLUMED QUIET plumed) - if(PLUMED_FOUND) - set(DOWNLOAD_PLUMED_DEFAULT OFF) - endif() +find_package(PkgConfig QUIET) +set(DOWNLOAD_PLUMED_DEFAULT ON) +if(PKG_CONFIG_FOUND) + pkg_check_modules(PLUMED QUIET plumed) + if(PLUMED_FOUND) + set(DOWNLOAD_PLUMED_DEFAULT OFF) endif() +endif() - option(DOWNLOAD_PLUMED "Download Plumed package instead of using an already installed one" ${DOWNLOAD_PLUMED_DEFAULT}) - if(DOWNLOAD_PLUMED) - if(BUILD_MPI) - set(PLUMED_CONFIG_MPI "--enable-mpi") - set(PLUMED_CONFIG_CC ${CMAKE_MPI_C_COMPILER}) - set(PLUMED_CONFIG_CXX ${CMAKE_MPI_CXX_COMPILER}) - else() - set(PLUMED_CONFIG_MPI "--disable-mpi") - set(PLUMED_CONFIG_CC ${CMAKE_C_COMPILER}) - set(PLUMED_CONFIG_CXX ${CMAKE_CXX_COMPILER}) - endif() - if(BUILD_OMP) - set(PLUMED_CONFIG_OMP "--enable-openmp") - else() - set(PLUMED_CONFIG_OMP "--disable-openmp") - endif() - message(STATUS "PLUMED download requested - we will build our own") - if(PLUMED_MODE STREQUAL "STATIC") - set(PLUMED_BUILD_BYPRODUCTS "/lib/libplumed.a") - elseif(PLUMED_MODE STREQUAL "SHARED") - set(PLUMED_BUILD_BYPRODUCTS "/lib/libplumed${CMAKE_SHARED_LIBRARY_SUFFIX};/lib/libplumedKernel${CMAKE_SHARED_LIBRARY_SUFFIX}") - elseif(PLUMED_MODE STREQUAL "RUNTIME") - set(PLUMED_BUILD_BYPRODUCTS "/lib/libplumedWrapper.a") - endif() - include(ExternalProject) - ExternalProject_Add(plumed_build - URL https://github.com/plumed/plumed2/releases/download/v2.6.0/plumed-src-2.6.0.tgz - URL_MD5 204d2edae58d9b10ba3ad460cad64191 - BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND /configure --prefix= - ${CONFIGURE_REQUEST_PIC} - --enable-modules=all - ${PLUMED_CONFIG_MPI} - ${PLUMED_CONFIG_OMP} - CXX=${PLUMED_CONFIG_CXX} - CC=${PLUMED_CONFIG_CC} - BUILD_BYPRODUCTS ${PLUMED_BUILD_BYPRODUCTS} - ) - ExternalProject_get_property(plumed_build INSTALL_DIR) - set(PLUMED_INSTALL_DIR ${INSTALL_DIR}) - list(APPEND LAMMPS_DEPS plumed_build) - if(PLUMED_MODE STREQUAL "STATIC") - add_definitions(-D__PLUMED_WRAPPER_CXX=1) - list(APPEND LAMMPS_LINK_LIBS ${PLUMED_INSTALL_DIR}/lib/libplumed.a ${PLUMED_LINK_LIBS} ${CMAKE_DL_LIBS}) - elseif(PLUMED_MODE STREQUAL "SHARED") - list(APPEND LAMMPS_LINK_LIBS ${PLUMED_INSTALL_DIR}/lib/libplumed${CMAKE_SHARED_LIBRARY_SUFFIX} ${PLUMED_INSTALL_DIR}/lib/libplumedKernel${CMAKE_SHARED_LIBRARY_SUFFIX} ${CMAKE_DL_LIBS}) - elseif(PLUMED_MODE STREQUAL "RUNTIME") - add_definitions(-D__PLUMED_HAS_DLOPEN=1 -D__PLUMED_DEFAULT_KERNEL=${PLUMED_INSTALL_DIR}/lib/libplumedKernel${CMAKE_SHARED_LIBRARY_SUFFIX}) - list(APPEND LAMMPS_LINK_LIBS ${PLUMED_INSTALL_DIR}/lib/libplumedWrapper.a -rdynamic ${CMAKE_DL_LIBS}) - endif() - set(PLUMED_INCLUDE_DIRS "${PLUMED_INSTALL_DIR}/include") +option(DOWNLOAD_PLUMED "Download Plumed package instead of using an already installed one" ${DOWNLOAD_PLUMED_DEFAULT}) +if(DOWNLOAD_PLUMED) + if(BUILD_MPI) + set(PLUMED_CONFIG_MPI "--enable-mpi") + set(PLUMED_CONFIG_CC ${CMAKE_MPI_C_COMPILER}) + set(PLUMED_CONFIG_CXX ${CMAKE_MPI_CXX_COMPILER}) else() - find_package(PkgConfig REQUIRED) - pkg_check_modules(PLUMED REQUIRED plumed) - if(PLUMED_MODE STREQUAL "STATIC") - add_definitions(-D__PLUMED_WRAPPER_CXX=1) - include(${PLUMED_LIBDIR}/plumed/src/lib/Plumed.cmake.static) - elseif(PLUMED_MODE STREQUAL "SHARED") - include(${PLUMED_LIBDIR}/plumed/src/lib/Plumed.cmake.shared) - elseif(PLUMED_MODE STREQUAL "RUNTIME") - add_definitions(-D__PLUMED_HAS_DLOPEN=1 -D__PLUMED_DEFAULT_KERNEL=${PLUMED_LIBDIR}/libplumedKernel${CMAKE_SHARED_LIBRARY_SUFFIX}) - include(${PLUMED_LIBDIR}/plumed/src/lib/Plumed.cmake.runtime) - endif() - list(APPEND LAMMPS_LINK_LIBS ${PLUMED_LOAD}) + set(PLUMED_CONFIG_MPI "--disable-mpi") + set(PLUMED_CONFIG_CC ${CMAKE_C_COMPILER}) + set(PLUMED_CONFIG_CXX ${CMAKE_CXX_COMPILER}) + endif() + if(BUILD_OMP) + set(PLUMED_CONFIG_OMP "--enable-openmp") + else() + set(PLUMED_CONFIG_OMP "--disable-openmp") + endif() + message(STATUS "PLUMED download requested - we will build our own") + if(PLUMED_MODE STREQUAL "STATIC") + set(PLUMED_BUILD_BYPRODUCTS "/lib/libplumed.a") + elseif(PLUMED_MODE STREQUAL "SHARED") + set(PLUMED_BUILD_BYPRODUCTS "/lib/libplumed${CMAKE_SHARED_LIBRARY_SUFFIX};/lib/libplumedKernel${CMAKE_SHARED_LIBRARY_SUFFIX}") + elseif(PLUMED_MODE STREQUAL "RUNTIME") + set(PLUMED_BUILD_BYPRODUCTS "/lib/libplumedWrapper.a") + endif() + include(ExternalProject) + ExternalProject_Add(plumed_build + URL https://github.com/plumed/plumed2/releases/download/v2.6.0/plumed-src-2.6.0.tgz + URL_MD5 204d2edae58d9b10ba3ad460cad64191 + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND /configure --prefix= + ${CONFIGURE_REQUEST_PIC} + --enable-modules=all + ${PLUMED_CONFIG_MPI} + ${PLUMED_CONFIG_OMP} + CXX=${PLUMED_CONFIG_CXX} + CC=${PLUMED_CONFIG_CC} + BUILD_BYPRODUCTS ${PLUMED_BUILD_BYPRODUCTS} + ) + ExternalProject_get_property(plumed_build INSTALL_DIR) + add_library(LAMMPS::PLUMED UNKNOWN IMPORTED) + add_dependencies(LAMMPS::PLUMED plumed_build) + if(NOT BUILD_SHARED_LIBS) + install(CODE "MESSAGE(FATAL_ERROR \"Installing liblammps with downloaded libraries is currently not supported.\")") + endif() + if(PLUMED_MODE STREQUAL "STATIC") + set_target_properties(LAMMPS::PLUMED PROPERTIES INTERFACE_COMPILE_DEFINITIONS "__PLUMED_WRAPPER_CXX=1") + set_target_properties(LAMMPS::PLUMED PROPERTIES IMPORTED_LOCATION ${INSTALL_DIR}/lib/libplumed.a INTERFACE_LINK_LIBRARIES "${PLUMED_LINK_LIBS};${CMAKE_DL_LIBS}") + elseif(PLUMED_MODE STREQUAL "SHARED") + set_target_properties(LAMMPS::PLUMED PROPERTIES IMPORTED_LOCATION ${INSTALL_DIR}/lib/libplumed${CMAKE_SHARED_LIBRARY_SUFFIX} INTERFACE_LINK_LIBRARIES "${INSTALL_DIR}/lib/libplumedKernel${CMAKE_SHARED_LIBRARY_SUFFIX};${CMAKE_DL_LIBS}") + elseif(PLUMED_MODE STREQUAL "RUNTIME") + set_target_properties(LAMMPS::PLUMED PROPERTIES INTERFACE_COMPILE_DEFINITIONS "__PLUMED_HAS_DLOPEN=1;__PLUMED_DEFAULT_KERNEL=${INSTALL_DIR}/lib/libplumedKernel${CMAKE_SHARED_LIBRARY_SUFFIX}") + set_target_properties(LAMMPS::PLUMED PROPERTIES IMPORTED_LOCATION ${INSTALL_DIR}/lib/libplumedWrapper.a INTERFACE_LINK_LIBRARIES "${CMAKE_DL_LIBS}") + endif() + set_target_properties(LAMMPS::PLUMED PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) + file(MAKE_DIRECTORY ${INSTALL_DIR}/include) +else() + find_package(PkgConfig REQUIRED) + pkg_check_modules(PLUMED REQUIRED plumed) + add_library(LAMMPS::PLUMED INTERFACE IMPORTED) + if(PLUMED_MODE STREQUAL "STATIC") + set_target_properties(LAMMPS::PLUMED PROPERTIES INTERFACE_COMPILE_DEFINITIONS "__PLUMED_WRAPPER_CXX=1") + include(${PLUMED_LIBDIR}/plumed/src/lib/Plumed.cmake.static) + elseif(PLUMED_MODE STREQUAL "SHARED") + include(${PLUMED_LIBDIR}/plumed/src/lib/Plumed.cmake.shared) + elseif(PLUMED_MODE STREQUAL "RUNTIME") + set_target_properties(LAMMPS::PLUMED PROPERTIES INTERFACE_COMPILE_DEFINITIONS "__PLUMED_HAS_DLOPEN=1;__PLUMED_DEFAULT_KERNEL=${PLUMED_LIBDIR}/libplumedKernel${CMAKE_SHARED_LIBRARY_SUFFIX}") + include(${PLUMED_LIBDIR}/plumed/src/lib/Plumed.cmake.runtime) endif() - include_directories(${PLUMED_INCLUDE_DIRS}) + set_target_properties(LAMMPS::PLUMED PROPERTIES INTERFACE_LINK_LIBRARIES "${PLUMED_LOAD}") + set_target_properties(LAMMPS::PLUMED PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${PLUMED_INCLUDE_DIRS}") endif() +target_link_libraries(lammps PRIVATE LAMMPS::PLUMED) diff --git a/cmake/Modules/Packages/USER-QMMM.cmake b/cmake/Modules/Packages/USER-QMMM.cmake index 544455868e55189232b3774afb5f3e088eaac118..76bb76fa5afe6160dc797566592fed7d145b2160 100644 --- a/cmake/Modules/Packages/USER-QMMM.cmake +++ b/cmake/Modules/Packages/USER-QMMM.cmake @@ -1,13 +1,12 @@ -if(PKG_USER-QMMM) - enable_language(C) +enable_language(C) - if(NOT BUILD_LIB) - message(FATAL_ERROR "Building a QM/MM executable with USER-QMMM requires BUILD_LIB=yes") - endif() - if(NOT BUILD_SHARED_LIBS) - message(WARNING "It is recommended to use BUILD_SHARED_LIBS=yes with USER-QMMM") - endif() - add_library(qmmm STATIC ${LAMMPS_LIB_SOURCE_DIR}/qmmm/libqmmm.c) - list(APPEND LAMMPS_LINK_LIBS qmmm) - target_include_directories(qmmm PUBLIC ${LAMMPS_LIB_SOURCE_DIR}/qmmm) +if(NOT BUILD_SHARED_LIBS) + message(WARNING "It is recommended to use BUILD_SHARED_LIBS=yes with USER-QMMM") endif() +add_library(qmmm STATIC ${LAMMPS_LIB_SOURCE_DIR}/qmmm/libqmmm.c) +if(NOT BUILD_SHARED_LIBS) + install(TARGETS qmmm EXPORT LAMMPS_Targets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) +endif() +set_target_properties(qmmm PROPERTIES OUTPUT_NAME lammps_qmmm${LAMMPS_MACHINE}) +target_link_libraries(lammps PRIVATE qmmm) +target_include_directories(qmmm PUBLIC ${LAMMPS_LIB_SOURCE_DIR}/qmmm) diff --git a/cmake/Modules/Packages/USER-QUIP.cmake b/cmake/Modules/Packages/USER-QUIP.cmake index 93096a2f54bec679eda388df567f5b35662641cf..790f27def7bfffa7341e1fea45aab8292cda9464 100644 --- a/cmake/Modules/Packages/USER-QUIP.cmake +++ b/cmake/Modules/Packages/USER-QUIP.cmake @@ -1,5 +1,3 @@ -if(PKG_USER-QUIP) - enable_language(Fortran) - find_package(QUIP REQUIRED) - list(APPEND LAMMPS_LINK_LIBS ${QUIP_LIBRARIES} ${LAPACK_LIBRARIES}) -endif() +enable_language(Fortran) +find_package(QUIP REQUIRED) +target_link_libraries(lammps PRIVATE QUIP::QUIP ${LAPACK_LIBRARIES}) diff --git a/cmake/Modules/Packages/USER-SCAFACOS.cmake b/cmake/Modules/Packages/USER-SCAFACOS.cmake index 8bb9e63605970e4b36bddb8964f41a713d7e9bd7..ee12e754e648b497577cb172d03271f134b6dae4 100644 --- a/cmake/Modules/Packages/USER-SCAFACOS.cmake +++ b/cmake/Modules/Packages/USER-SCAFACOS.cmake @@ -1,76 +1,62 @@ -if(PKG_USER-SCAFACOS) - enable_language(Fortran) - enable_language(C) +enable_language(Fortran) +enable_language(C) - find_package(GSL REQUIRED) - find_package(PkgConfig QUIET) - find_package(MPI REQUIRED) - set(DOWNLOAD_SCAFACOS_DEFAULT ON) - if(PKG_CONFIG_FOUND) - pkg_check_modules(SCAFACOS QUIET scafacos) - if(SCAFACOS_FOUND) - set(DOWNLOAD_SCAFACOS_DEFAULT OFF) - endif() +find_package(GSL REQUIRED) +find_package(PkgConfig QUIET) +find_package(MPI REQUIRED) +set(DOWNLOAD_SCAFACOS_DEFAULT ON) +if(PKG_CONFIG_FOUND) + pkg_check_modules(SCAFACOS QUIET scafacos) + if(SCAFACOS_FOUND) + set(DOWNLOAD_SCAFACOS_DEFAULT OFF) endif() - option(DOWNLOAD_SCAFACOS "Download ScaFaCoS library instead of using an already installed one" ${DOWNLOAD_SCAFACOS_DEFAULT}) - if(DOWNLOAD_SCAFACOS) - message(STATUS "ScaFaCoS download requested - we will build our own") - include(ExternalProject) - ExternalProject_Add(scafacos_build - URL https://github.com/scafacos/scafacos/releases/download/v1.0.1/scafacos-1.0.1.tar.gz - URL_MD5 bd46d74e3296bd8a444d731bb10c1738 - CONFIGURE_COMMAND /configure --prefix= --disable-doc - --enable-fcs-solvers=fmm,p2nfft,direct,ewald,p3m - --with-internal-fftw --with-internal-pfft - --with-internal-pnfft ${CONFIGURE_REQUEST_PIC} - FC=${CMAKE_MPI_Fortran_COMPILER} - CXX=${CMAKE_MPI_CXX_COMPILER} - CC=${CMAKE_MPI_C_COMPILER} - F77= - BUILD_BYPRODUCTS - /lib/libfcs.a - /lib/libfcs_direct.a - /lib/libfcs_ewald.a - /lib/libfcs_fmm.a - /lib/libfcs_p2nfft.a - /lib/libfcs_p3m.a - /lib/libfcs_near.a - /lib/libfcs_gridsort.a - /lib/libfcs_resort.a - /lib/libfcs_redist.a - /lib/libfcs_common.a - /lib/libfcs_pnfft.a - /lib/libfcs_pfft.a - /lib/libfcs_fftw3_mpi.a - /lib/libfcs_fftw3.a - ) - ExternalProject_get_property(scafacos_build INSTALL_DIR) - set(SCAFACOS_BUILD_DIR ${INSTALL_DIR}) - set(SCAFACOS_INCLUDE_DIRS ${SCAFACOS_BUILD_DIR}/include) - list(APPEND LAMMPS_DEPS scafacos_build) - # list and order from pkg_config file of ScaFaCoS build - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs.a) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_direct.a) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_ewald.a) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_fmm.a) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_p2nfft.a) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_p3m.a) - list(APPEND LAMMPS_LINK_LIBS ${GSL_LIBRARIES}) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_near.a) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_gridsort.a) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_resort.a) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_redist.a) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_common.a) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_pnfft.a) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_pfft.a) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_fftw3_mpi.a) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_fftw3.a) - list(APPEND LAMMPS_LINK_LIBS ${MPI_Fortran_LIBRARIES}) - list(APPEND LAMMPS_LINK_LIBS ${MPI_C_LIBRARIES}) - else() - find_package(PkgConfig REQUIRED) - pkg_check_modules(SCAFACOS REQUIRED scafacos) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_LDFLAGS}) +endif() +option(DOWNLOAD_SCAFACOS "Download ScaFaCoS library instead of using an already installed one" ${DOWNLOAD_SCAFACOS_DEFAULT}) +if(DOWNLOAD_SCAFACOS) + message(STATUS "ScaFaCoS download requested - we will build our own") + include(ExternalProject) + ExternalProject_Add(scafacos_build + URL https://github.com/scafacos/scafacos/releases/download/v1.0.1/scafacos-1.0.1.tar.gz + URL_MD5 bd46d74e3296bd8a444d731bb10c1738 + CONFIGURE_COMMAND /configure --prefix= --disable-doc + --enable-fcs-solvers=fmm,p2nfft,direct,ewald,p3m + --with-internal-fftw --with-internal-pfft + --with-internal-pnfft ${CONFIGURE_REQUEST_PIC} + FC=${CMAKE_MPI_Fortran_COMPILER} + CXX=${CMAKE_MPI_CXX_COMPILER} + CC=${CMAKE_MPI_C_COMPILER} + F77= + BUILD_BYPRODUCTS + /lib/libfcs.a + /lib/libfcs_direct.a + /lib/libfcs_ewald.a + /lib/libfcs_fmm.a + /lib/libfcs_p2nfft.a + /lib/libfcs_p3m.a + /lib/libfcs_near.a + /lib/libfcs_gridsort.a + /lib/libfcs_resort.a + /lib/libfcs_redist.a + /lib/libfcs_common.a + /lib/libfcs_pnfft.a + /lib/libfcs_pfft.a + /lib/libfcs_fftw3_mpi.a + /lib/libfcs_fftw3.a + ) + ExternalProject_get_property(scafacos_build INSTALL_DIR) + file(MAKE_DIRECTORY ${INSTALL_DIR}/include) + add_library(LAMMPS::SCAFACOS UNKNOWN IMPORTED) + set_target_properties(LAMMPS::SCAFACOS PROPERTIES + IMPORTED_LOCATION "${INSTALL_DIR}/lib/libfcs.a" + INTERFACE_INCLUDE_DIRECTORIES "${INSTALL_DIR}/include" + INTERFACE_LINK_LIBRARIES "${INSTALL_DIR}/lib/libfcs.a;${INSTALL_DIR}/lib/libfcs_direct.a;${INSTALL_DIR}/lib/libfcs_ewald.a;${INSTALL_DIR}/lib/libfcs_fmm.a;${INSTALL_DIR}/lib/libfcs_p2nfft.a;${INSTALL_DIR}/lib/libfcs_p3m.a;GSL::gsl;${INSTALL_DIR}/lib/libfcs_near.a;${INSTALL_DIR}/lib/libfcs_gridsort.a;${INSTALL_DIR}/lib/libfcs_resort.a;${INSTALL_DIR}/lib/libfcs_redist.a;${INSTALL_DIR}/lib/libfcs_common.a;${INSTALL_DIR}/lib/libfcs_pnfft.a;${INSTALL_DIR}/lib/libfcs_pfft.a;${INSTALL_DIR}/lib/libfcs_fftw3_mpi.a;${INSTALL_DIR}/lib/libfcs_fftw3.a;MPI::MPI_Fortran;MPI::MPI_C") + target_link_libraries(lammps PRIVATE LAMMPS::SCAFACOS) + add_dependencies(LAMMPS::SCAFACOS scafacos_build) + if(NOT BUILD_SHARED_LIBS) + install(CODE "MESSAGE(FATAL_ERROR \"Installing liblammps with downloaded libraries is currently not supported.\")") endif() - include_directories(${SCAFACOS_INCLUDE_DIRS}) +else() + find_package(PkgConfig REQUIRED) + pkg_check_modules(SCAFACOS REQUIRED IMPORTED_TARGET scafacos) + target_link_libraries(lammps PRIVATE PkgConfig::SCAFACOS) endif() diff --git a/cmake/Modules/Packages/USER-SDPD.cmake b/cmake/Modules/Packages/USER-SDPD.cmake index 530dcf2bd915446b24e5521e97f59b24504df340..c9f419d91849f1ebebc7aeaab91eea167255fcb8 100644 --- a/cmake/Modules/Packages/USER-SDPD.cmake +++ b/cmake/Modules/Packages/USER-SDPD.cmake @@ -1,13 +1,13 @@ # Fix rigid/meso requires RIGID to be installed -if(PKG_USER-SDPD) - set(USER-SDPD_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/USER-SDPD) +set(USER-SDPD_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/USER-SDPD) - get_property(hlist GLOBAL PROPERTY FIX) - if(NOT PKG_RIGID) - list(REMOVE_ITEM hlist ${USER-SDPD_SOURCES_DIR}/fix_rigid_meso.h) - list(REMOVE_ITEM LIB_SOURCES ${USER-SDPD_SOURCES_DIR}/fix_rigid_meso.cpp) - endif() - set_property(GLOBAL PROPERTY FIX "${hlist}") - - include_directories(${USER-SDPD_SOURCES_DIR}) +get_property(hlist GLOBAL PROPERTY FIX) +if(NOT PKG_RIGID) + list(REMOVE_ITEM hlist ${USER-SDPD_SOURCES_DIR}/fix_rigid_meso.h) + get_target_property(LAMMPS_SOURCES lammps SOURCES) + list(REMOVE_ITEM LAMMPS_SOURCES ${USER-SDPD_SOURCES_DIR}/fix_rigid_meso.cpp) + set_property(TARGET lammps PROPERTY SOURCES ${LAMMPS_SOURCES}) endif() +set_property(GLOBAL PROPERTY FIX "${hlist}") + +target_include_directories(lammps PRIVATE ${USER-SDPD_SOURCES_DIR}) diff --git a/cmake/Modules/Packages/USER-SMD.cmake b/cmake/Modules/Packages/USER-SMD.cmake index a868918e378649a501b4a616ba1bf0aafd49b2db..0a5165471c6b22949c1fe737d67fdb5f2c6b045a 100644 --- a/cmake/Modules/Packages/USER-SMD.cmake +++ b/cmake/Modules/Packages/USER-SMD.cmake @@ -1,28 +1,31 @@ -if(PKG_USER-SMD) - find_package(Eigen3 NO_MODULE) - if(EIGEN3_FOUND) - set(DOWNLOAD_EIGEN3_DEFAULT OFF) - else() - set(DOWNLOAD_EIGEN3_DEFAULT ON) +find_package(Eigen3 NO_MODULE) +if(EIGEN3_FOUND) + set(DOWNLOAD_EIGEN3_DEFAULT OFF) +else() + set(DOWNLOAD_EIGEN3_DEFAULT ON) +endif() +option(DOWNLOAD_EIGEN3 "Download Eigen3 instead of using an already installed one)" ${DOWNLOAD_EIGEN3_DEFAULT}) +if(DOWNLOAD_EIGEN3) + message(STATUS "Eigen3 download requested - we will build our own") + include(ExternalProject) + ExternalProject_Add(Eigen3_build + URL http://bitbucket.org/eigen/eigen/get/3.3.7.tar.gz + URL_MD5 f2a417d083fe8ca4b8ed2bc613d20f07 + CONFIGURE_COMMAND "" BUILD_COMMAND "" INSTALL_COMMAND "" + ) + ExternalProject_get_property(Eigen3_build SOURCE_DIR) + add_library(LAMMPS::EIGEN3 INTERFACE IMPORTED) + set_target_properties(LAMMPS::EIGEN3 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${SOURCE_DIR}") + target_link_libraries(lammps PRIVATE LAMMPS::EIGEN3) + add_dependencies(LAMMPS::EIGEN3 Eigen3_build) + if(NOT BUILD_SHARED_LIBS) + install(CODE "MESSAGE(FATAL_ERROR \"Installing liblammps with downloaded libraries is currently not supported.\")") endif() - option(DOWNLOAD_EIGEN3 "Download Eigen3 instead of using an already installed one)" ${DOWNLOAD_EIGEN3_DEFAULT}) - if(DOWNLOAD_EIGEN3) - message(STATUS "Eigen3 download requested - we will build our own") - include(ExternalProject) - ExternalProject_Add(Eigen3_build - URL http://bitbucket.org/eigen/eigen/get/3.3.7.tar.gz - URL_MD5 f2a417d083fe8ca4b8ed2bc613d20f07 - CONFIGURE_COMMAND "" BUILD_COMMAND "" INSTALL_COMMAND "" - ) - ExternalProject_get_property(Eigen3_build SOURCE_DIR) - set(EIGEN3_INCLUDE_DIR ${SOURCE_DIR}) - list(APPEND LAMMPS_DEPS Eigen3_build) - else() - find_package(Eigen3 NO_MODULE) - mark_as_advanced(Eigen3_DIR) - if(NOT EIGEN3_FOUND) - message(FATAL_ERROR "Eigen3 not found, help CMake to find it by setting EIGEN3_INCLUDE_DIR, or set DOWNLOAD_EIGEN3=ON to download it") - endif() +else() + find_package(Eigen3 NO_MODULE) + mark_as_advanced(Eigen3_DIR) + if(NOT EIGEN3_FOUND) + message(FATAL_ERROR "Eigen3 not found, help CMake to find it by setting EIGEN3_INCLUDE_DIR, or set DOWNLOAD_EIGEN3=ON to download it") endif() - include_directories(${EIGEN3_INCLUDE_DIR}) + target_link_libraries(lammps PRIVATE Eigen3::Eigen) endif() diff --git a/cmake/Modules/Packages/USER-VTK.cmake b/cmake/Modules/Packages/USER-VTK.cmake index d264577ca2f77f5af3b2cbac590f4d9fc07500d9..8c4445167a10cf1e66c2e0444c7eb15afde85b12 100644 --- a/cmake/Modules/Packages/USER-VTK.cmake +++ b/cmake/Modules/Packages/USER-VTK.cmake @@ -1,6 +1,4 @@ -if(PKG_USER-VTK) - find_package(VTK REQUIRED NO_MODULE) - include(${VTK_USE_FILE}) - add_definitions(-DLAMMPS_VTK) - list(APPEND LAMMPS_LINK_LIBS ${VTK_LIBRARIES}) -endif() +find_package(VTK REQUIRED NO_MODULE) +include(${VTK_USE_FILE}) +target_compile_definitions(lammps PRIVATE -DLAMMPS_VTK) +target_link_libraries(lammps PRIVATE ${VTK_LIBRARIES}) diff --git a/cmake/Modules/Packages/VORONOI.cmake b/cmake/Modules/Packages/VORONOI.cmake index 5418132034f369dcc9e4d939125754aac4a2e287..4635466a50096808744d697def16a87c4fcf0255 100644 --- a/cmake/Modules/Packages/VORONOI.cmake +++ b/cmake/Modules/Packages/VORONOI.cmake @@ -1,43 +1,47 @@ -if(PKG_VORONOI) - find_package(VORO) - if(VORO_FOUND) - set(DOWNLOAD_VORO_DEFAULT OFF) +find_package(VORO) +if(VORO_FOUND) + set(DOWNLOAD_VORO_DEFAULT OFF) +else() + set(DOWNLOAD_VORO_DEFAULT ON) +endif() +option(DOWNLOAD_VORO "Download and compile the Voro++ library instead of using an already installed one" ${DOWNLOAD_VORO_DEFAULT}) +if(DOWNLOAD_VORO) + message(STATUS "Voro++ download requested - we will build our own") + include(ExternalProject) + + if(BUILD_SHARED_LIBS) + set(VORO_BUILD_CFLAGS "${CMAKE_SHARED_LIBRARY_CXX_FLAGS} ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BTYPE}}") else() - set(DOWNLOAD_VORO_DEFAULT ON) + set(VORO_BUILD_CFLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BTYPE}}") endif() - option(DOWNLOAD_VORO "Download and compile the Voro++ library instead of using an already installed one" ${DOWNLOAD_VORO_DEFAULT}) - if(DOWNLOAD_VORO) - message(STATUS "Voro++ download requested - we will build our own") - include(ExternalProject) - - if(BUILD_SHARED_LIBS) - set(VORO_BUILD_CFLAGS "${CMAKE_SHARED_LIBRARY_CXX_FLAGS} ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BTYPE}}") - else() - set(VORO_BUILD_CFLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BTYPE}}") - endif() - if(APPLE) - get_filename_component(VORO_CXX ${CMAKE_CXX_COMPILER} NAME_WE) - set(VORO_BUILD_OPTIONS CXX=${VORO_CXX} CFLAGS=${VORO_BUILD_CFLAGS}) - else() - set(VORO_BUILD_OPTIONS CXX=${CMAKE_CXX_COMPILER} CFLAGS=${VORO_BUILD_CFLAGS}) - endif() - - ExternalProject_Add(voro_build - URL https://download.lammps.org/thirdparty/voro++-0.4.6.tar.gz - URL_MD5 2338b824c3b7b25590e18e8df5d68af9 - CONFIGURE_COMMAND "" BUILD_COMMAND make ${VORO_BUILD_OPTIONS} BUILD_IN_SOURCE 1 INSTALL_COMMAND "" - BUILD_BYPRODUCTS /src/libvoro++.a - ) - ExternalProject_get_property(voro_build SOURCE_DIR) - set(VORO_LIBRARIES ${SOURCE_DIR}/src/libvoro++.a) - set(VORO_INCLUDE_DIRS ${SOURCE_DIR}/src) - list(APPEND LAMMPS_DEPS voro_build) + if(APPLE) + get_filename_component(VORO_CXX ${CMAKE_CXX_COMPILER} NAME_WE) + set(VORO_BUILD_OPTIONS CXX=${VORO_CXX} CFLAGS=${VORO_BUILD_CFLAGS}) else() - find_package(VORO) - if(NOT VORO_FOUND) - message(FATAL_ERROR "Voro++ library not found. Help CMake to find it by setting VORO_LIBRARY and VORO_INCLUDE_DIR, or set DOWNLOAD_VORO=ON to download it") - endif() + set(VORO_BUILD_OPTIONS CXX=${CMAKE_CXX_COMPILER} CFLAGS=${VORO_BUILD_CFLAGS}) + endif() + + ExternalProject_Add(voro_build + URL https://download.lammps.org/thirdparty/voro++-0.4.6.tar.gz + URL_MD5 2338b824c3b7b25590e18e8df5d68af9 + CONFIGURE_COMMAND "" BUILD_COMMAND make ${VORO_BUILD_OPTIONS} BUILD_IN_SOURCE 1 INSTALL_COMMAND "" + BUILD_BYPRODUCTS /src/libvoro++.a + ) + ExternalProject_get_property(voro_build SOURCE_DIR) + file(MAKE_DIRECTORY ${SOURCE_DIR}/src) + add_library(LAMMPS::VORO UNKNOWN IMPORTED) + set_target_properties(LAMMPS::VORO PROPERTIES + IMPORTED_LOCATION "${SOURCE_DIR}/src/libvoro++.a" + INTERFACE_INCLUDE_DIRECTORIES "${SOURCE_DIR}/src") + target_link_libraries(lammps PRIVATE LAMMPS::VORO) + add_dependencies(LAMMPS::VORO voro_build) + if(NOT BUILD_SHARED_LIBS) + install(CODE "MESSAGE(FATAL_ERROR \"Installing liblammps with downloaded libraries is currently not supported.\")") + endif() +else() + find_package(VORO) + if(NOT VORO_FOUND) + message(FATAL_ERROR "Voro++ library not found. Help CMake to find it by setting VORO_LIBRARY and VORO_INCLUDE_DIR, or set DOWNLOAD_VORO=ON to download it") endif() - include_directories(${VORO_INCLUDE_DIRS}) - list(APPEND LAMMPS_LINK_LIBS ${VORO_LIBRARIES}) + target_link_libraries(lammps PRIVATE VORO::VORO) endif() diff --git a/cmake/Modules/Testing.cmake b/cmake/Modules/Testing.cmake index 0eeef00fe9ab4b603fe319825662a6b827f46ac7..f0a5f4519d4c3367f38f0f81b784647407df9457 100644 --- a/cmake/Modules/Testing.cmake +++ b/cmake/Modules/Testing.cmake @@ -2,7 +2,7 @@ # Testing ############################################################################### option(ENABLE_TESTING "Enable testing" OFF) -if(ENABLE_TESTING AND BUILD_EXE) +if(ENABLE_TESTING) enable_testing() option(LAMMPS_TESTING_SOURCE_DIR "Location of lammps-testing source directory" "") option(LAMMPS_TESTING_GIT_TAG "Git tag of lammps-testing" "master") @@ -28,7 +28,7 @@ if(ENABLE_TESTING AND BUILD_EXE) "https://github.com/lammps/lammps-testing in LAMMPS_TESTING_SOURCE_DIR") endif() - add_test(ShowHelp ${CMAKE_BINARY_DIR}/${LAMMPS_BINARY} -help) + add_test(NAME ShowHelp COMMAND $ -help) if(EXISTS ${LAMMPS_TESTING_SOURCE_DIR}) message(STATUS "Running test discovery...") @@ -42,8 +42,7 @@ if(ENABLE_TESTING AND BUILD_EXE) string(REPLACE "-" "_" TEST_NAME ${TEST_NAME}) string(REPLACE "+" "_" TEST_NAME ${TEST_NAME}) set(TEST_NAME "test_core_${TEST_NAME}_serial") - add_test(${TEST_NAME} ${CMAKE_BINARY_DIR}/${LAMMPS_BINARY} -in ${SCRIPT_NAME}) - set_tests_properties(${TEST_NAME} PROPERTIES WORKING_DIRECTORY ${PARENT_DIR}) + add_test(NAME ${TEST_NAME} COMMAND $ -in ${SCRIPT_NAME} WORKING_DIRECTORY ${PARENT_DIR}) endforeach() list(LENGTH TEST_SCRIPTS NUM_TESTS) diff --git a/cmake/README.md b/cmake/README.md deleted file mode 100644 index b9dd6d437370c395ac1075304159c0052c097f33..0000000000000000000000000000000000000000 --- a/cmake/README.md +++ /dev/null @@ -1,2320 +0,0 @@ -# Building LAMMPS using CMake - -LAMMPS recently acquired support for building with CMake thanks to the efforts -of Christoph Junghans (LANL) and Richard Berger (Temple U). One of the key -strengths of CMake is that it can generate the necessary build system files of -your own personal preference. It also enables using common development IDEs such -as Eclipse, Visual Studio, QtCreator, Xcode and many more for LAMMPS -development. - -CMake can both be used as a command-line (CLI) utility `cmake` or through one of -its GUIs. `ccmake` is a text-based ui to configure and build CMake projects. -`cmake-gui` is a graphical frontend for running CMake operations that works on -Linux desktop environments, Windows and MacOS X. - -The following is a tutorial-style introduction in using the CMake system. It -should give you the necessary foundation to understand how to do the most common -tasks, act as a reference and provide examples of typical use cases. - -## Table of Contents - - * [Quick Start for the Impatient](#quick-start-for-the-impatient) - * [Building LAMMPS using cmake](#building-lammps-using-cmake-1) - * [Prerequisites](#prerequisites) - * [Build directory vs. Source Directory](#build-directory-vs-source-directory) - * [Defining and using presets](#defining-and-using-presets) - * [Reference](#reference) - * [Common CMake Configuration Options](#common-cmake-configuration-options) - * [LAMMPS Configuration Options](#lammps-configuration-options) - * [Parallelization and Accelerator Packages](#parallelization-and-accelerator-packages) - * [Default Packages](#default-packages) - * [Other Packages](#other-packages) - * [User Packages](#user-packages) - * [Package-Specific Configuration Options](#package-specific-configuration-options) - * [KSPACE Package](#kspace-package) - * [MKL](#mkl) - * [FFTW3](#fftw3) - * [BLAS](#blas) - * [LAPACK](#lapack) - * [PYTHON Package](#python-package) - * [GPU Package](#gpu-package) - * [MESSAGE Package](#message-package) - * [MSCG Package](#mscg-package) - * [VORONOI Package](#voronoi-package) - * [USER-LATTE Package](#user-latte-package) - * [USER-PLUMED Package](#user-plumed-package) - * [USER-SCAFACOS Package](#user-scafacos-package) - * [USER-SMD Package](#user-smd-package) - * [Optional Features](#optional-features) - * [zlib support](#zlib-support) - * [JPEG support](#jpeg-support) - * [PNG support](#png-support) - * [GZIP support](#gzip-support) - * [FFMPEG support](#ffmpeg-support) - * [Compilers](#compilers) - * [Building with GNU Compilers](#building-with-gnu-compilers) - * [Building with Intel Compilers](#building-with-intel-compilers) - * [Building with LLVM/Clang Compilers](#building-with-llvmclang-compilers) - -## Quick Start for the Impatient -If you want to skip ahead and just run the compilation using `cmake`, please -find a minimal example below. Together with the options reference below, this -should get you started. - -```bash -git clone https://github.com/lammps/lammps.git -mkdir lammps/build -cd lammps/build -cmake [-D OPTION_A=VALUE_A -D OPTION_B=VALUE_B ...] ../cmake -make -``` - -# Building LAMMPS using `cmake` - -## Prerequisites -This tutorial assumes you are running in a command-line environment using a -shell like Bash. - -* Linux: any terminal window will work -* MacOS X: launch the Terminal app -* Windows 10: install and run "Bash on Windows" (aka Ubuntu on Windows) - -Before we start, please download the latest and greatest version of LAMMPS from -GitHub. You can either download it as a tarball or ZIP file, or via git. When -you start with a fresh lammps directory, the contents should look like this: - -```bash -$ ls -bench doc lib potentials README tools -cmake examples LICENSE python src -``` - -## Build directory vs. Source Directory - -By using CMake we separate building LAMMPS into multiple phases: - -1. **Configuration**: define which features we want to enable/disable and how it should be compiled -2. **Compilation**: compile each source file and generate binaries and libraries based on the configuration -3. **Installation** (Optional): finally we can install the generated binaries on our system - -In the GNU Make based build system of LAMMPS, configuration occurs by running -special make targets like `make yes-MOLECULAR`. These targets modify the -**source directory** (`src/`) directory by copying package files and patching -Makefile. In some cases you are force to manually edit Makefiles to add compiler -options and/or correct include directory and library paths. - -These edits and copy operations are no longer necessary when compiling with -CMake. The source directory stays untouched, so you compile LAMMPS in many -different variants using the same source code checkout. It enables true -**out-of-source** builds. - -When using Cmake, you can compile in **any** folder outside of the source -directory. Any working directory you choose becomes a so-called **build -directory**. All configuration files and compilation results are stored in this -folder. We recommend calling it something like `build/`. - -Let's have a look a quick example, where we get the greatest and latest version -of LAMMPS from GitHub via git: -```bash -git clone https://github.com/lammps/lammps.git -``` - -We then create a new `build` folder and make it our current working directory: -``` -mkdir lammps/build -cd lammps/build -``` - -To configure LAMMPS we run `cmake` inside of this folder. However it requires at -least one argument. `cmake` needs to read the LAMMPS `CMakeLists.txt` file to -know what to do. This file is located in the `cmake/` subdirectory of the -lammps checkout. To run `cmake` add the relative or absolute path to the `cmake/` -directory as first argument. - -E.g., if the current working directory is `lammps/build` you can specify the -relative path to `lammps/cmake` as follows: -``` -cmake ../cmake -``` - -You could also specify the absolute path: -``` -cmake /path/to/my/lammps/folder/cmake -``` - -Please note: **This does NOT compile the code!** Running cmake only configures -the next build. It generates the necessary files to compile the code. On -Unix/Linux it defaults to generating Makefiles. You can also choose other output -formats to generate files for Eclipse, Xcode or Visual Studio which are -supported on other platorms. - -To compile LAMMPS once the Makefiles are generated, simply type `make` in the -build directory. - -``` -make -``` -# Defining and using presets - -The CMake build exposes a lot of different options. In the old build system -some of the package selections were possible by using special make target like -`make yes-std` or `make no-lib`. Achieving a similar result with cmake requires -specifying all options manually. This can quickly become a very long command -line that is hard to handle. While these could be stored in a simple script -file, there is another way of defining "presets" to compile LAMMPS in a certain -way. Since the cmake build process - contrary to the conventional build system - -includes the compilation of the bundled libraries into the standard build process, -the grouping of those presets is somewhat different. - -A preset is a regular CMake script file that can use constructs such as -variables, lists and for-loops to manipulate configuration options and create -an [*initial cache*](https://cmake.org/cmake/help/v3.12/manual/cmake.1.html). -Options must be set with the `CACHE` and `FORCE` flag to ensure they are -considered even during a second cmake run. - -Such a file can then be passed to cmake via the `-C` flag. Several examples of -presets can be found in the `cmake/presets` folder. - -```bash -# build LAMMPS with all packages enabled which don't use external libraries and enable GPU package -mkdir build -cd build -cmake -C ../cmake/presets/all_on.cmake -C ../cmake/presets/nolib.cmake -D PKG_GPU=on ../cmake -``` - -# Reference - -## Common CMake Configuration Options - - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
CMAKE_INSTALL_PREFIXInstall location where LAMMPS files will be copied to. In the Unix/Linux case with Makefiles this controls what `make install` will do. - Default setting is $HOME/.local. -
CMAKE_BUILD_TYPEControls if debugging symbols are added to the generated binaries -
-
RelWithDebInfo (default)
-
Release
-
Debug
-
MinSizeRel
-
-
CMAKE_VERBOSE_MAKEFILEEnable verbose output from Makefile builds (useful for debugging), the same can be achieved by adding `VERBOSE=1` to the `make` call. -
-
off (default)
-
on
-
-
- - -## LAMMPS Configuration Options - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
LAMMPS_SIZE_LIMITControls the integer sizes used by LAMMPS internally -
-
LAMMPS_SMALLBIG (default)
-
32bit , 64bit
-
LAMMPS_SMALLSMALL
-
32bit , 32bit
-
LAMMPS_BIGBIG
-
64bit , 64bit
-
-
LAMMPS_LONGLONG_TO_LONGWorkaround if your system or MPI version does not recognize long long data types -
-
off (default)
-
on
-
-
LAMMPS_MEMALIGNcontrols the alignment of blocks of memory allocated by LAMMPS -
-
64 (default)
-
-
LAMMPS_EXCEPTIONScontrols whether LAMMPS dies after an error or throws a C++ exception. This is particularly useful when running through the C library interface, since an error - in LAMMPS then doesn't kill the parent process -
-
off (default)
-
on
-
-
LAMMPS_MACHINEallows appending a machine suffix to the generated LAMMPS binary -
-
*none* (default)
-
-
LAMMPS_LIB_SUFFIXallows appending a suffix to the generated LAMMPS library -
-
*none* (default)
-
-
BUILD_LIBcontrol whether to build LAMMPS as a library -
-
off (default)
-
on
-
-
BUILD_EXEcontrol whether to build LAMMPS executable -
-
on (default)
-
off
-
-
BUILD_SHARED_LIBScontrol whether to build LAMMPS as a shared-library -
-
off (default)
-
on
-
-
BUILD_DOCcontrol whether to build LAMMPS documentation -
-
off (default)
-
on
-
-
BUILD_TOOLScontrol whether to build LAMMPS tools -
-
off (default)
-
on
-
-
- -## Parallelization and Accelerator Packages - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
BUILD_MPIcontrol whether to build LAMMPS with MPI support. This will look for - `mpicxx` in your path and use this MPI implementation. -
-
on (default, if found)
-
off
-
-
BUILD_OMPcontrol whether to build LAMMPS with OpenMP support. -
-
on (default, if found)
-
off
-
-
PKG_OPT - A handful of pair styles which are optimized for improved CPU performance on - single or multiple cores. These include EAM, LJ, CHARMM, and Morse potentials. - -
-
off (default)
-
on
-
-
PKG_USER-OMP - Hundreds of pair, fix, compute, bond, angle, dihedral, improper, and kspace - styles which are altered to enable threading on many-core CPUs via OpenMP - directives. - -
-
off (default)
-
on
-
-
PKG_USER-INTEL - Dozens of pair, fix, bond, angle, dihedral, improper, and kspace styles which - are optimized for Intel CPUs and KNLs (Knights Landing). - -
-
off (default)
-
on
-
-
PKG_GPU - Dozens of pair styles and a version of the PPPM long-range Coulombic solver - optimized for GPUs. All such styles have a “gpu†as a suffix in their style - name. The GPU code can be compiled with either CUDA or OpenCL, however the - OpenCL variants are no longer actively maintained and only the CUDA versions - are regularly tested. - -
-
off (default)
-
on
-
-
PKG_KOKKOSDozens of atom, pair, bond, angle, dihedral, improper, fix, compute styles adapted to compile using the Kokkos library which can convert them to OpenMP or CUDA code so that they run efficiently on multicore CPUs, KNLs, or GPUs. -
-
off (default)
-
on
-
-
- -## Default Packages - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
PKG_ASPHEREComputes, time-integration fixes, and pair styles for aspherical particle models including ellipsoids, 2d lines, and 3d triangles. -
-
off (default)
-
on
-
-
PKG_BODYBody-style particles with internal structure. Computes, time-integration fixes, pair styles, as well as the body styles themselves. -
-
off (default)
-
on
-
-
PKG_CLASS2Bond, angle, dihedral, improper, and pair styles for the COMPASS CLASS2 molecular force field. -
-
off (default)
-
on
-
-
PKG_COLLOIDCoarse-grained finite-size colloidal particles. Pair styles and fix wall styles for colloidal interactions. Includes the Fast Lubrication Dynamics (FLD) method for hydrodynamic interactions, which is a simplified approximation to Stokesian dynamics. -
-
off (default)
-
on
-
-
PKG_COMPRESSCompressed output of dump files via the zlib compression library, using dump styles with a “gz†in their style name. -
-
off (default)
-
on
-
-
PKG_CORESHELLCompute and pair styles that implement the adiabatic core/shell model for polarizability. The pair styles augment Born, Buckingham, and Lennard-Jones styles with core/shell capabilities. The compute temp/cs command calculates the temperature of a system with core/shell particles. -
-
off (default)
-
on
-
-
PKG_DIPOLEAn atom style and several pair styles for point dipole models with short-range or long-range interactions. -
-
off (default)
-
on
-
-
PKG_GRANULARPair styles and fixes for finite-size granular particles, which interact with each other and boundaries via frictional and dissipative potentials. -
-
off (default)
-
on
-
-
PKG_KSPACEA variety of long-range Coulombic solvers, as well as pair styles which compute the corresponding short-range pairwise Coulombic interactions. These include Ewald, particle-particle particle-mesh (PPPM), and multilevel summation method (MSM) solvers. -
-
off (default)
-
on
-
-
PKG_MANYBODY - A variety of manybody and bond-order potentials. These include (AI)REBO, BOP, - EAM, EIM, Stillinger-Weber, and Tersoff potentials. - -
-
off (default)
-
on
-
-
PKG_MC - Several fixes and a pair style that have Monte Carlo (MC) or MC-like - attributes. These include fixes for creating, breaking, and swapping bonds, - for performing atomic swaps, and performing grand-canonical MC (GCMC) in - conjunction with dynamics. - -
-
off (default)
-
on
-
-
PKG_MISC - A variety of compute, fix, pair, dump styles with specialized capabilities that - don’t align with other packages. - -
-
off (default)
-
on
-
-
PKG_MOLECULE - A large number of atom, pair, bond, angle, dihedral, improper styles that are - used to model molecular systems with fixed covalent bonds. The pair styles - include the Dreiding (hydrogen-bonding) and CHARMM force fields, and a TIP4P - water model. - -
-
off (default)
-
on
-
-
PKG_PERI - An atom style, several pair styles which implement different Peridynamics - materials models, and several computes which calculate diagnostics. - Peridynamics is a a particle-based meshless continuum model. - -
-
off (default)
-
on
-
-
PKG_QEQ - Several fixes for performing charge equilibration (QEq) via different - algorithms. These can be used with pair styles that perform QEq as part of - their formulation. - -
-
off (default)
-
on
-
-
PKG_REPLICA - A collection of multi-replica methods which can be used when running multiple - LAMMPS simulations (replicas). See Section 6.5 for an overview of how to run - multi-replica simulations in LAMMPS. Methods in the package include nudged - elastic band (NEB), parallel replica dynamics (PRD), temperature accelerated - dynamics (TAD), parallel tempering, and a verlet/split algorithm for - performing long-range Coulombics on one set of processors, and the remainder - of the force field calcalation on another set. - -
-
off (default)
-
on
-
-
PKG_RIGID - Fixes which enforce rigid constraints on collections of atoms or particles. - This includes SHAKE and RATTLE, as well as varous rigid-body integrators for a - few large bodies or many small bodies. Also several computes which calculate - properties of rigid bodies. - -
-
off (default)
-
on
-
-
PKG_SHOCK - Fixes for running impact simulations where a shock-wave passes through a - material. - -
-
off (default)
-
on
-
-
PKG_SPINModel atomic magnetic spins classically, coupled to atoms moving in the usual manner via MD. Various pair, fix, and compute styles. -
-
off (default)
-
on
-
-
PKG_SNAP - A pair style for the spectral neighbor analysis potential (SNAP). SNAP is - methodology for deriving a highly accurate classical potential fit to a large - archive of quantum mechanical (DFT) data. Also several computes which analyze - attributes of the potential. - -
-
off (default)
-
on
-
-
PKG_SRD - A pair of fixes which implement the Stochastic Rotation Dynamics (SRD) method - for coarse-graining of a solvent, typically around large colloidal particles. - -
-
off (default)
-
on
-
-
- -## Other Packages - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
PKG_KIMA pair_style kim command which is a wrapper on the Knowledge Base for Interatomic Models (KIM) repository of interatomic potentials, enabling any of them to be used in LAMMPS simulations. -
-
off (default)
-
on
-
-
PKG_PYTHONEnable support for Python scripting inside of LAMMPS. -
-
off (default)
-
on
-
-
PKG_MESSAGECommands to use LAMMPS as either a client or server and couple it to another application. -
-
off (default)
-
on
-
-
PKG_MSCG - A fix mscg command which can parameterize a Multi-Scale Coarse-Graining (MSCG) - model using the open-source MS-CG library. - -
-
off (default)
-
on
-
-
PKG_MPIIO - Support for parallel output/input of dump and restart files via the MPIIO library. - -
-
off (default)
-
on
-
-
PKG_POEMS - A fix that wraps the Parallelizable Open source Efficient Multibody Software - (POEMS) library, which is able to simulate the dynamics of articulated body - systems. These are systems with multiple rigid bodies (collections of - particles) whose motion is coupled by connections at hinge points. - -
-
off (default)
-
on
-
-
PKG_LATTE - A fix command which wraps the LATTE DFTB code, so that molecular dynamics can - be run with LAMMPS using density-functional tight-binding quantum forces - calculated by LATTE. - -
-
off (default)
-
on
-
-
PKG_VORONOI - A compute command which calculates the Voronoi tesselation of a collection of atoms by wrapping the Voro++ library. This can be used to calculate the local volume or each atoms or its near neighbors. - -
-
off (default)
-
on
-
-
- -## User Packages - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
PKG_USER-ADIOSADIOS is a high-performance I/O library. This package implements the dump “atom/adios†and dump “custom/adios†commands to write data using the ADIOS library. -
-
off (default)
-
on
-
-
PKG_USER-ATC - ATC stands for atoms-to-continuum. This package implements a fix atc command - to either couple molecular dynamics with continuum finite element equations or - perform on-the-fly conversion of atomic information to continuum fields. - -
-
off (default)
-
on
-
-
PKG_USER-AWPMD - AWPMD stands for Antisymmetrized Wave Packet Molecular Dynamics. This package - implements an atom, pair, and fix style which allows electrons to be treated - as explicit particles in a classical molecular dynamics model. - -
-
off (default)
-
on
-
-
PKG_USER-BOCS - This package provides fix bocs, a modified version of fix npt which includes the pressure correction to the barostat as outlined in: N. J. H. Dunn and W. G. Noid, “Bottom-up coarse-grained models that accurately describe the structure, pressure, and compressibility of molecular liquids,†J. Chem. Phys. 143, 243148 (2015). - -
-
off (default)
-
on
-
-
PKG_USER-CGDNA - Several pair styles, a bond style, and integration fixes for coarse-grained - models of single- and double-stranded DNA based on the oxDNA model of Doye, - Louis and Ouldridge at the University of Oxford. This includes Langevin-type - rigid-body integrators with improved stability. - -
-
off (default)
-
on
-
-
PKG_USER-CGSDK - Several pair styles and an angle style which implement the coarse-grained SDK - model of Shinoda, DeVane, and Klein which enables simulation of ionic liquids, - electrolytes, lipids and charged amino acids. - -
-
off (default)
-
on
-
-
PKG_USER-COLVARS - COLVARS stands for collective variables, which can be used to implement - various enhanced sampling methods, including Adaptive Biasing Force, - Metadynamics, Steered MD, Umbrella Sampling and Restraints. A fix colvars - command is implemented which wraps a COLVARS library, which implements these - methods. simulations. - -
-
off (default)
-
on
-
-
PKG_USER-DIFFRACTION - Two computes and a fix for calculating x-ray and electron diffraction - intensities based on kinematic diffraction theory. - -
-
off (default)
-
on
-
-
PKG_USER-DPD - DPD stands for dissipative particle dynamics. This package implements - coarse-grained DPD-based models for energetic, reactive molecular crystalline - materials. It includes many pair styles specific to these systems, including - for reactive DPD, where each particle has internal state for multiple species - and a coupled set of chemical reaction ODEs are integrated each timestep. - Highly accurate time integrators for isothermal, isoenergetic, isobaric and - isenthalpic conditions are included. These enable long timesteps via the - Shardlow splitting algorithm. - -
-
off (default)
-
on
-
-
PKG_USER-DRUDE - Fixes, pair styles, and a compute to simulate thermalized Drude oscillators as - a model of polarization. - -
-
off (default)
-
on
-
-
PKG_USER-EFF - EFF stands for electron force field which allows a classical MD code to model - electrons as particles of variable radius. This package contains atom, pair, - fix and compute styles which implement the eFF as described in A. - Jaramillo-Botero, J. Su, Q. An, and W.A. Goddard III, JCC, 2010. The eFF - potential was first introduced by Su and Goddard, in 2007. - -
-
off (default)
-
on
-
-
PKG_USER-FEP - FEP stands for free energy perturbation. This package provides methods for - performing FEP simulations by using a fix adapt/fep command with soft-core - pair potentials, which have a “soft†in their style name. - -
-
off (default)
-
on
-
-
PKG_USER-H5MD - H5MD stands for HDF5 for MD. HDF5 is a portable, binary, self-describing file - format, used by many scientific simulations. H5MD is a format for molecular - simulations, built on top of HDF5. This package implements a dump h5md command - to output LAMMPS snapshots in this format. - -
-
off (default)
-
on
-
-
PKG_USER-LB - Fixes which implement a background Lattice-Boltzmann (LB) fluid, which can be - used to model MD particles influenced by hydrodynamic forces. - -
-
off (default)
-
on
-
-
PKG_USER-MANIFOLD - Several fixes and a “manifold†class which enable simulations of particles - constrained to a manifold (a 2D surface within the 3D simulation box). This is - done by applying the RATTLE constraint algorithm to formulate single-particle - constraint functions g(xi,yi,zi) = 0 and their derivative (i.e. the normal of - the manifold) n = grad(g). - -
-
off (default)
-
on
-
-
PKG_USER-MEAMC - A pair style for the modified embedded atom (MEAM) potential translated from - the Fortran version in the MEAM package to plain C++. In contrast to the MEAM - package, no library needs to be compiled and the pair style can be - instantiated multiple times. - -
-
off (default)
-
on
-
-
PKG_USER-MESO - Several extensions of the the dissipative particle dynamics (DPD) method. - Specifically, energy-conserving DPD (eDPD) that can model non-isothermal - processes, many-body DPD (mDPD) for simulating vapor-liquid coexistence, and - transport DPD (tDPD) for modeling advection-diffusion-reaction systems. The - equations of motion of these DPD extensions are integrated through a modified - velocity-Verlet (MVV) algorithm. - -
-
off (default)
-
on
-
-
PKG_USER-MGPT - A pair style which provides a fast implementation of the quantum-based MGPT - multi-ion potentials. The MGPT or model GPT method derives from - first-principles DFT-based generalized pseudopotential theory (GPT) through a - series of systematic approximations valid for mid-period transition metals - with nearly half-filled d bands. The MGPT method was originally developed by - John Moriarty at LLNL. The pair style in this package calculates forces and - energies using an optimized matrix-MGPT algorithm due to Tomas Oppelstrup at - LLNL. - -
-
off (default)
-
on
-
-
PKG_USER-MISC - A potpourri of (mostly) unrelated features contributed to LAMMPS by users. - Each feature is a single fix, compute, pair, bond, angle, dihedral, improper, - or command style. - -
-
off (default)
-
on
-
-
PKG_USER-MOFFF - Pair, angle and improper styles needed to employ the MOF-FF force field by - Schmid and coworkers with LAMMPS. MOF-FF is a first principles derived force - field with the primary aim to simulate MOFs and related porous framework - materials, using spherical Gaussian charges. It is described in S. Bureekaew - et al., Phys. Stat. Sol. B 2013, 250, 1128-1141. For the usage of MOF-FF see - the example in the example directory as well as the MOF+ website. - -
-
off (default)
-
on
-
-
PKG_USER-MOLFILE - A dump molfile command which uses molfile plugins that are bundled with the - VMD molecular visualization and analysis program, to enable LAMMPS to dump - snapshots in formats compatible with various molecular simulation tools. - -
-
off (default)
-
on
-
-
PKG_USER-NETCDF - Dump styles for writing NetCDF formatted dump files. NetCDF is a portable, - binary, self-describing file format developed on top of HDF5. The file - contents follow the AMBER NetCDF trajectory conventions - (http://ambermd.org/netcdf/nctraj.xhtml), but include extensions. - -
-
off (default)
-
on
-
-
PKG_USER-PHONON - A fix phonon command that calculates dynamical matrices, which can then be - used to compute phonon dispersion relations, directly from molecular dynamics - simulations. - -
-
off (default)
-
on
-
-
PKG_USER-PLUMED - The fix plumed command allows you to use the PLUMED free energy plugin for molecular dynamics to analyze and bias your LAMMPS trajectory on the fly. The PLUMED library is called from within the LAMMPS input script by using the fix plumed command. - -
-
off (default)
-
on
-
-
PKG_USER-PTM - A compute ptm/atom command that calculates local structure characterization using the Polyhedral Template Matching methodology. - -
-
off (default)
-
on
-
-
PKG_USER-QTB - Two fixes which provide a self-consistent quantum treatment of vibrational modes in a classical molecular dynamics simulation. By coupling the MD simulation to a colored thermostat, it introduces zero point energy into the system, altering the energy power spectrum and the heat capacity to account for their quantum nature. This is useful when modeling systems at temperatures lower than their classical limits or when temperatures ramp across the classical limits in a simulation. - -
-
off (default)
-
on
-
-
PKG_USER-QUIP - A pair_style quip command which wraps the QUIP libAtoms library, which - includes a variety of interatomic potentials, including Gaussian Approximation - Potential (GAP) models developed by the Cambridge University group. - -
-
off (default)
-
on
-
-
PKG_USER-QMMM - A fix qmmm command which allows LAMMPS to be used in a QM/MM simulation, - currently only in combination with the Quantum ESPRESSO package. - -
-
off (default)
-
on
-
-
PKG_USER-REAXC - A pair style which implements the ReaxFF potential in C/C++ (in contrast to - the REAX package and its Fortran library). ReaxFF is universal reactive force - field. See the src/USER-REAXC/README file for more info on differences between - the two packages. Also two fixes for monitoring molecules as bonds are created - and destroyed. - -
-
off (default)
-
on
-
-
PKG_USER-SCAFACOS - A KSpace style which wraps the ScaFaCoS Coulomb solver library to compute long-range Coulombic interactions. - -
-
off (default)
-
on
-
-
PKG_USER-SDPD - A pair style for smoothed dissipative particle dynamics (SDPD), which is an - extension of smoothed particle hydrodynamics (SPH) to mesoscale where thermal - fluctuations are important (see the USER-SPH package). Also two fixes for - moving and rigid body integration of SPH/SDPD particles (particles of - atom_style meso). -
-
off (default)
-
on
-
-
PKG_USER-SMD - An atom style, fixes, computes, and several pair styles which implements - smoothed Mach dynamics (SMD) for solids, which is a model related to smoothed - particle hydrodynamics (SPH) for liquids (see the USER-SPH package). - -
-
off (default)
-
on
-
-
PKG_USER-SMTBQ - A pair style which implements a Second Moment Tight Binding model with QEq - charge equilibration (SMTBQ) potential for the description of ionocovalent - bonds in oxides. - -
-
off (default)
-
on
-
-
PKG_USER-SPH - An atom style, fixes, computes, and several pair styles which implements - smoothed particle hydrodynamics (SPH) for liquids. See the related USER-SMD - package package for smooth Mach dynamics (SMD) for solids. - -
-
off (default)
-
on
-
-
PKG_USER-TALLY - Several compute styles that can be called when pairwise interactions are - calculated to tally information (forces, heat flux, energy, stress, etc) about - individual interactions. - -
-
off (default)
-
on
-
-
PKG_USER-UEF - A fix style for the integration of the equations of motion under extensional - flow with proper boundary conditions, as well as several supporting compute - styles and an output option. - -
-
off (default)
-
on
-
-
PKG_USER-VTK - A dump vtk command which outputs snapshot info in the VTK format, enabling - visualization by Paraview or other visualization packages. - -
-
off (default)
-
on
-
-
PKG_USER-YAFF - Some potentials that are also implemented in the Yet Another Force Field (YAFF) code. - The expressions and their use are discussed in the following papers: - - -
-
off (default)
-
on
-
-
- -## Package-Specific Configuration Options - -### KSPACE Package - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
FFT -

FFT library for KSPACE package

-

If either MKL or FFTW is selected cmake will try to locate - these libraries automatically. To control which one should be used please see - the options below for each FFT library. Otherwise it will default to KISS - FFT.

- 		-
-
FFTW3
-
FFTW2
-
MKL
-
KISS (default)
-
-
FFT_SINGLEUse single-precision floating-point in FFT -
-
off (default = double precision)
-
on
-
-
FFT_PACKOptimization for FFT -
-
array (default)
-
pointer
-
memcpy
-
-
- -### FFTW3 - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
FFTW3_INCLUDE_DIRSpath to FFTW3 include files -
FFTW3_LIBRARIESlist of paths to FFTW3 libraries -
- -### MKL - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
MKL_INCLUDE_DIRSpath to MKL include files -
MKL_LIBRARIESlist of paths to MKL libraries -
- -### BLAS - -See [FindBLAS documentation](https://cmake.org/cmake/help/latest/module/FindBLAS.html) - -### LAPACK - -See [FindLAPACK documentation](https://cmake.org/cmake/help/latest/module/FindLAPACK.html) - -### PYTHON Package - -See [FindPYTHON documentation](https://cmake.org/cmake/help/latest/module/FindPython.html) - -### USER-INTEL Package - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
INTEL_ARCHTarget architecture for USER-INTEL package -
-
cpu (default)
-
knl
-
-
INTEL_LRT_MODEHow to support Long-range thread mode in Verlet integration -
-
threads (default, if pthreads available)
-
none (default, if pthreads not available)
-
c++11
-
-
- -### GPU Package -The GPU package builds a support library which can either use OpenCL or CUDA as -target API. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
GPU_APIAPI used by GPU package -
-
opencl (default)
-
cuda
-
-
GPU_PRECPrecision size used by GPU package kernels -
-
mixed (default)
-
single
-
double
-
-
OCL_TUNE (OpenCL only)Tuning target for OpenCL driver code -
-
generic (default)
-
intel (Intel CPU)
-
phi (Intel Xeon Phi)
-
fermi (NVIDIA)
-
kepler (NVIDIA)
-
cypress (AMD)
-
-
GPU_ARCH (CUDA only)CUDA SM architecture targeted by GPU package -
-
sm_20 (Fermi)
-
sm_30 (Kepler) (default)
-
sm_50 (Maxwell)
-
sm_60 (Pascal)
-
sm_70 (Volta)
-
sm_75 (Turing)
-
-
CUDPP_OPT (CUDA only)Enable CUDA Performance Primitives Optimizations -
-
on (default)
-
off
-
-
CUDA_MPS_SUPPORT (CUDA only)Enable tweaks for running with Nvidia CUDA Multi-process services daemon -
-
on
-
off (default)
-
-
BIN2C (CUDA only)Path to bin2c executable, will automatically pick up the first one in your $PATH.(automatic)
- -### KIM Package - -Requires installation of the KIM library with API v2 - -If `DOWNLOAD_KIM` is set, the KIM library will be downloaded and built inside -the CMake build directory. If the KIM library is already on your system (in a -location CMake cannot find it), set the `PKG_CONFIG_PATH` environment variable -so that `libkim-api` can be found. - - - - - - - - - - - - - - - - -
OptionDescriptionValues
DOWNLOAD_KIMDownload KIM API v2 and compile it as part of the build. -
-
off (default)
-
on
-
-
- -### MESSAGE Package - -This package can optionally include support for messaging via sockets, using the open-source [ZeroMQ library](http://zeromq.org/), which must be installed on your system. - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
MESSAGE_ZMQBuild with ZeroMQ support -
-
off (default)
-
on
-
-
ZMQ_LIBRARY - ZMQ library file (only needed if at custom location) - -
ZMG_INCLUDE_DIR - Provide include directory of existing ZMQ installation (only needed if at custom location) - -
- -### MSCG Package - -Requires installation of the MSCG library - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
DOWNLOAD_MSCGDownload MSCG and compile it as part of the build -
-
off (default)
-
on
-
-
MSCG_LIBRARY - MSCG library file (only needed if at custom location) - -
MSCG_INCLUDE_DIR - Provide include directory of existing MSCG installation (only needed if at custom location) - -
- -### VORONOI Package - -Requires installation of the Voro++ library - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
DOWNLOAD_VORODownload Voro++ and compile it as part of the build -
-
off (default)
-
on
-
-
VORO_LIBRARY - Voro++ library file (only needed if at custom location) - -
VORO_INCLUDE_DIR - Provide include directory of existing Voro++ installation (only needed if at custom location) - -
- -### USER-COLVARS Package - -Requires a C++11 compiler to compile with the Lepton library included. - - - - - - - - - - - - - - - -
OptionDescriptionValues
COLVARS_LEPTONEnable the use of the Lepton library inside the Colvars library. - -
-
on (default)
-
off
-
-
- -### USER-LATTE Package - -Requires installation of the LATTE library - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
DOWNLOAD_LATTEDownload LATTE and compile it as part of the build -
-
off (default)
-
on
-
-
LATTE_LIBRARY - LATTE library file (only needed if at custom location) - -
- -### USER-PLUMED Package - -Requires installation of the PLUMED library - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
DOWNLOAD_PLUMEDDownload PLUMED and compile it as part of the build -
-
off (default)
-
on
-
-
PLUMED_MODE - Determines the linkage mode for the PLUMED library. - -
-
static (default)
-
shared
-
runtime
-
-
- -### USER-LATTE Package - -Requires installation of the LATTE library - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
DOWNLOAD_LATTEDownload LATTE and compile it as part of the build -
-
off (default)
-
on
-
-
LATTE_LIBRARY - LATTE library file (only needed if at custom location) - -
- -### USER-SMD Package - -Requires installation of the Eigen3 library - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
DOWNLOAD_EIGEN3Download Eigen3 and compile it as part of the build -
-
off (default)
-
on
-
-
EIGEN3_INCLUDE_DIR - Provide include directory of existing Eigen3 installation (only needed if at custom location) - -
- - -### USER-SCAFACOS Package - -To build with this package, you must download and build the [ScaFaCoS Coulomb solver library](http://www.scafacos.de/) - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
DOWNLOAD_SCAFACOSDownload SCAFACOS and compile it as part of the build -
-
off (default)
-
on
-
-
SCAFACOS_LIBRARY - SCAFACOS library file (only needed if at custom location) - -
SCAFACOS_INCLUDE_DIR - SCAFACOS include directory (only needed if at custom location) - -
- -## Optional Features - -### zlib support - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
ZLIB_INCLUDE_DIR -
ZLIB_LIBRARIES -
- -### JPEG support - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
WITH_JPEGEnables/Disable JPEG support in LAMMPS -
-
yes (default, if found)
-
no
-
-
JPEG_INCLUDE_DIR -
JPEG_LIBRARIES -
- -### PNG support -(requires zlib support) - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
WITH_PNGEnables/Disable PNG support in LAMMPS -
-
yes (default, if found)
-
no
-
-
PNG_INCLUDE_DIR -
PNG_LIBRARIES -
- -### GZIP support - -requires `gzip` to be in your `PATH` - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
WITH_GZIPEnables/Disable GZIP support in LAMMPS -
-
yes (default, if found)
-
no
-
-
GZIP_EXECUTABLEPath to gzip executable, will automatically pick up the first one in your $PATH.(automatic)
- -### FFMPEG support - -requires `ffmpeg` to be in your `PATH` - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
WITH_FFMPEGEnables/Disable FFMPEG support in LAMMPS -
-
yes (default, if found)
-
no
-
-
FFMPEG_EXECUTABLEPath to ffmpeg executable, will automatically pick up the first one in your $PATH.(automatic)
- - -## Compilers - -By default, `cmake` will use your environment C/C++/Fortran compilers for a -build. It uses the `CC`, `CXX` and `FC` environment variables to detect which -compilers should be used. However, these values will be cached after the first -run of `cmake`. Subsequent runs of `cmake` will ignore changes in these -environment variables. To ensure the correct values are used you avoid the -cache by setting the `CMAKE_C_COMPILER`, `CMAKE_CXX_COMPILER`, -`CMAKE_Fortran_COMPILER` options directly. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionDefault
CMAKE_C_COMPILERC Compiler which should be used by CMakevalue of `CC` environment variable at first `cmake` run
CMAKE_CXX_COMPILERC++ compiler which should be used by CMake - value of `CXX` environment variable at first `cmake` run -
CMAKE_Fortran_COMPILERC++ compiler which should be used by CMake - value of `FC` environment variable at first `cmake` run -
CXX_COMPILER_LAUNCHERCMake will run this tool and pass the compiler and its arguments to the tool. Some example tools are distcc and ccache. - (empty) -
- -### Building with GNU Compilers - -```bash -cmake -D CMAKE_C_COMPILER=gcc -D CMAKE_CXX_COMPILER=g++ -D CMAKE_Fortran_COMPILER=gfortran ../cmake -``` - -### Building with Intel Compilers - -```bash -cmake -D CMAKE_C_COMPILER=icc -D CMAKE_CXX_COMPILER=icpc -D CMAKE_Fortran_COMPILER=ifort ../cmake -``` - - -### Building with LLVM/Clang Compilers - -```bash -cmake -D CMAKE_C_COMPILER=clang -D CMAKE_CXX_COMPILER=clang++ -D CMAKE_Fortran_COMPILER=flang ../cmake -``` - -## LAMMPS Developer Options - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
ENABLE_TESTINGControl wheather to add tests via CTest -
-
off (default)
-
on
-
-
LAMMPS_TESTING_SOURCE_DIRCustom location of lammps-testing repository (optional). If not specified it will download it via Git -
LAMMPS_TESTING_GIT_TAGIf lammps-testing repository is cloned, this is the tag/commit that will be checked out -
-
master (default)
-
-
ENABLE_COVERAGEEnables code coverage support via gcov and adds a gcovr build target to generate a coverage report. -
-
off (default)
-
on
-
-
ENABLE_SANITIZE_ADDRESSEnables Address Sanitizer support when compiling using GCC or Clang for detecting memory leaks in binaries while running them. See https://clang.llvm.org/docs/AddressSanitizer.html -
-
off (default)
-
on
-
-
ENABLE_SANITIZE_UNDEFINEDEnables Undefined Behavior Sanitizer support when compiling using GCC or Clang for detecting code that is running into undefined behavior of the language. See https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html -
-
off (default)
-
on
-
-
ENABLE_SANITIZE_THREADEnables Thread Sanitizer support when compiling using GCC or Clang for detecting data races in binaries while running them. See https://clang.llvm.org/docs/ThreadSanitizer.html -
-
off (default)
-
on
-
-
diff --git a/cmake/pkgconfig/liblammps.pc.in b/cmake/pkgconfig/liblammps.pc.in index a8710ca224c55e413a1ea9a3b8feee2c9e0c3a18..ca2c6191b7846dc298021b60ba2b95518b3859e2 100644 --- a/cmake/pkgconfig/liblammps.pc.in +++ b/cmake/pkgconfig/liblammps.pc.in @@ -18,12 +18,6 @@ # myapp_CFLAGS = $(LAMMPS_CFLAGS) # myapp_LDADD = $(LAMMPS_LIBS) -# Use this in CMake: -# CMakeLists.txt: -# find_package(PkgConfig) -# pkg_check_modules(LAMMPS IMPORTED_TARGET lammps) -# target_link_libraries( PkgConfig::LAMMPS) - prefix=@CMAKE_INSTALL_PREFIX@ libdir=@CMAKE_INSTALL_FULL_LIBDIR@ includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ @@ -31,8 +25,8 @@ includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ Name: liblammps@LAMMPS_MACHINE@ Description: Large-scale Atomic/Molecular Massively Parallel Simulator Library URL: http://lammps.sandia.gov -Version: @LAMMPS_VERSION@ +Version: @PROJECT_VERSION@ Requires: -Libs: -L${libdir} -llammps@LAMMPS_LIB_SUFFIX@ +Libs: -L${libdir} -llammps@LAMMPS_MACHINE@ Libs.private: -lm Cflags: -I${includedir} @LAMMPS_API_DEFINES@ diff --git a/cmake/presets/intel.cmake b/cmake/presets/intel.cmake index 869bf7bca236fa0ad76a8bb7f25eed13a2a86a71..099d744ad34f887fe146aa377105162f591394f7 100644 --- a/cmake/presets/intel.cmake +++ b/cmake/presets/intel.cmake @@ -2,7 +2,6 @@ set(CMAKE_CXX_COMPILER "icpc" CACHE STRING "" FORCE) set(CMAKE_C_COMPILER "icc" CACHE STRING "" FORCE) -set(CMAKE_CXX_FLAGS "-O3 -DNDEBG" CACHE STRING "" FORCE) set(MPI_CXX "icpc" CACHE STRING "" FORCE) set(MPI_CXX_COMPILER "mpicxx" CACHE STRING "" FORCE) unset(HAVE_OMP_H_INCLUDE CACHE) diff --git a/cmake/presets/kokkos-cuda.cmake b/cmake/presets/kokkos-cuda.cmake new file mode 100644 index 0000000000000000000000000000000000000000..36d099479d4af80a02a7f845651aa7618ff690c0 --- /dev/null +++ b/cmake/presets/kokkos-cuda.cmake @@ -0,0 +1,11 @@ +# preset that enables KOKKOS and selects CUDA compilation with OpenMP +# enabled as well. This preselects CC 5.0 as default GPU arch, since +# that is compatible with all higher CC, but not the default CC 3.5 +set(PKG_KOKKOS ON CACHE BOOL "" FORCE) +set(Kokkos_ENABLE_SERIAL ON CACHE BOOL "" FORCE) +set(Kokkos_ENABLE_OPENMP ON CACHE BOOL "" FORCE) +set(Kokkos_ENABLE_CUDA ON CACHE BOOL "" FORCE) +set(Kokkos_ARCH_MAXWELL50 on CACHE BOOL "" FORCE) +set(BUILD_OMP ON CACHE BOOL "" FORCE) +get_filename_component(NVCC_WRAPPER_CMD ${CMAKE_CURRENT_SOURCE_DIR}/../lib/kokkos/bin/nvcc_wrapper ABSOLUTE) +set(CMAKE_CXX_COMPILER ${NVCC_WRAPPER_CMD} CACHE FILEPATH "" FORCE) diff --git a/cmake/presets/kokkos-openmp.cmake b/cmake/presets/kokkos-openmp.cmake new file mode 100644 index 0000000000000000000000000000000000000000..27d09f62cfc454fb031a1bac4c7fe48202bbf903 --- /dev/null +++ b/cmake/presets/kokkos-openmp.cmake @@ -0,0 +1,6 @@ +# preset that enables KOKKOS and selects OpenMP (only) compilation +set(PKG_KOKKOS ON CACHE BOOL "" FORCE) +set(Kokkos_ENABLE_SERIAL ON CACHE BOOL "" FORCE) +set(Kokkos_ENABLE_OPENMP ON CACHE BOOL "" FORCE) +set(Kokkos_ENABLE_CUDA OFF CACHE BOOL "" FORCE) +set(BUILD_OMP ON CACHE BOOL "" FORCE) diff --git a/cmake/presets/kokkos-serial.cmake b/cmake/presets/kokkos-serial.cmake new file mode 100644 index 0000000000000000000000000000000000000000..0208d2ee3a408f4e08313d33076ea27751266bbc --- /dev/null +++ b/cmake/presets/kokkos-serial.cmake @@ -0,0 +1,5 @@ +# preset that enables KOKKOS and selects serial compilation only +set(PKG_KOKKOS ON CACHE BOOL "" FORCE) +set(Kokkos_ENABLE_SERIAL ON CACHE BOOL "" FORCE) +set(Kokkos_ENABLE_OPENMP OFF CACHE BOOL "" FORCE) +set(Kokkos_ENABLE_CUDA OFF CACHE BOOL "" FORCE) diff --git a/cmake/presets/mingw-cross.cmake b/cmake/presets/mingw-cross.cmake index a733142455a37929df4a4fec985453f3da22cb42..61c4bbc45b428b191daf484e81d7e8dc63c708be 100644 --- a/cmake/presets/mingw-cross.cmake +++ b/cmake/presets/mingw-cross.cmake @@ -23,6 +23,7 @@ set(DOWNLOAD_VORO ON CACHE BOOL "" FORCE) set(DOWNLOAD_EIGEN3 ON CACHE BOOL "" FORCE) set(LAMMPS_MEMALIGN "0" CACHE STRING "" FORCE) set(CMAKE_TUNE_FLAGS "-Wno-missing-include-dirs" CACHE STRING "" FORCE) -set(CMAKE_EXE_LINKER_FLAGS "--enable-stdcall-fixup" CACHE STRING "" FORCE) +set(CMAKE_EXE_LINKER_FLAGS "-Wl,--enable-stdcall-fixup" CACHE STRING "" FORCE) +set(CMAKE_SHARED_LINKER_FLAGS "-Wl,--enable-stdcall-fixup" CACHE STRING "" FORCE) set(BUILD_TOOLS ON CACHE BOOL "" FORCE) set(CMAKE_INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lammps-installer") diff --git a/doc/Makefile b/doc/Makefile index ff56226e83efa176ee4e173b64b4ac8d22fad843..c1f299a2f7f19957e040d69648d2ac9d42f4719d 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -12,6 +12,7 @@ PYTHON = $(shell which python3) VIRTUALENV = virtualenv HAS_PYTHON3 = NO HAS_VIRTUALENV = NO +HAS_PDFLATEX = NO ifeq ($(shell which python3 >/dev/null 2>&1; echo $$?), 0) HAS_PYTHON3 = YES @@ -27,6 +28,11 @@ VIRTUALENV = virtualenv HAS_VIRTUALENV = YES endif +ifeq ($(shell which pdflatex >/dev/null 2>&1; echo $$?), 0) +HAS_PDFLATEX = YES +endif + + SPHINXEXTRA = -j $(shell $(PYTHON) -c 'import multiprocessing;print(multiprocessing.cpu_count())') .PHONY: help clean-all clean clean-spelling epub mobi rst html pdf spelling anchor_check style_check @@ -51,7 +57,7 @@ help: # ------------------------------------------ clean-all: clean - rm -rf $(BUILDDIR)/docenv $(BUILDDIR)/doctrees $(BUILDDIR)/mathjax + rm -rf $(BUILDDIR)/docenv $(BUILDDIR)/doctrees $(BUILDDIR)/mathjax Manual.pdf Developer.pdf clean: clean-spelling rm -rf html epub latex @@ -77,7 +83,7 @@ html: $(ANCHORCHECK) $(MATHJAX) @rm -rf html/JPG @cp -r src/PDF html/PDF @mkdir -p html/JPG - @cp `grep -A2 '\.\. .*image::' src/*.rst | grep ':target:' | sed -e 's,.*:target: JPG/,src/JPG/,' | sort | uniq` html/JPG/ + @cp `grep -A2 '\.\. .*\(image\|figure\)::' src/*.rst | grep ':target:' | sed -e 's,.*:target: JPG/,src/JPG/,' | sort | uniq` html/JPG/ @rm -rf html/PDF/.[sg]* @mkdir -p html/_static/mathjax @cp -r $(MATHJAX)/es5 html/_static/mathjax/ @@ -86,8 +92,7 @@ html: $(ANCHORCHECK) $(MATHJAX) spelling: $(VENV) utils/sphinx-config/false_positives.txt @(\ . $(VENV)/bin/activate ;\ - pip install sphinxcontrib-spelling ;\ - cp utils/sphinx-config/false_positives.txt $(RSTDIR)/ ;\ + cp utils/sphinx-config/false_positives.txt $(RSTDIR)/ ; env PYTHONWARNINGS= \ sphinx-build -b spelling -c utils/sphinx-config -d $(BUILDDIR)/doctrees $(RSTDIR) spelling ;\ deactivate ;\ ) @@ -113,6 +118,7 @@ mobi: epub @echo "Conversion finished. The MOBI manual file is created." pdf: $(ANCHORCHECK) + @if [ "$(HAS_PDFLATEX)" == "NO" ] ; then echo "PDFLaTeX was not found! Please check README.md for further instructions" 1>&2; exit 1; fi @(\ cd src/Developer; \ pdflatex developer; \ @@ -187,7 +193,10 @@ $(VENV): @( \ $(VIRTUALENV) -p $(PYTHON) $(VENV); \ . $(VENV)/bin/activate; \ + pip install --upgrade pip; \ pip install Sphinx; \ + pip install sphinxcontrib-spelling ;\ + pip install breathe; \ deactivate;\ ) diff --git a/doc/README b/doc/README index 1416584eed4abbf9334370e5e6e326e377f0ea6a..cbc0c425dc8c61894779e3b653697299ce7df8af 100644 --- a/doc/README +++ b/doc/README @@ -93,12 +93,18 @@ support for PDFLaTeX. Also the following LaTeX packages need to be installed (e.g. from texlive): - amsmath - babel +- capt-of - cmap - fncychap +- framed - geometry - hyperref - hypcap +- needspace - times +- tabulary +- upquote +- wrapfig ---------------- Installing prerequisites for epub build diff --git a/doc/src/.gitignore b/doc/src/.gitignore index e0b9693457ca0dfc081cfc7da4d53a6fa6286b94..afe24cc10c7128a38123ac9eb6095acc3e239b30 100644 --- a/doc/src/.gitignore +++ b/doc/src/.gitignore @@ -1,3 +1 @@ -/Eqs -/JPG /false_positives.txt diff --git a/doc/src/Build.rst b/doc/src/Build.rst index c96fb0b5b9cb72502c2c7ca2fa7e6b748b2f7b7f..441af79009ad86069ee866aec82e001056fab0e2 100644 --- a/doc/src/Build.rst +++ b/doc/src/Build.rst @@ -1,12 +1,14 @@ Build LAMMPS ************ -LAMMPS can be built as an executable or library from source code via -either traditional makefiles (which may require manual editing) -for use with GNU make or gmake, or a build environment generated by CMake -(Unix Makefiles, Xcode, Visual Studio, KDevelop or more). As an -alternative you can download a package with pre-built executables -as described on the :doc:`Install ` doc page. +LAMMPS is built as a library and an executable from source code using +either traditional makefiles for use with GNU make (which may require +manual editing), or using a build environment generated by CMake (Unix +Makefiles, Ninja, Xcode, Visual Studio, KDevelop, CodeBlocks and more). + +As an alternative you can download a package with pre-built executables +or automated build trees as described on the :doc:`Install ` +doc page. .. toctree:: :maxdepth: 1 diff --git a/doc/src/Build_basics.rst b/doc/src/Build_basics.rst index 8ad48b0810b87147bce2ea8ecc0965fa3ccc46c5..335c1a51899836d3f818aebdb2f89c696111bfde 100644 --- a/doc/src/Build_basics.rst +++ b/doc/src/Build_basics.rst @@ -6,7 +6,8 @@ CMake and make: * :ref:`Serial vs parallel build ` * :ref:`Choice of compiler and compile/link options ` -* :ref:`Build LAMMPS as an executable or a library ` +* :ref:`Build the LAMMPS executable and library ` +* :ref:`Including and removing debug support ` * :ref:`Build the LAMMPS documentation ` * :ref:`Install LAMMPS after a build ` @@ -15,7 +16,7 @@ CMake and make: .. _serial: Serial vs parallel build -------------------------------------- +------------------------ LAMMPS is written to use the ubiquitous `MPI (Message Passing Interface) `_ library API @@ -28,8 +29,8 @@ MPI STUBS library. Independent of the distributed memory MPI parallelization, parts of LAMMPS are also written with support for shared memory parallelization -using the OpenMP threading standard. A more detailed discussion of that -is below. +using the `OpenMP `_ threading +standard. A more detailed discussion of that is below. **CMake build**\ : @@ -41,7 +42,7 @@ is below. # no default value The executable created by CMake (after running make) is named ``lmp`` unless -the LAMMPS_MACHINE option is set. When setting ``LAMMPS_MACHINE=name`` +the ``LAMMPS_MACHINE`` option is set. When setting ``LAMMPS_MACHINE=name`` the executable will be called ``lmp_name``. Using ``BUILD_MPI=no`` will enforce building a serial executable using the MPI STUBS library. @@ -55,18 +56,20 @@ The build with traditional makefiles has to be done inside the source folder ``s make serial # serial build, produces lmp_serial using Makefile/serial make mybox # uses Makefile.mybox to produce lmp_mybox -Any "make machine" command will look up the make settings from a file -Makefile.machine, create a folder Obj_machine with all objects and -generated files and an executable called ``lmp_machine``\ . The standard -parallel build with ``make mpi`` assumes a standard MPI installation with -MPI compiler wrappers where all necessary compiler and linker flags to -get access and link with the suitable MPI headers and libraries are set -by the wrapper programs. For other cases or the serial build, you have -to adjust the make file variables ``MPI_INC``, ``MPI_PATH``, ``MPI_LIB`` -as well as ``CC`` and ``LINK``\ . To enable OpenMP threading usually -a compiler specific flag needs to be added to the compile and link -commands. For the GNU compilers, this is ``-fopenmp``\ , which can be -added to the ``CC`` and ``LINK`` makefile variables. +Any ``make machine`` command will look up the make settings from a file +``Makefile.machine`` in the folder ``src/MAKE`` or one of its +sub-directories ``MINE``, ``MACHINES``, or ``OPTIONS``, create a folder +``Obj_machine`` with all objects and generated files and an executable +called ``lmp_machine``\ . The standard parallel build with ``make mpi`` +assumes a standard MPI installation with MPI compiler wrappers where all +necessary compiler and linker flags to get access and link with the +suitable MPI headers and libraries are set by the wrapper programs. For +other cases or the serial build, you have to adjust the make file +variables ``MPI_INC``, ``MPI_PATH``, ``MPI_LIB`` as well as ``CC`` and +``LINK``\ . To enable OpenMP threading usually a compiler specific flag +needs to be added to the compile and link commands. For the GNU +compilers, this is ``-fopenmp``\ , which can be added to the ``CC`` and +``LINK`` makefile variables. For the serial build the following make variables are set (see src/MAKE/Makefile.serial): @@ -80,18 +83,19 @@ For the serial build the following make variables are set (see src/MAKE/Makefile You also need to build the STUBS library for your platform before making LAMMPS itself. A ``make serial`` build does this for you automatically, -otherwise, type ``make mpi-stubs`` from the src directory, or ``make`` from -the src/STUBS dir. If the build fails, you will need to edit the -STUBS/Makefile for your platform. The stubs library does not provide -MPI/IO functions required by some LAMMPS packages, e.g. MPIIO or USER-LB, -and thus is not compatible with those packages. +otherwise, type ``make mpi-stubs`` from the src directory, or ``make`` +from the ``src/STUBS`` dir. If the build fails, you may need to edit +the ``STUBS/Makefile`` for your platform. The stubs library does not +provide MPI/IO functions required by some LAMMPS packages, +e.g. ``MPIIO`` or ``USER-LB``, and thus is not compatible with those +packages. .. note:: The file ``src/STUBS/mpi.c`` provides a CPU timer function called - MPI_Wtime() that calls gettimeofday() . If your operating system - does not support gettimeofday() , you will need to insert code to - call another timer. Note that the ANSI-standard function clock() + ``MPI_Wtime()`` that calls ``gettimeofday()``. If your operating system + does not support ``gettimeofday()``, you will need to insert code to + call another timer. Note that the ANSI-standard function ``clock()`` rolls over after an hour or so, and is therefore insufficient for timing long LAMMPS simulations. @@ -111,20 +115,22 @@ self-installed MPICH or OpenMPI, so you should study the provided documentation to find out how to build and link with it. The majority of OpenMP (threading) support in LAMMPS is provided by the -USER-OMP package; see the :doc:`Speed omp ` doc page for -details. The USER-INTEL package also includes OpenMP threading (it is -compatible with USER-OMP) and adds vectorization support when compiled -with compatible compilers, in particular the Intel compilers on top of -OpenMP. Also, the KOKKOS package can be compiled to include OpenMP -threading. +``USER-OMP`` package; see the :doc:`Speed omp ` doc page for +details. The ``USER-INTEL`` package also includes OpenMP threading (it +is compatible with ``USER-OMP`` and will usually fall back on styles +from that package, if a ``USER-INTEL`` does not exist) and adds +vectorization support when compiled with compatible compilers, in +particular the Intel compilers on top of OpenMP. Also, the ``KOKKOS`` +package can be compiled to include OpenMP threading. In addition, there are a few commands in LAMMPS that have native OpenMP -support included as well. These are commands in the MPIIO, SNAP, -USER-DIFFRACTION, and USER-DPD packages. In addition some packages -support OpenMP threading indirectly through the libraries they interface -to: e.g. LATTE and USER-COLVARS. See the :doc:`Packages details -` doc page for more info on these packages and the doc -pages for their respective commands for OpenMP threading info. +support included as well. These are commands in the ``MPIIO``, +``SNAP``, ``USER-DIFFRACTION``, and ``USER-DPD`` packages. In addition +some packages support OpenMP threading indirectly through the libraries +they interface to: e.g. ``LATTE``, ``KSPACE``, and ``USER-COLVARS``. +See the :doc:`Packages details ` doc page for more +info on these packages and the doc pages for their respective commands +for OpenMP threading info. For CMake, if you use ``BUILD_OMP=yes``, you can use these packages and turn on their native OpenMP support and turn on their native OpenMP @@ -142,14 +148,15 @@ please refer to its documentation. **OpenMP Compiler compatibility info**\ : Some compilers do not fully support the ``default(none)`` directive -and others (e.g. GCC version 9 and beyond) may implement OpenMP 4.0 -semantics, which are incompatible with the OpenMP 3.1 semantics used -in LAMMPS (for maximal compatibility with compiler versions in use). -In those case, all ``default(none)`` directives (which aid in detecting -incorrect and unwanted sharing) can be replaced with ``default(shared)`` -while dropping all ``shared()`` directives. The script -'src/USER-OMP/hack_openmp_for_pgi_gcc9.sh' can be used to automate -this conversion. +and others (e.g. GCC version 9 and beyond, Clang version 10 and later) +may implement strict OpenMP 4.0 and later semantics, which are incompatible +with the OpenMP 3.1 semantics used in LAMMPS for maximal compatibility +with compiler versions in use. If compilation with OpenMP enabled fails +because of your compiler requiring strict OpenMP 4.0 semantic, you can +change the behavior by adding ``-D LAMMPS_OMP_COMPAT=4`` to the ``LMP_INC`` +variable in your makefile, or add it to the command line while configuring +with CMake. CMake will detect the suitable setting for the GNU, Clang, +and Intel compilers. ---------- @@ -158,15 +165,18 @@ this conversion. Choice of compiler and compile/link options --------------------------------------------------------- -The choice of compiler and compiler flags can be important for +The choice of compiler and compiler flags can be important for maximum performance. Vendor provided compilers for a specific hardware can produce faster code than open-source compilers like the GNU compilers. -On x86 hardware most popular compilers are quite similar in performance -of C/C++ code at high optimization levels. When using the USER-INTEL -package, there is a distinct advantage in using the `Intel C++ compiler -`_ due to much improved vectorization through SSE and AVX -instructions on compatible hardware as the source code includes changes -and compiler directives to enable high degrees of vectorization. +On the most common x86 hardware most popular C++ compilers are quite +similar in performance of C/C++ code at high optimization levels. When +using the ``USER-INTEL`` package, there is a distinct advantage in using +the `Intel C++ compiler `_ due to much improved vectorization +through SSE and AVX instructions on compatible hardware as the source +code includes changes and Intel compiler specific directives to enable +high degrees of vectorization. This may change over time as equivalent +vectorization directives are included into OpenMP standard revisions and +other compilers adopt them. .. _intel: https://software.intel.com/en-us/intel-compilers @@ -177,17 +187,20 @@ LAMMPS. **CMake build**\ : -By default CMake will use a compiler it finds and it will add -optimization flags appropriate to that compiler and any -:doc:`accelerator packages ` you have included in the -build. - -You can tell CMake to look for a specific compiler with these variable -settings. Likewise you can specify the corresponding ``CMAKE_*_FLAGS`` -variables if you want to experiment with alternate optimization flags. -You should specify all 3 compilers, so that the small number of LAMMPS -source files written in C or Fortran are built with a compiler consistent -with the one used for all the C++ files: +By default CMake will use a compiler it finds according to internal +preferences and it will add optimization flags appropriate to that +compiler and any :doc:`accelerator packages ` you have +included in the build. + +You can tell CMake to look for a specific compiler with setting CMake +variable during configuration. For a few common choices, there are also +presets in the ``cmake/presets`` folder. For convenience, there is a +``CMAKE_TUNE_FLAGS`` variable that can be set to apply global compiler +options. More on that below, but you can also specify the corresponding +``CMAKE_*_FLAGS`` variables individually if you want to experiment with +alternate optimization flags. You should specify all 3 compilers, so +that the (few) LAMMPS source files written in C or Fortran are built +with a compiler consistent with the one used for the C++ files: .. code-block:: bash @@ -210,16 +223,17 @@ A few example command lines are: # Building with LLVM/Clang Compilers: cmake ../cmake -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_Fortran_COMPILER=flang -For compiling with the Clang/LLVM compilers a special CMake preset is -included that can be loaded with `-C ../cmake/presets/clang.cmake`. +For compiling with the Clang/LLVM compilers a CMake preset is provided that +can be loaded with `-C ../cmake/presets/clang.cmake`. Similarly, +`-C ../cmake/presets/intel.cmake` should switch the -In addition you can set ``CMAKE_TUNE_FLAGS`` to specifically add compiler -flags to tune for optimal performance on given hosts. By default these are -initialized to some compiler specific flags, where known, to optimize the -LAMMPS executable with optimizations and instructions available on the host -where LAMMPS is compiled. For example, for Intel compilers this would be -``-xHost`` and for GNU compilers this would be ``-march=native``. To turn -these flags off, set ``-D CMAKE_TUNE_FLAGS=``. +In addition you can set ``CMAKE_TUNE_FLAGS`` to specifically add +compiler flags to tune for optimal performance on given hosts. By +default these are initialized to some compiler specific flags, to +optimize the LAMMPS executable with optimizations and instructions +available on the host where LAMMPS is compiled. For example, for Intel +compilers this would be ``-xHost`` and for GNU compilers this would be +``-march=native``. To turn these flags off, do ``-D CMAKE_TUNE_FLAGS=``. .. note:: @@ -235,11 +249,11 @@ these flags off, set ``-D CMAKE_TUNE_FLAGS=``. The "compiler/linker settings" section of a Makefile.machine lists compiler and linker settings for your C++ compiler, including optimization flags. For a parallel build it is recommended to use -mpicxx or mpiCC, since these compiler wrappers will include a variety of -settings appropriate for your MPI installation and thus avoiding the -guesswork of finding the right flags. +``mpicxx`` or ``mpiCC``, since these compiler wrappers will include a +variety of settings appropriate for your MPI installation and thus +avoiding the guesswork of finding the right flags. -Parallel build (see src/MAKE/Makefile.mpi): +Parallel build (see ``src/MAKE/Makefile.mpi``): .. code-block:: bash @@ -248,7 +262,7 @@ Parallel build (see src/MAKE/Makefile.mpi): LINK = mpicxx LINKFLAGS = -g -O -Serial build (see src/MAKE/Makefile.serial): +Serial build with GNU gcc (see ``src/MAKE/Makefile.serial``): .. code-block:: make @@ -257,17 +271,35 @@ Serial build (see src/MAKE/Makefile.serial): LINK = g++ LINKFLAGS = -g -O + .. note:: - If you build LAMMPS with any :doc:`accelerator packages ` - included, there may be specific optimization flags that are either - required or recommended to enable required features and to achieve - optimal performance. You need to include these in the CCFLAGS and - LINKFLAGS settings above. For details, see the individual package - doc pages listed on the :doc:`Speed packages ` doc - page. Or examine these files in the src/MAKE/OPTIONS directory. - They correspond to each of the 5 accelerator packages and their - hardware variants: + If compilation stops with a message like the following: + + .. code-block:: + + g++ -g -O3 -DLAMMPS_GZIP -DLAMMPS_MEMALIGN=64 -I../STUBS -c ../main.cpp + In file included from ../pointers.h:24:0, + from ../input.h:17, + from ../main.cpp:16: + ../lmptype.h:34:2: error: #error LAMMPS requires a C++11 (or later) compliant compiler. Enable C++11 compatibility or upgrade the compiler. + + then you have either an unsupported (old) compiler or you have to + turn on C++11 mode. The latter applies to GCC 4.8.x shipped with + RHEL 7.x and CentOS 7.x. For those compilers, you need to add the + ``-std=c++11`` flag. Otherwise, you would have to install a newer + compiler that supports C++11; either as a binary package or through + compiling from source. + +If you build LAMMPS with any :doc:`accelerator packages +` included, there may be specific optimization flags +that are either required or recommended to enable required features and +to achieve optimal performance. You need to include these in the +CCFLAGS and LINKFLAGS settings above. For details, see the individual +package doc pages listed on the :doc:`Speed packages ` +doc page. Or examine these files in the src/MAKE/OPTIONS directory. +They correspond to each of the 5 accelerator packages and their hardware +variants: .. code-block:: bash @@ -284,69 +316,74 @@ Serial build (see src/MAKE/Makefile.serial): .. _exe: -Build LAMMPS as an executable or a library ----------------------------------------------------- +Build the LAMMPS executable and library +--------------------------------------- -LAMMPS can be built as either an executable or as a static or shared -library. The LAMMPS library can be called from another application or -a scripting language. See the :doc:`Howto couple ` doc +LAMMPS is always built as a library of C++ classes plus an executable. +The executable is a simple ``main()`` function that sets up MPI and then +creates a LAMMPS class instance from the LAMMPS library, which +will then process commands provided via a file or from the console +input. The LAMMPS library can also be called from another application +or a scripting language. See the :doc:`Howto couple ` doc page for more info on coupling LAMMPS to other codes. See the :doc:`Python ` doc page for more info on wrapping and running LAMMPS from Python via its library interface. **CMake build**\ : -For CMake builds, you can select through setting CMake variables which -files the compilation produces during the configuration step. If none -are set, defaults are applied. +For CMake builds, you can select through setting CMake variables between +building a shared or a static LAMMPS library and what kind of suffix is +added to them (in case you want to concurrently install multiple variants +of binaries with different settings). If none are set, defaults are applied. .. code-block:: bash - -D BUILD_EXE=value # yes (default) or no - -D BUILD_LIB=value # yes or no (default) -D BUILD_SHARED_LIBS=value # yes or no (default) - -D LAMMPS_LIB_SUFFIX=name # name = mpi, serial, mybox, titan, laptop, etc + -D LAMMPS_MACHINE=name # name = mpi, serial, mybox, titan, laptop, etc # no default value -Setting ``BUILD_EXE=no`` will not produce an executable. Setting -``BUILD_LIB=yes`` will produce a static library named ``liblammps.a``\ . -Setting both ``BUILD_LIB=yes`` and ``BUILD_SHARED_LIBS=yes`` will produce a -shared library named ``liblammps.so`` instead. If ``LAMMPS_LIB_SUFFIX=name`` -is set in addition, the name of the generated libraries will be changed to -either ``liblammps_name.a`` or ``liblammps_name.so``\ , respectively. +The compilation will always produce a LAMMPS library and an executable +linked to it. By default this will be a static library named +``liblammps.a`` and an executable named ``lmp`` Setting +``BUILD_SHARED_LIBS=yes`` will instead produce a shared library called +``liblammps.so`` (or ``liblammps.dylib`` or ``liblammps.dll`` depending +on the platform) If ``LAMMPS_MACHINE=name`` is set in addition, the name +of the generated libraries will be changed to either +``liblammps_name.a`` or ``liblammps_name.so``\ , respectively and the +executable will be called ``lmp_name``. **Traditional make**\ : With the traditional makefile based build process, the choice of the generated executable or library depends on the "mode" setting. -Several options are available and ``mode=exe`` is the default. +Several options are available and ``mode=static`` is the default. .. code-block:: bash make machine # build LAMMPS executable lmp_machine - make mode=exe machine # same as "make machine" - make mode=lib machine # build LAMMPS static lib liblammps_machine.a - make mode=shlib machine # build LAMMPS shared lib liblammps_machine.so - make mode=shexe machine # same as "mode=exe" but uses objects from "mode=shlib" - -The two "exe" builds will generate and executable ``lmp_machine``\ , -while the two library builds will create a file ``liblammps_machine.a`` -or ``liblammps_machine.so``\ . They will also create generic soft links, -named ``liblammps.a`` and ``liblammps.so``\ , which point to the specific + make mode=static machine # same as "make machine" + make mode=shared machine # build LAMMPS shared lib liblammps_machine.so instead + +The "static" build will generate a static library called +``liblammps_machine.a`` and an executable named ``lmp_machine``\ , while +the "shared" build will generate a shared library +``liblammps_machine.so`` instead and ``lmp_machine`` will be linked to +it. The build step will also create generic soft links, named +``liblammps.a`` and ``liblammps.so``\ , which point to the specific ``liblammps_machine.a/so`` files. **CMake and make info**\ : -Note that for a shared library to be usable by a calling program, all -the auxiliary libraries it depends on must also exist as shared -libraries. This will be the case for libraries included with LAMMPS, -such as the dummy MPI library in src/STUBS or any package libraries in -the lib/packages directory, since they are always built in a shared -library compatible way using the ``-fPIC`` switch. However, if a library -like MPI or FFTW does not exist as a shared library, the shared library -build may generate an error. This means you will need to install a -shared library version of the auxiliary library. The build instructions -for the library should tell you how to do this. +Note that for creating a shared library, all the libraries it depends on +must be compiled to be compatible with shared libraries. This should be +the case for libraries included with LAMMPS, such as the dummy MPI +library in ``src/STUBS`` or any package libraries in the ``lib`` +directory, since they are always built in a shared library compatible +way using the ``-fPIC`` compiler switch. However, if an auxiliary +library (like MPI or FFTW) does not exist as a compatible format, the +shared library linking step may generate an error. This means you will +need to install a compatible version of the auxiliary library. The +build instructions for that library should tell you how to do this. As an example, here is how to build and install the `MPICH library `_, a popular open-source version of MPI, as a shared library @@ -360,10 +397,12 @@ in the default /usr/local/lib location: make make install -You may need to use ``sudo make install`` in place of the last line if you -do not have write privileges for ``/usr/local/lib``. The end result should -be the file ``/usr/local/lib/libmpich.so``. On many Linux installations the -folder ``${HOME}/.local`` is an alternative to using ``/usr/local`` and does +You may need to use ``sudo make install`` in place of the last line if +you do not have write privileges for ``/usr/local/lib`` or use the +``--prefix`` configuration option to select an installation folder, +where you do have write access. The end result should be the file +``/usr/local/lib/libmpich.so``. On many Linux installations the folder +``${HOME}/.local`` is an alternative to using ``/usr/local`` and does not require superuser or sudo access. In that case the configuration step becomes: @@ -371,12 +410,41 @@ step becomes: ./configure --enable-shared --prefix=${HOME}/.local -Avoiding using "sudo" for custom software installation (i.e. from source +Avoiding to use "sudo" for custom software installation (i.e. from source and not through a package manager tool provided by the OS) is generally recommended to ensure the integrity of the system software installation. ---------- +.. _debug: + +Excluding or removing debug support +----------------------------------- + +By default the compilation settings will include the *-g* flag which +instructs the compiler to include debug information (e.g. which line of +source code a particular instruction correspond to). This can be +extremely useful in case LAMMPS crashes and can help to provide crucial +information in :doc:`tracking down the origin of a crash ` +and help the LAMMPS developers fix bugs in the source code. However, +this increases the storage requirements for object files, libraries, and +the executable 3-5 fold. + +If this is a concern, you can change the compilation settings or remove +the debug information from the LAMMPS executable: + +- **Traditional make**: edit your ``Makefile.`` to remove the + *-g* flag from the ``CCFLAGS`` and ``LINKFLAGS`` definitions +- **CMake**: use ``-D CMAKE_BUILD_TYPE=Release`` or explicitly reset + the applicable compiler flags (best done using the text mode or + graphical user interface). +- **Remove debug info**: If you are only concerned about the executable + being too large, you can use the ``strip`` tool (e.g. ``strip + lmp_serial``) to remove the debug information from the executable file. + Do not strip libraries or object files, as that will render them unusable. + +---------- + .. _doc: Build the LAMMPS documentation @@ -427,7 +495,8 @@ It is also possible to create the HTML version of the manual within the :doc:`CMake build directory `. The reason for this option is to include the installation of the HTML manual pages into the "install" step when installing LAMMPS after the CMake build via -``make install``. +``make install``. The documentation build is included in the default +build target, but can also be requested independently with ``make doc``. .. code-block:: bash diff --git a/doc/src/Build_cmake.rst b/doc/src/Build_cmake.rst index 7a60ffa43754e3cdef7a79180f282cd51e3d5f69..3473d8314353140566345ad1de107d0a67e9c288 100644 --- a/doc/src/Build_cmake.rst +++ b/doc/src/Build_cmake.rst @@ -1,205 +1,158 @@ Build LAMMPS with CMake ======================= -This page is a short summary of how to use CMake to build LAMMPS. -Details on CMake variables that enable specific LAMMPS build options -are given on the pages linked to from the :doc:`Build ` doc page. - -Richard Berger (Temple U) has also written a `more comprehensive guide `_ -for how to use CMake to build LAMMPS. If you are new to CMake it is a -good place to start. - ----------- - -Building LAMMPS with CMake is a two-step process. First you use CMake -to create a build environment in a new directory. On Linux systems, -this will be by default based on Unix-style makefiles for use with make. -Then you use the *make* command to build LAMMPS, which uses the created -Makefile(s). Example: - -.. code-block:: bash - - cd lammps # change to the LAMMPS distribution directory - mkdir build; cd build # create a new directory (folder) for build - cmake [options ...] ../cmake # configuration with (command-line) cmake - cmake --build . # compilation (or type "make") - -The ``cmake`` command will detect available features, enable selected -packages and options, and will generate the build environment. By default -this build environment will be created for "Unix Makefiles" on most -platforms and particularly on Linux. However, alternate build tools -(e.g. Ninja) and project files for Integrated Development Environments -(IDEs) like Eclipse, CodeBlocks, or Kate can be generated, too. This is -selected via the ``-G`` command line flag. Further details about features -and settings for CMake are in the `CMake online documentation `_ - -.. _cmake_doc: https://cmake.org/documentation/ - -For the rest of the documentation -we will assume that the build environment is generated for "Unix Makefiles" -and thus the ``make`` command will be used to compile and link LAMMPS as -indicated above, producing (by default) an executable called ``lmp`` and -a library called ``liblammps.a`` in the ``build`` folder. - -If your machine has multiple CPU cores (most do these days), you can -compile sources in parallel with a command like ``make -j N`` (with N -being the maximum number of concurrently executed tasks). Also -installation of the ``ccache`` (= Compiler Cache) software may speed -up repeated compilation, e.g. during code development, significantly. - -After compilation, you may optionally install the LAMMPS executable into -your system with: - -.. code-block:: bash - - make install # optional, copy LAMMPS executable & library elsewhere - -This will install the lammps executable and library (if requested), some -tools (if configured) and additional files like library API headers, -manpages, potential and force field files. The location of the installation -tree is set by the CMake variable "CMAKE_INSTALL_PREFIX" which defaults -to ${HOME}/.local - ----------- - -.. _cmake_build: - -There are 3 variants of the CMake command itself: a command-line version -(``cmake`` or ``cmake3``), a text mode UI version (``ccmake`` or ``ccmake3``), -and a graphical GUI version (``cmake-gui``). You can use any of them -interchangeably to configure and create the LAMMPS build environment. -On Linux all the versions produce a Makefile as their output by default. -See more details on each below. - -You can specify a variety of options with any of the 3 versions, which -affect how the build is performed and what is included in the LAMMPS -executable. Links to pages explaining all the options are listed on -the :doc:`Build ` doc page. - -You must perform the CMake build system generation and compilation in -a new directory you create. It can be anywhere on your local machine. -In these Build pages we assume that you are building in a directory -called ``lammps/build``. You can perform separate builds independently -with different options, so long as you perform each of them in a -separate directory you create. All the auxiliary files created by one -build process (executable, object files, log files, etc) are stored in -this directory or sub-directories within it that CMake creates. +This page describes how to use `CMake `_ in general +to build LAMMPS. Details for specific compile time settings and options +to enable and configure add-on packages are discussed with those +packages. Links to those pages on the :doc:`Build overview ` +page. + +The following text assumes some familiarity with CMake and focuses on +using the command line tool ``cmake`` and what settings are supported +for building LAMMPS. A more detailed tutorial on how to use ``cmake`` +itself, the text mode or graphical user interface, change the generated +output files for different build tools and development environments is +on a :doc:`separate page `. .. note:: - To perform a CMake build, no packages can be installed or a build - been previously attempted in the LAMMPS src directory by using ``make`` - commands to :doc:`perform a conventional LAMMPS build `. - CMake detects if this is the case and generates an error, telling you - to type ``make no-all purge`` in the src directory to un-install all - packages. The purge removes all the \*.h files auto-generated by - make. - -You must have CMake version 3.10 or later on your system to build -LAMMPS. Installation instructions for CMake are below. + LAMMPS currently requires that CMake version 3.10 or later is available; + version 3.12 or later is preferred. + +.. warning:: + + You must not mix the :doc:`traditional make based ` + LAMMPS build procedure with using CMake. Thus no packages may be + installed or a build been previously attempted in the LAMMPS source + directory by using ``make ``. CMake will detect if this is + the case and generate an error. To remove conflicting files from the + ``src`` you can use the command ``make no-all purge`` which will + un-install all packages and delete all auto-generated files. + + +Advantages of using CMake +^^^^^^^^^^^^^^^^^^^^^^^^^ + +CMake is an alternative to compiling LAMMPS in the traditional way +through :doc:`(manually customized) makefiles ` and a recent +addition to LAMMPS thanks to the efforts of Christoph Junghans (LANL) +and Richard Berger (Temple U). Using CMake has multiple advantages that +are specifically helpful for people with limited experience in compiling +software or for people that want to modify or extend LAMMPS. + +- CMake can detect available hardware, tools, features, and libraries + and adapt the LAMMPS default build configuration accordingly. +- CMake can generate files for different build tools and integrated + development environments (IDE). +- CMake supports customization of settings with a text mode or graphical + user interface. No knowledge of file formats or and complex command + line syntax required. +- All enabled components are compiled in a single build operation. +- Automated dependency tracking for all files and configuration options. +- Support for true out-of-source compilation. Multiple configurations + and settings with different choices of LAMMPS packages, settings, or + compilers can be configured and built concurrently from the same + source tree. +- Simplified packaging of LAMMPS for Linux distributions, environment + modules, or automated build tools like `Homebrew `_. +- Integration of automated regression testing (the LAMMPS side for that + is still under development). -After the initial build, if you edit LAMMPS source files, or add your -own new files to the source directory, you can just re-type make from -your build directory and it will re-compile only the files that have -changed. If you want to change CMake options you can run cmake (or -ccmake or cmake-gui) again from the same build directory and alter -various options; see details below. Or you can remove the entire build -folder, recreate the directory and start over. +.. _cmake_build: ----------- +Getting started +^^^^^^^^^^^^^^^ -**Command-line version of CMake**\ : +Building LAMMPS with CMake is a two-step process. First you use CMake +to generate a build environment in a new directory. For that purpose +you can use either the command-line utility ``cmake`` (or ``cmake3``), +the text-mode UI utility ``ccmake`` (or ``ccmake3``) or the graphical +utility ``cmake-gui``, or use them interchangeably. The second step is +then the compilation and linking of all objects, libraries, and +executables. Here is a minimal example using the command line version of +CMake to build LAMMPS with no add-on packages enabled and no +customization: .. code-block:: bash - cmake [options ...] /path/to/lammps/cmake # build from any dir - cmake [options ...] ../cmake # build from lammps/build - cmake3 [options ...] ../cmake # build from lammps/build - -The cmake command takes one required argument, which is the LAMMPS -cmake directory which contains the CMakeLists.txt file. + cd lammps # change to the LAMMPS distribution directory + mkdir build; cd build # create and use a build directory + cmake ../cmake # configuration reading CMake scripts from ../cmake + cmake --build . # compilation (or type "make") + +This will create and change into a folder called ``build``, then run the +configuration step to generate build files for the default build command +and then launch that build command to compile LAMMPS. During the +configuration step CMake will try to detect whether support for MPI, +OpenMP, FFTW, gzip, JPEG, PNG, and ffmpeg are available and enable the +corresponding configuration settings. The progress of this +configuration can be followed on the screen and a summary of selected +options and settings will be printed at the end. The ``cmake --build +.`` command will launch the compilation, which, if successful, will +ultimately produce a library ``liblammps.a`` and the LAMMPS executable +``lmp`` inside the ``build`` folder. + +Compilation can take a long time, since LAMMPS is a large project with +many features. If your machine has multiple CPU cores (most do these +days), you can speed this up by compiling sources in parallel with +``make -j N`` (with N being the maximum number of concurrently executed +tasks). Also installation of the `ccache `_ (= +Compiler Cache) software may speed up repeated compilation even more, +e.g. during code development. + +After the initial build, whenever you edit LAMMPS source files, enable +or disable packages, change compiler flags or build options, you must +re-compile and relink the LAMMPS executable with ``cmake --build .`` (or +``make``). If the compilation fails for some reason, try running +``cmake .`` and then compile again. The included dependency tracking +should make certain that only the necessary subset of files are +re-compiled. You can also delete compiled objects, libraries and +executables with ``cmake --build . clean`` (or ``make clean``). -The argument can be prefixed or followed by various CMake -command-line options. Several useful ones are: +After compilation, you may optionally install the LAMMPS executable into +your system with: .. code-block:: bash - -D CMAKE_INSTALL_PREFIX=path # where to install LAMMPS executable/lib if desired - -D CMAKE_BUILD_TYPE=type # type = RelWithDebInfo (default), Release, MinSizeRel, or Debug - -G output # style of output CMake generates (e.g. "Unix Makefiles" or "Ninja") - -D CMAKE_MAKE_PROGRAM=builder # name of the builder executable (e.g. when using "gmake" instead of "make") - -DVARIABLE=value # setting for a LAMMPS feature to enable - -D VARIABLE=value # ditto, but cannot come after CMakeLists.txt dir - -C path/to/preset/file # load some CMake settings before configuring - -All the LAMMPS-specific -D variables that a LAMMPS build supports are -described on the pages linked to from the :doc:`Build ` doc page. -All of these variable names are upper-case and their values are -lower-case, e.g. -D LAMMPS_SIZES=smallbig. For boolean values, any of -these forms can be used: yes/no, on/off, 1/0. - -On Unix/Linux machines, CMake generates a Makefile by default to -perform the LAMMPS build. Alternate forms of build info can be -generated via the -G switch, e.g. Visual Studio on a Windows machine, -Xcode on MacOS, or KDevelop on Linux. Type ``cmake --help`` to see the -"Generator" styles of output your system supports. - -.. note:: + make install # optional, copy compiled files into installation location - When CMake runs, it prints configuration info to the screen. - You should review this to verify all the features you requested were - enabled, including packages. You can also see what compilers and - compile options will be used for the build. Any errors in CMake - variable syntax will also be flagged, e.g. mis-typed variable names or - variable values. +This will install the LAMMPS executable and library, some tools (if +configured) and additional files like LAMMPS API headers, manpages, +potential and force field files. The location of the installation tree +defaults to ``${HOME}/.local``. -CMake creates a CMakeCache.txt file when it runs. This stores all the -settings, so that when running CMake again you can use the current -folder '.' instead of the path to the LAMMPS cmake folder as the -required argument to the CMake command. Either way the existing -settings will be inherited unless the CMakeCache.txt file is removed. +.. _cmake_options: -If you later want to change a setting you can rerun cmake in the build -directory with different setting. Please note that some automatically -detected variables will not change their value when you rerun cmake. -In these cases it is usually better to first remove all the -files/directories in the build directory, or start with a fresh build -directory. +Configuration and build options +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ----------- +The CMake commands have one mandatory argument: a folder containing a +file called ``CMakeLists.txt`` (for LAMMPS it is located in the +``cmake`` folder) or a build folder containing a file called +``CMakeCache.txt``, which is generated at the end of the CMake +configuration step. The cache file contains all current CMake settings. -**Curses version (terminal-style menu) of CMake**\ : - -.. code-block:: bash - - ccmake ../cmake - -You initiate the configuration and build environment generation steps -separately. For the first you have to type **c**\ , for the second you -have to type **g**\ . You may need to type **c** multiple times, and may be -required to edit some of the entries of CMake configuration variables -in between. Please see the `ccmake manual `_ for -more information. - ----------- - -**GUI version of CMake**\ : - -.. code-block:: bash +To modify settings, enable or disable features, you need to set *variables* +with either the *-D* command line flag (``-D VARIABLE1_NAME=value``) or +change them in the text mode of graphical user interface. The *-D* flag +can be used several times in one command. - cmake-gui ../cmake +For your convenience we provide :ref:`CMake presets ` +that combine multiple settings to enable optional LAMMPS packages or use +a different compiler tool chain. Those are loaded with the *-C* flag +(``-C ../cmake/presets/minimal.cmake``). This step would only be needed +once, as the settings from the preset files are stored in the +``CMakeCache.txt`` file. It is also possible to customize the build +by adding one or more *-D* flags to the CMake command line. -You initiate the configuration and build environment generation steps -separately. For the first you have to click on the **Configure** button, -for the second you have to click on the **Generate** button. You may -need to click on **Configure** multiple times, and may be required to -edit some of the entries of CMake configuration variables in between. -Please see the `cmake-gui manual `_ -for more information. +Generating files for alternate build tools (e.g. Ninja) and project files +for IDEs like Eclipse, CodeBlocks, or Kate can be selected using the *-G* +command line flag. A list of available generator settings for your +specific CMake version is given when running ``cmake --help``. ----------- -**Installing CMake** +Installing CMake +^^^^^^^^^^^^^^^^ Check if your machine already has CMake installed: @@ -216,11 +169,12 @@ software packages, do this: module list # is a module for cmake already loaded? module avail # is a module for cmake available? - module load cmake3 # load cmake module with appropriate name - -Most Linux distributions offer pre-compiled cmake packages through -their package management system. If you do not have CMake or a new -enough version, you can download the latest version at -`https://cmake.org/download/ `_. -Instructions on how to install it on various platforms can be found -`on this page `_. + module load cmake # load cmake module with appropriate name + +Most Linux distributions offer pre-compiled cmake packages through their +package management system. If you do not have CMake or a recent enough +version (Note: for CentOS 7.x you need to enable the EPEL repository), +you can download the latest version from `https://cmake.org/download/ +`_. Instructions on how to install it on +various platforms can be found `on this page +`_. diff --git a/doc/src/Build_development.rst b/doc/src/Build_development.rst index 21e99d69be16bfd9efd82a597b9d73de91e7706b..474e1fb833f2cec13861a5d06e5937386ddd2dac 100644 --- a/doc/src/Build_development.rst +++ b/doc/src/Build_development.rst @@ -1,15 +1,15 @@ Development build options (CMake only) ====================================== -The CMake build of LAMMPS has a few extra options which are useful during -development, testing or debugging. +The CMake build procedure of LAMMPS offers a few extra options which are +useful during development, testing or debugging. ---------- .. _compilation: Verify compilation flags ------------------------------------------- +------------------------ Sometimes it is necessary to verify the complete sequence of compilation flags generated by the CMake build. To enable a more verbose output during @@ -30,7 +30,7 @@ Another way of doing this without reconfiguration is calling make with variable .. _sanitizer: Address, Undefined Behavior, and Thread Sanitizer Support -------------------------------------------------------------------------- +--------------------------------------------------------- Compilers such as GCC and Clang support generating instrumented binaries which use different sanitizer libraries to detect problems in code @@ -41,10 +41,11 @@ during run-time. They can detect issues like: - `data races `_ Please note that this kind of instrumentation usually comes with a small -performance hit (much less than using tools like `Valgrind `_). -The to enable these features additional compiler flags need to be added -to the compilation and linking stages. This is most easily done through -setting the ``CMAKE_TUNE_FLAGS`` variable during configuration. Examples: +performance hit (much less than using tools like `Valgrind +`_). The to enable these features additional +compiler flags need to be added to the compilation and linking stages. +This is most easily done through setting the ``CMAKE_TUNE_FLAGS`` +variable during configuration. Examples: .. code-block:: bash @@ -52,8 +53,6 @@ setting the ``CMAKE_TUNE_FLAGS`` variable during configuration. Examples: -D CMAKE_TUNE_FLAGS=-fsanitize=undefined # enable undefined behavior sanitizer -D CMAKE_TUNE_FLAGS=-fsanitize=thread # enable thread sanitizer -.. _valgrind: https://valgrind.org - ---------- .. _testing: @@ -75,24 +74,26 @@ developers can run the tests directly on their workstation. -D LAMMPS_TESTING_SOURCE_DIR=path # path to lammps-testing repository (option if in custom location) -D LAMMPS_TESTING_GIT_TAG=value # version of lammps-testing repository that should be used, value = master (default) or custom git commit or tag -If you enable testing in the CMake build it will create an additional target called "test". You can run them with: +If you enable testing in the CMake build it will create an additional +target called "test". You can run them with: .. code-block:: bash - make test + cmake --build . test The test cases used come from the lammps-testing repository. They are -derivatives of the examples folder with some modifications to make the run -faster. +derivatives of the examples folder with some modifications to make the +run faster. -You can also collect code coverage metrics while running the tests by enabling -coverage support during building. +You can also collect code coverage metrics while running the tests by +enabling coverage support during building. .. code-block:: bash -D ENABLE_COVERAGE=value # enable coverage measurements, value = no (default) or yes -This will also add the following targets to generate coverage reports after running the LAMMPS executable: +This will also add the following targets to generate coverage reports +after running the LAMMPS executable: .. code-block:: bash @@ -100,7 +101,8 @@ This will also add the following targets to generate coverage reports after runn make gen_coverage_html # generate coverage report in HTML format make gen_coverage_xml # generate coverage report in XML format -These reports require GCOVR to be installed. The easiest way to do this to install it via pip: +These reports require GCOVR to be installed. The easiest way to do this +to install it via pip: .. code-block:: bash diff --git a/doc/src/Build_extras.rst b/doc/src/Build_extras.rst index 4589015e35bf2f68d486664843d6396e44104146..5457494c877024d0124c2378fd64c1e527026d1f 100644 --- a/doc/src/Build_extras.rst +++ b/doc/src/Build_extras.rst @@ -87,27 +87,33 @@ GPU package --------------------- To build with this package, you must choose options for precision and -which GPU hardware to build for. +which GPU hardware to build for. The GPU package currently supports +three different types of backends: OpenCL, CUDA and HIP. **CMake build**\ : .. code-block:: bash - -D GPU_API=value # value = opencl (default) or cuda - -D GPU_PREC=value # precision setting - # value = double or mixed (default) or single - -D OCL_TUNE=value # hardware choice for GPU_API=opencl - # generic (default) or intel (Intel CPU) or fermi, kepler, cypress (NVIDIA) - -D GPU_ARCH=value # primary GPU hardware choice for GPU_API=cuda - # value = sm_XX, see below - # default is sm_30 - -D CUDPP_OPT=value # optimization setting for GPU_API=cuda - # enables CUDA Performance Primitives Optimizations - # value = yes (default) or no - -D CUDA_MPS_SUPPORT=value # enables some tweaks required to run with active nvidia-cuda-mps daemon - # value = yes or no (default) - -GPU_ARCH settings for different GPU hardware is as follows: + -D GPU_API=value # value = opencl (default) or cuda or hip + -D GPU_PREC=value # precision setting + # value = double or mixed (default) or single + -D OCL_TUNE=value # hardware choice for GPU_API=opencl + # generic (default) or intel (Intel CPU) or fermi, kepler, cypress (NVIDIA) + -D GPU_ARCH=value # primary GPU hardware choice for GPU_API=cuda + # value = sm_XX, see below + # default is sm_30 + -D HIP_ARCH=value # primary GPU hardware choice for GPU_API=hip + # value depends on selected HIP_PLATFORM + # default is 'gfx906' for HIP_PLATFORM=hcc and 'sm_30' for HIP_PLATFORM=nvcc + -D HIP_USE_DEVICE_SORT=value # enables GPU sorting + # value = yes (default) or no + -D CUDPP_OPT=value # optimization setting for GPU_API=cuda + # enables CUDA Performance Primitives Optimizations + # value = yes (default) or no + -D CUDA_MPS_SUPPORT=value # enables some tweaks required to run with active nvidia-cuda-mps daemon + # value = yes or no (default) + +:code:`GPU_ARCH` settings for different GPU hardware is as follows: * sm_12 or sm_13 for GT200 (supported by CUDA 3.2 until CUDA 6.5) * sm_20 or sm_21 for Fermi (supported by CUDA 3.2 until CUDA 7.5) @@ -126,6 +132,28 @@ Thus the GPU_ARCH setting is merely an optimization, to have code for the preferred GPU architecture directly included rather than having to wait for the JIT compiler of the CUDA driver to translate it. +If you are compiling with HIP, note that before running CMake you will have to +set appropriate environment variables. Some variables such as +:code:`HCC_AMDGPU_TARGET` or :code:`CUDA_PATH` are necessary for :code:`hipcc` +and the linker to work correctly. + +.. code:: bash + + # AMDGPU target + export HIP_PLATFORM=hcc + export HCC_AMDGPU_TARGET=gfx906 + cmake -D PKG_GPU=on -D GPU_API=HIP -D HIP_ARCH=gfx906 -D CMAKE_CXX_COMPILER=hipcc .. + make -j 4 + +.. code:: bash + + # CUDA target + # !!! DO NOT set CMAKE_CXX_COMPILER !!! + export HIP_PLATFORM=nvcc + export CUDA_PATH=/usr/local/cuda + cmake -D PKG_GPU=on -D GPU_API=HIP -D HIP_ARCH=sm_70 .. + make -j 4 + **Traditional make**\ : Before building LAMMPS, you must build the GPU library in ``lib/gpu``\ . @@ -189,13 +217,27 @@ KIM package --------------------- To build with this package, the KIM library with API v2 must be downloaded -and built on your system. It must include the KIM models that you want to -use with LAMMPS. If you want to use the :doc:`kim_query ` +and built on your system. It must include the KIM models that you want to +use with LAMMPS. + +If you would like to use the :doc:`kim_query ` command, you also need to have libcurl installed with the matching development headers and the curl-config tool. -See the `Obtaining KIM Models `_ -web page to +If you would like to use the :doc:`kim_property ` +command, you need to build LAMMPS with the Python 3.6 or later package +installed. See the :doc:`Python ` doc page for more info on building +LAMMPS with the version of Python on your system. +After successfully building LAMMPS with Python, you need to +install the kim-property Python package, which can be easily done using +*pip* as ``pip install kim-property``, or from the *conda-forge* channel as +``conda install kim-property`` if LAMMPS is built in Conda. More detailed +information is available at: +`kim-property installation `_. + +In addition to installing the KIM API, it is also necessary to install the +library of KIM models (interatomic potentials). +See `Obtaining KIM Models `_ to learn how to install a pre-build binary of the OpenKIM Repository of Models. See the list of all KIM models here: https://openkim.org/browse/models @@ -265,74 +307,131 @@ using. For example: .. _kokkos: KOKKOS package ---------------------------- +-------------- -To build with this package, you must choose which hardware you want to -build for, either CPUs (multi-threading via OpenMP) or KNLs (OpenMP) -or GPUs (NVIDIA Cuda). - -For a CMake or make build, these are the possible choices for the -``KOKKOS_ARCH`` settings described below. Note that for CMake, these are -really Kokkos variables, not LAMMPS variables. Hence you must use -case-sensitive values, e.g. BDW, not bdw. - -* AMDAVX = AMD 64-bit x86 CPUs -* EPYC = AMD EPYC Zen class CPUs -* ARMv80 = ARMv8.0 Compatible CPU -* ARMv81 = ARMv8.1 Compatible CPU -* ARMv8-ThunderX = ARMv8 Cavium ThunderX CPU -* ARMv8-TX2 = ARMv8 Cavium ThunderX2 CPU -* WSM = Intel Westmere CPUs -* SNB = Intel Sandy/Ivy Bridge CPUs -* HSW = Intel Haswell CPUs -* BDW = Intel Broadwell Xeon E-class CPUs -* SKX = Intel Sky Lake Xeon E-class HPC CPUs (AVX512) -* KNC = Intel Knights Corner Xeon Phi -* KNL = Intel Knights Landing Xeon Phi -* BGQ = IBM Blue Gene/Q CPUs -* Power7 = IBM POWER8 CPUs -* Power8 = IBM POWER8 CPUs -* Power9 = IBM POWER9 CPUs -* Kepler = NVIDIA Kepler default (generation CC 3.5) -* Kepler30 = NVIDIA Kepler generation CC 3.0 -* Kepler32 = NVIDIA Kepler generation CC 3.2 -* Kepler35 = NVIDIA Kepler generation CC 3.5 -* Kepler37 = NVIDIA Kepler generation CC 3.7 -* Maxwell = NVIDIA Maxwell default (generation CC 5.0) -* Maxwell50 = NVIDIA Maxwell generation CC 5.0 -* Maxwell52 = NVIDIA Maxwell generation CC 5.2 -* Maxwell53 = NVIDIA Maxwell generation CC 5.3 -* Pascal60 = NVIDIA Pascal generation CC 6.0 -* Pascal61 = NVIDIA Pascal generation CC 6.1 -* Volta70 = NVIDIA Volta generation CC 7.0 -* Volta72 = NVIDIA Volta generation CC 7.2 -* Turing75 = NVIDIA Turing generation CC 7.5 +Using the KOKKOS package requires choosing several settings. You have +to select whether you want to compile with parallelization on the host +and whether you want to include offloading of calculations to a device +(e.g. a GPU). The default setting is to have no host parallelization +and no device offloading. In addition, you can select the hardware +architecture to select the instruction set. Since most hardware is +backward compatible, you may choose settings for an older architecture +to have an executable that will run on this and newer architectures. -**CMake build**\ : +.. note:: + NVIDIA GPUs with CC 5.0 (Maxwell) and newer are not compatible with + CC 3.x (Kepler). If you run Kokkos on a newer architecture than what + LAMMPS was compiled with, there will be a significant delay during + device initialization since the just-in-time compiler has to + recompile the GPU kernel code for the new hardware. + +The settings discussed below have been tested with LAMMPS and are +confirmed to work. Kokkos is an active project with ongoing improvements +and projects working on including support for additional architectures. +More information on Kokkos can be found on the +`Kokkos GitHub project `_. + +Available Architecture settings +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +These are the possible choices for the Kokkos architecture ID. They must +be specified in uppercase. + +.. list-table:: + :header-rows: 0 + :widths: auto + + * - **Arch-ID** + - **Description** + * - AMDAVX + - AMD 64-bit x86 CPU (AVX 1) + * - EPYC + - AMD EPYC Zen class CPU (AVX 2) + * - ARMV80 + - ARMv8.0 Compatible CPU + * - ARMV81 + - ARMv8.1 Compatible CPU + * - ARMV8_THUNDERX + - ARMv8 Cavium ThunderX CPU + * - ARMV8_THUNDERX2 + - ARMv8 Cavium ThunderX2 CPU + * - WSM + - Intel Westmere CPU (SSE 4.2) + * - SNB + - Intel Sandy/Ivy Bridge CPU (AVX 1) + * - HSW + - Intel Haswell CPU (AVX 2) + * - BDW + - Intel Broadwell Xeon E-class CPU (AVX 2 + transactional mem) + * - SKX + - Intel Sky Lake Xeon E-class HPC CPU (AVX512 + transactional mem) + * - KNC + - Intel Knights Corner Xeon Phi + * - KNL + - Intel Knights Landing Xeon Phi + * - BGQ + - IBM Blue Gene/Q CPU + * - POWER7 + - IBM POWER8 CPU + * - POWER8 + - IBM POWER8 CPU + * - POWER9 + - IBM POWER9 CPU + * - KEPLER30 + - NVIDIA Kepler generation CC 3.0 GPU + * - KEPLER32 + - NVIDIA Kepler generation CC 3.2 GPU + * - KEPLER35 + - NVIDIA Kepler generation CC 3.5 GPU + * - KEPLER37 + - NVIDIA Kepler generation CC 3.7 GPU + * - MAXWELL50 + - NVIDIA Maxwell generation CC 5.0 GPU + * - MAXWELL52 + - NVIDIA Maxwell generation CC 5.2 GPU + * - MAXWELL53 + - NVIDIA Maxwell generation CC 5.3 GPU + * - PASCAL60 + - NVIDIA Pascal generation CC 6.0 GPU + * - PASCAL61 + - NVIDIA Pascal generation CC 6.1 GPU + * - VOLTA70 + - NVIDIA Volta generation CC 7.0 GPU + * - VOLTA72 + - NVIDIA Volta generation CC 7.2 GPU + * - TURING75 + - NVIDIA Turing generation CC 7.5 GPU + +CMake build settings: +^^^^^^^^^^^^^^^^^^^^^ For multicore CPUs using OpenMP, set these 2 variables. .. code-block:: bash - -D KOKKOS_ARCH=archCPU # archCPU = CPU from list above - -D KOKKOS_ENABLE_OPENMP=yes + -D Kokkos_ARCH_CPUARCH=yes # CPUARCH = CPU from list above + -D Kokkos_ENABLE_OPENMP=yes + -D BUILD_OMP=yes -For Intel KNLs using OpenMP, set these 2 variables: +Please note that enabling OpenMP for KOKKOS requires that OpenMP is +also :ref:`enabled for the rest of LAMMPS `. + +For Intel KNLs using OpenMP, set these variables: .. code-block:: bash - -D KOKKOS_ARCH=KNL - -D KOKKOS_ENABLE_OPENMP=yes + -D Kokkos_ARCH_KNL=yes + -D Kokkos_ENABLE_OPENMP=yes -For NVIDIA GPUs using CUDA, set these 4 variables: +For NVIDIA GPUs using CUDA, set these variables: .. code-block:: bash - -D KOKKOS_ARCH="archCPU;archGPU" # archCPU = CPU from list above that is hosting the GPU - # archGPU = GPU from list above - -D KOKKOS_ENABLE_CUDA=yes - -D KOKKOS_ENABLE_OPENMP=yes - -D CMAKE_CXX_COMPILER=wrapper # wrapper = full path to Cuda nvcc wrapper + -D Kokkos_ARCH_CPUARCH=yes # CPUARCH = CPU from list above + -D Kokkos_ARCH_GPUARCH=yes # GPUARCH = GPU from list above + -D Kokkos_ENABLE_CUDA=yes + -D Kokkos_ENABLE_OPENMP=yes + -D CMAKE_CXX_COMPILER=wrapper # wrapper = full path to Cuda nvcc wrapper The wrapper value is the Cuda nvcc compiler wrapper provided in the Kokkos library: ``lib/kokkos/bin/nvcc_wrapper``\ . The setting should @@ -340,20 +439,35 @@ include the full path name to the wrapper, e.g. .. code-block:: bash - -D CMAKE_CXX_COMPILER=/home/username/lammps/lib/kokkos/bin/nvcc_wrapper + -D CMAKE_CXX_COMPILER=${HOME}/lammps/lib/kokkos/bin/nvcc_wrapper -**Traditional make**\ : +To simplify the compilation, three preset files are included in the +``cmake/presets`` folder, ``kokkos-serial.cmake``, ``kokkos-openmp.cmake``, +and ``kokkos-cuda.cmake``. They will enable the KOKKOS package and +enable some hardware choice. So to compile with OpenMP host parallelization, +CUDA device parallelization (for GPUs with CC 5.0 and up) with some +common packages enabled, you can do the following: + +.. code-block:: bash + + mkdir build-kokkos-cuda + cd build-kokkos-cuda + cmake -C ../cmake/presets/minimal.cmake -C ../cmake/presets/kokkos-cuda.cmake ../cmake + cmake --build . + +Traditional make settings: +^^^^^^^^^^^^^^^^^^^^^^^^^^ Choose which hardware to support in ``Makefile.machine`` via ``KOKKOS_DEVICES`` and ``KOKKOS_ARCH`` settings. See the -``src/MAKE/OPTIONS/Makefile.kokkos\*`` files for examples. +``src/MAKE/OPTIONS/Makefile.kokkos*`` files for examples. For multicore CPUs using OpenMP: .. code-block:: make KOKKOS_DEVICES = OpenMP - KOKKOS_ARCH = archCPU # archCPU = CPU from list above + KOKKOS_ARCH = CPUARCH # CPUARCH = CPU from list above For Intel KNLs using OpenMP: @@ -367,22 +481,28 @@ For NVIDIA GPUs using CUDA: .. code-block:: make KOKKOS_DEVICES = Cuda - KOKKOS_ARCH = archCPU,archGPU # archCPU = CPU from list above that is hosting the GPU - # archGPU = GPU from list above - FFT_INC = -DFFT_CUFFT # enable use of cuFFT (optional) - FFT_LIB = -lcufft # link to cuFFT library + KOKKOS_ARCH = CPUARCH,GPUARCH # CPUARCH = CPU from list above that is hosting the GPU + # GPUARCH = GPU from list above + FFT_INC = -DFFT_CUFFT # enable use of cuFFT (optional) + FFT_LIB = -lcufft # link to cuFFT library -For GPUs, you also need the following 2 lines in your Makefile.machine -before the CC line is defined, in this case for use with OpenMPI mpicxx. -The 2 lines define a nvcc wrapper compiler, which will use nvcc for -compiling CUDA files and use a C++ compiler for non-Kokkos, non-CUDA -files. +For GPUs, you also need the following lines in your ``Makefile.machine`` +before the CC line is defined. They tell ``mpicxx`` to use an ``nvcc`` +compiler wrapper, which will use ``nvcc`` for compiling CUDA files and a +C++ compiler for non-Kokkos, non-CUDA files. .. code-block:: make + # For OpenMPI KOKKOS_ABSOLUTE_PATH = $(shell cd $(KOKKOS_PATH); pwd) export OMPI_CXX = $(KOKKOS_ABSOLUTE_PATH)/config/nvcc_wrapper - CC = mpicxx + CC = mpicxx + +.. code-block:: make + + # For MPICH and derivatives + KOKKOS_ABSOLUTE_PATH = $(shell cd $(KOKKOS_PATH); pwd) + CC = mpicxx -cxx=$(KOKKOS_ABSOLUTE_PATH)/config/nvcc_wrapper ---------- @@ -421,7 +541,7 @@ args: $ make lib-latte args="-b" # download and build in lib/latte/LATTE-master $ make lib-latte args="-p $HOME/latte" # use existing LATTE installation in $HOME/latte $ make lib-latte args="-b -m gfortran" # download and build in lib/latte and - # copy Makefile.lammps.gfortran to Makefile.lammps + # copy Makefile.lammps.gfortran to Makefile.lammps Note that 3 symbolic (soft) links, "includelink" and "liblink" and "filelink.o", are created in lib/latte to point into the LATTE home diff --git a/doc/src/Build_link.rst b/doc/src/Build_link.rst index b18dc49e38f51cd23d80298be730b9e3bbdd5860..39fc4fe40a2f43bc075478d9ad94ba758f8d3492 100644 --- a/doc/src/Build_link.rst +++ b/doc/src/Build_link.rst @@ -1,33 +1,35 @@ Link LAMMPS as a library to another code ======================================== -LAMMPS can be used as a library by another application, including -Python scripts. The files src/library.cpp and library.h define the +LAMMPS is designed as a library of C++ objects and can thus be +integrated into other applications including Python scripts. +The files ``src/library.cpp`` and ``src/library.h`` define a C-style API for using LAMMPS as a library. See the :doc:`Howto library ` doc page for a description of the interface and how to extend it for your needs. The :doc:`Build basics ` doc page explains how to build -LAMMPS as either a shared or static library. This results in one of -these 2 files: - -.. code-block:: bash - - liblammps.so # shared library - liblammps.a # static library +LAMMPS as either a shared or static library. This results in a file +in the compilation folder called ``liblammps.a`` or ``liblammps_.a`` +in case of building a static library. In case of a shared library +the name is the same only that the suffix is going to be either ``.so`` +or ``.dylib`` or ``.dll`` instead of ``.a`` depending on the OS. +In some cases the ``.so`` file may be a symbolic link to a file with +the suffix ``.so.0`` (or some other number). .. note:: - Care should be taken to use the same MPI library for the calling - code and the LAMMPS library. The library.h file includes mpi.h and - uses definitions from it so those need to be available and - consistent. When LAMMPS is compiled with the MPI STUBS library, - then its mpi.h file needs to be included. While it is technically - possible to use a full MPI library in the calling code and link to - a serial LAMMPS library compiled with MPI STUBS, it is recommended - to use the *same* MPI library for both, and then use MPI_Comm_split() - in the calling code to pass a suitable communicator with a subset - of MPI ranks to the function creating the LAMMPS instance. + Care should be taken to use the same MPI library for the calling code + and the LAMMPS library. The ``library.h`` file includes ``mpi.h`` + and uses definitions from it so those need to be available and + consistent. When LAMMPS is compiled with the included STUBS MPI + library, then its ``mpi.h`` file needs to be included. While it is + technically possible to use a full MPI library in the calling code + and link to a serial LAMMPS library compiled with MPI STUBS, it is + recommended to use the *same* MPI library for both, and then use + ``MPI_Comm_split()`` in the calling code to pass a suitable + communicator with a subset of MPI ranks to the function creating the + LAMMPS instance. ---------- @@ -42,11 +44,11 @@ executable code from the library is copied into the calling executable. *CMake build*\ : -This assumes that LAMMPS has been configured with "-D BUILD_LIB=yes" -and installed with "make install" and the PKG_CONFIG_PATH environment -variable updated to include the *liblammps.pc* file installed into the -configured destination folder, if needed. The commands to compile and -link the coupled executable are then: +This assumes that LAMMPS has been configured without setting a +``LAMMPS_MACHINE`` name, installed with "make install", and the +``PKG_CONFIG_PATH`` environment variable has been updated to include the +``liblammps.pc`` file installed into the configured destination folder. +The commands to compile and link a coupled executable are then: .. code-block:: bash @@ -56,30 +58,35 @@ link the coupled executable are then: *Traditional make*\ : This assumes that LAMMPS has been compiled in the folder -"${HOME}/lammps/src" with "make mode=lib mpi". The commands to compile -and link the coupled executable are then: +``${HOME}/lammps/src`` with "make mpi". The commands to compile and link +a coupled executable are then: .. code-block:: bash mpicc -c -O -I${HOME}/lammps/src caller.c - mpicxx -o caller caller.o -L${HOME}/lammps/src -llammps + mpicxx -o caller caller.o -L${HOME}/lammps/src -llammps_mpi -The *-I* argument is the path to the location of the *library.h* +The *-I* argument is the path to the location of the ``library.h`` header file containing the interface to the LAMMPS C-style library -interface. The *-L* argument is the path to where the *liblammps.a* -file is located. The *-llammps* argument is shorthand for telling the -compiler to link the file *liblammps.a*\ . +interface. The *-L* argument is the path to where the ``liblammps_mpi.a`` +file is located. The *-llammps_mpi* argument is shorthand for telling the +compiler to link the file ``liblammps_mpi.a``. If LAMMPS has been +built as a shared library, then the linker will use ``liblammps_mpi.so`` +instead. If both files are available, the linker will usually prefer +the shared library. In case of a shared library, you may need to update +the ``LD_LIBRARY_PATH`` environment variable or running the ``caller`` +executable will fail since it cannot find the shared library at runtime. However, it is only as simple as shown above for the case of a plain LAMMPS library without any optional packages that depend on libraries -(bundled or external). Otherwise, you need to include all flags, -libraries, and paths for the coupled executable, that are also -required to link the LAMMPS executable. +(bundled or external) or when using a shared library. Otherwise, you +need to include all flags, libraries, and paths for the coupled +executable, that are also required to link the LAMMPS executable. *CMake build*\ : When using CMake, additional libraries with sources in the lib folder -are built, but not included in liblammps.a and (currently) not +are built, but not included in ``liblammps.a`` and (currently) not installed with "make install" and not included in the *pkgconfig* configuration file. They can be found in the top level build folder, but you have to determine the necessary link flags manually. It is @@ -87,23 +94,26 @@ therefore recommended to either use the traditional make procedure to build and link with a static library or build and link with a shared library instead. +.. TODO: this needs to be updated to reflect that latest CMake changes after they are complete. + *Traditional make*\ : After you have compiled a static LAMMPS library using the conventional -build system for example with "make mode=lib serial". And you also -have installed the POEMS package after building its bundled library in -lib/poems. Then the commands to build and link the coupled executable +build system for example with "make mode=static serial". And you also +have installed the ``POEMS`` package after building its bundled library +in ``lib/poems``. Then the commands to build and link the coupled executable change to: .. code-block:: bash gcc -c -O -I${HOME}/lammps/src/STUBS -I${HOME}/lammps/src -caller.c g++ -o caller caller.o -L${HOME}/lammps/lib/poems \ - -L${HOME}/lammps/src/STUBS -L${HOME}/lammps/src -llammps -lpoems -lmpi_stubs + -L${HOME}/lammps/src/STUBS -L${HOME}/lammps/src -llammps_serial -lpoems -lmpi_stubs -Note, that you need to link with "g++" instead of "gcc", since LAMMPS -is C++ code. You can display the currently applied settings for building -LAMMPS for the "serial" machine target by using the command: +Note, that you need to link with "g++" instead of "gcc", since the +LAMMPS library is C++ code. You can display the currently applied +settings for building LAMMPS for the "serial" machine target by using +the command: .. code-block:: bash @@ -113,16 +123,16 @@ Which should output something like: .. code-block:: bash - # Compiler: + # Compiler: CXX=g++ - # Linker: + # Linker: LD=g++ - # Compilation: - CXXFLAGS=-g -O3 -DLAMMPS_GZIP -DLAMMPS_MEMALIGN=64 -I${HOME}/lammps/lib/poems -I${HOME}/lammps/src/STUBS - # Linking: + # Compilation: + CXXFLAGS=-g -O3 -DLAMMPS_GZIP -DLAMMPS_MEMALIGN=64 -I${HOME}/compile/lammps/lib/poems -I${HOME}/compile/lammps/src/STUBS + # Linking: LDFLAGS=-g -O - # Libraries: - LDLIBS=-L${HOME}/lammps/lib/poems -L${HOME}/lammps/src/STUBS -lpoems -lmpi_stubs + # Libraries: + LDLIBS=-L${HOME}/compile/lammps/src -llammps_serial -L${HOME}/compile/lammps/lib/poems -L${HOME}/compile/lammps/src/STUBS -lpoems -lmpi_stubs From this you can gather the necessary paths and flags. With makefiles for other *machine* configurations you need to do the @@ -133,14 +143,13 @@ of the makefile. **Link with LAMMPS as a shared library**\ : -When linking to LAMMPS built as a shared library, the situation -becomes much simpler, as all dependent libraries and objects are -included in the shared library, which is - technically speaking - -effectively a regular LAMMPS executable that is missing the `main()` -function. Thus those libraries need not to be specified when linking -the calling executable. Only the *-I* flags are needed. So the -example case from above of the serial version static LAMMPS library -with the POEMS package installed becomes: +When linking to LAMMPS built as a shared library, the situation becomes +much simpler, as all dependent libraries and objects are either included +in the shared library or registered as a dependent library in the shared +library file. Thus those libraries need not to be specified when +linking the calling executable. Only the *-I* flags are needed. So the +example case from above of the serial version static LAMMPS library with +the POEMS package installed becomes: *CMake build*\ : @@ -155,19 +164,19 @@ build process are the same as for the static library. *Traditional make*\ : The commands with a shared LAMMPS library compiled with the -traditional make build using "make mode=shlib serial" becomes: +traditional make build using "make mode=shared serial" becomes: .. code-block:: bash gcc -c -O -I${HOME}/lammps/src/STUBS -I${HOME}/lammps/src -caller.c - g++ -o caller caller.o -L${HOME}/lammps/src -llammps + g++ -o caller caller.o -L${HOME}/lammps/src -llammps_serial *Locating liblammps.so at runtime*\ : -However, now the `liblammps.so` file is required at runtime and needs +However, now the ``liblammps.so`` file is required at runtime and needs to be in a folder, where the shared linker program of the operating -system can find it. This would be either a folder like "/usr/local/lib64" -or "${HOME}/.local/lib64" or a folder pointed to by the LD_LIBRARY_PATH +system can find it. This would be either a folder like ``/usr/local/lib64`` +or ``${HOME}/.local/lib64`` or a folder pointed to by the ``LD_LIBRARY_PATH`` environment variable. You can type .. code-block:: bash @@ -177,11 +186,11 @@ environment variable. You can type to see what directories are in that list. Or you can add the LAMMPS src directory (or the directory you performed -a CMake style build in) to your LD_LIBRARY_PATH, so that the current +a CMake style build in) to your ``LD_LIBRARY_PATH``, so that the current version of the shared library is always available to programs that use it. For the Bourne or Korn shells (/bin/sh, /bin/ksh, /bin/bash etc.), you -would add something like this to your ~/.profile file: +would add something like this to your ``${HOME}/.profile`` file: .. code-block:: bash @@ -189,14 +198,14 @@ would add something like this to your ~/.profile file: export LD_LIBRARY_PATH For the csh or tcsh shells, you would equivalently add something like this -to your ~/.cshrc file: +to your ``${HOME}/.cshrc`` file: .. code-block:: csh setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${HOME}/lammps/src You can verify whether all required shared libraries are found with the -`ldd` tool. Example: +``ldd`` tool. Example: .. code-block:: bash @@ -229,20 +238,20 @@ If a required library is missing, you would get a 'not found' entry: Either flavor of library (static or shared) allows one or more LAMMPS objects to be instantiated from the calling program. When used from a C++ program, most of the symbols and functions in LAMMPS are wrapped -in a LAMMPS_NS namespace; you can safely use any of its classes and +in a ``LAMMPS_NS`` namespace; you can safely use any of its classes and methods from within the calling code, as needed, and you will not incur conflicts with functions and variables in your code that share the name. This, however, does not extend to all additional libraries bundled with LAMMPS in the lib folder and some of the low-level code of some packages. To be compatible with C, Fortran, Python programs, the library has a simple -C-style interface, provided in src/library.cpp and src/library.h. +C-style interface, provided in ``src/library.cpp`` and ``src/library.h``. See the :doc:`Python library ` doc page for a description of the Python interface to LAMMPS, which wraps the C-style interface from a shared library through the `ctypes python module `_. -See the sample codes in examples/COUPLE/simple for examples of C++ and +See the sample codes in ``examples/COUPLE/simple`` for examples of C++ and C and Fortran codes that invoke LAMMPS through its library interface. Other examples in the COUPLE directory use coupling ideas discussed on the :doc:`Howto couple ` doc page. diff --git a/doc/src/Build_make.rst b/doc/src/Build_make.rst index 0379a8379a1bfa42beb6acbf993dc0ce2004b400..af5e319121768b41b0016f9be88f83a31965dfb8 100644 --- a/doc/src/Build_make.rst +++ b/doc/src/Build_make.rst @@ -2,10 +2,14 @@ Build LAMMPS with make ====================== Building LAMMPS with traditional makefiles requires that you have a -Makefile."machine" file appropriate for your system in the src/MAKE, -src/MAKE/MACHINES, src/MAKE/OPTIONS, or src/MAKE/MINE directory (see -below). It can include various options for customizing your LAMMPS -build with a number of global compilation options and features. +``Makefile.`` file appropriate for your system in either the +``src/MAKE``, ``src/MAKE/MACHINES``, ``src/MAKE/OPTIONS``, or +``src/MAKE/MINE`` directory (see below). It can include various options +for customizing your LAMMPS build with a number of global compilation +options and features. + +Requirements +^^^^^^^^^^^^ Those makefiles are written for and tested with GNU make and may not be compatible with other make programs. In most cases, if the "make" @@ -16,44 +20,54 @@ with :doc:`CMake `. The makefiles of the traditional make based build process and the scripts they are calling expect a few additional tools to be available and functioning. - * a Bourne shell compatible "Unix" shell program (often this is bash) - * a few shell utilities: ls, mv, ln, rm, grep, sed, tr, cat, touch, diff, dirname - * python (optional, required for "make lib-XXX" in the src folder) - -To include LAMMPS packages (i.e. optional commands and styles) you -must enable them first, as discussed on the :doc:`Build package -` doc page. If a packages requires (provided or + * a working C/C++ compiler toolchain supporting the C++11 standard; on + Linux these are often the GNU compilers. Some older compilers + require adding flags like ``-std=c++11`` to enable the C++11 mode. + * a Bourne shell compatible "Unix" shell program (often this is ``bash``) + * a few shell utilities: ``ls``, ``mv``, ``ln``, ``rm``, ``grep``, ``sed``, ``tr``, ``cat``, ``touch``, ``diff``, ``dirname`` + * python (optional, required for ``make lib-`` in the src folder). + python scripts are currently tested with python 2.7 and 3.6. The procedure + for :doc:`building the documentation ` requires python 3. + +Getting started +^^^^^^^^^^^^^^^ + +To include LAMMPS packages (i.e. optional commands and styles) you must +enable (or "install") them first, as discussed on the :doc:`Build +package ` doc page. If a packages requires (provided or external) libraries, you must configure and build those libraries **before** building LAMMPS itself and especially **before** enabling -such a package with "make yes-". Building :doc:`LAMMPS -with CMake ` can automate much of this for many types of +such a package with ``make yes-``. Building :doc:`LAMMPS with +CMake ` can automate much of this for many types of machines, especially workstations, desktops, and laptops, so we suggest you try it first when building LAMMPS in those cases. The commands below perform a default LAMMPS build, producing the LAMMPS -executable lmp_serial and lmp_mpi in lammps/src: +executable ``lmp_serial`` and ``lmp_mpi`` in ``lammps/src``: .. code-block:: bash - cd lammps/src - make serial # build a serial LAMMPS executable + cd lammps/src # change to main LAMMPS source folder + make serial # build a serial LAMMPS executable using GNU g++ make mpi # build a parallel LAMMPS executable with MPI make # see a variety of make options -This initial compilation can take a long time, since LAMMPS is a large -project with many features. If your machine has multiple CPU cores -(most do these days), using a command like "make -jN mpi" (with N = -the number of available CPU cores) can be much faster. If you plan to -do development on LAMMPS or need to re-compile LAMMPS repeatedly, the -installation of the ccache (= Compiler Cache) software may speed up -compilation even more. +Compilation can take a long time, since LAMMPS is a large project with +many features. If your machine has multiple CPU cores (most do these +days), you can speed this up by compiling sources in parallel with +``make -j N`` (with N being the maximum number of concurrently executed +tasks). Also installation of the `ccache `_ (= +Compiler Cache) software may speed up repeated compilation even more, +e.g. during code development. After the initial build, whenever you edit LAMMPS source files, or add or remove new files to the source directory (e.g. by installing or uninstalling packages), you must re-compile and relink the LAMMPS -executable with the same "make" command. This makefiles dependencies -should insure that only the subset of files that need to be are -re-compiled. +executable with the same ``make `` command. The makefile's +dependency tracking should insure that only the necessary subset of +files are re-compiled. If you change settings in the makefile, you have +to recompile *everything*. To delete all objects you can use ``make +clean-``. .. note:: @@ -65,13 +79,15 @@ re-compiled. correctly detect which parts need to be recompiled after changes were made to the sources. ----------- +Customized builds and alternate makefiles +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The lammps/src/MAKE tree contains the Makefile.machine files included -in the LAMMPS distribution. Typing "make machine" uses -*Makefile.machine*\ . Thus the "make serial" or "make mpi" lines above -use Makefile.serial and Makefile.mpi, respectively. Other makefiles -are in these directories: +The ``src/MAKE`` directory tree contains the ``Makefile.`` +files included in the LAMMPS distribution. Typing ``make example`` uses +``Makefile.example`` from one of those folders, if available. Thus the +``make serial`` and ``make mpi`` lines above use +``src/MAKE/Makefile.serial`` and ``src/MAKE/Makefile.mpi``, +respectively. Other makefiles are in these directories: .. code-block:: bash @@ -79,11 +95,16 @@ are in these directories: MACHINES # Makefiles for specific machines MINE # customized Makefiles you create (you may need to create this folder) -Typing "make" lists all the available Makefile.machine files. A file -with the same name can appear in multiple folders (not a good idea). -The order the directories are searched is as follows: src/MAKE/MINE, -src/MAKE, src/MAKE/OPTIONS, src/MAKE/MACHINES. This gives preference -to a customized file you put in src/MAKE/MINE. +Simply typing ``make`` lists all the available ``Makefile.`` +files with a single line description toward the end of the output. A +file with the same name can appear in multiple folders (not a good +idea). The order the directories are searched is as follows: +``src/MAKE/MINE``, ``src/MAKE``, ``src/MAKE/OPTIONS``, +``src/MAKE/MACHINES``. This gives preference to a customized file you +put in ``src/MAKE/MINE``. If you create your own custom makefile under +a new name, please edit the first line with the description and machine +name, so you will not confuse yourself, when looking at the machine +summary. Makefiles you may wish to try include these (some require a package first be installed). Many of these include specific compiler flags diff --git a/doc/src/Build_package.rst b/doc/src/Build_package.rst index 2e0bea723a08befbf8fadcbfb23c5898c34aa86b..f2511bb6a4a229a2bd8ef16b69e4c1ef0b4fee46 100644 --- a/doc/src/Build_package.rst +++ b/doc/src/Build_package.rst @@ -130,6 +130,8 @@ src directory. ---------- +.. _cmake_presets: + **CMake shortcuts for installing many packages**\ : Instead of specifying all the CMake options via the command-line, diff --git a/doc/src/Commands_all.rst b/doc/src/Commands_all.rst index 36ae112527631e7e7fb443a2623624b78b2a9cdd..92c0ba107aadc080950f1a312291cb335fc4e022 100644 --- a/doc/src/Commands_all.rst +++ b/doc/src/Commands_all.rst @@ -70,6 +70,7 @@ An alphabetic list of all general LAMMPS commands. * :doc:`kim_init ` * :doc:`kim_interactions ` * :doc:`kim_param ` + * :doc:`kim_property ` * :doc:`kim_query ` * :doc:`kspace_modify ` * :doc:`kspace_style ` diff --git a/doc/src/Commands_pair.rst b/doc/src/Commands_pair.rst index 34b04969f9fa457cfafbf9f2f20ef2da39770372..b176e22a857af106e587d499a39cb6a7eb390016 100644 --- a/doc/src/Commands_pair.rst +++ b/doc/src/Commands_pair.rst @@ -131,6 +131,7 @@ OPT. * :doc:`lj/class2/coul/cut (ko) ` * :doc:`lj/class2/coul/cut/soft ` * :doc:`lj/class2/coul/long (gko) ` + * :doc:`lj/class2/coul/long/cs ` * :doc:`lj/class2/coul/long/soft ` * :doc:`lj/class2/soft ` * :doc:`lj/cubic (go) ` diff --git a/doc/src/Commands_removed.rst b/doc/src/Commands_removed.rst index b749467d13f9d5aac68d2df3d5720a242ab60ab8..e137e39a0868dc4b6b74c5dc36ce8acb1e22fc0e 100644 --- a/doc/src/Commands_removed.rst +++ b/doc/src/Commands_removed.rst @@ -48,7 +48,7 @@ The USER-CUDA package had been removed, since it had been unmaintained for a long time and had known bugs and problems. Significant parts of the design were transferred to the :ref:`KOKKOS package `, which has similar -performance characteristics on Nvidia GPUs. Both, the KOKKOS +performance characteristics on NVIDIA GPUs. Both, the KOKKOS and the :ref:`GPU package ` are maintained and allow running LAMMPS with GPU acceleration. diff --git a/doc/src/Errors.rst b/doc/src/Errors.rst index 896dfbcbef63d7f5151f437f2e1ecd0d331b3304..48db5f6472d7b039982d6a4857ff35f21482fb24 100644 --- a/doc/src/Errors.rst +++ b/doc/src/Errors.rst @@ -12,5 +12,6 @@ additional details for many of them. Errors_common Errors_bugs + Errors_debug Errors_messages Errors_warnings diff --git a/doc/src/Errors_bugs.rst b/doc/src/Errors_bugs.rst index e169b93d8189c14da09b91c20c1db0e4c26003c2..ff0b155795397efbaa0d0675c17e9a4ebb5a2821 100644 --- a/doc/src/Errors_bugs.rst +++ b/doc/src/Errors_bugs.rst @@ -1,7 +1,8 @@ Reporting bugs ============== -If you are confident that you have found a bug in LAMMPS, please follow the steps outlined below: +If you are confident that you have found a bug in LAMMPS, please follow +the steps outlined below: * Check the `New features and bug fixes `_ section of the `LAMMPS WWW site @@ -17,20 +18,22 @@ If you are confident that you have found a bug in LAMMPS, please follow the step * Check the `mailing list archives `_ to see if the issue has been discussed before. -If none of these steps yields any useful information, please file -a new bug report on the `GitHub Issue page `_\ . -The website will offer you to select a suitable template with explanations -and then you should replace those explanations with the information -that you can provide to reproduce your issue. +If none of these steps yields any useful information, please file a new +bug report on the `GitHub Issue page `_. The website will offer +you to select a suitable template with explanations and then you should +replace those explanations with the information that you can provide to +reproduce your issue. The most useful thing you can do to help us verify and fix a bug is to isolate the problem. Run it on the smallest number of atoms and fewest number of processors with the simplest input script that reproduces the -bug. Try to identify what command or combination of commands is -causing the problem and upload the complete input deck as a tar or zip -archive. Please avoid using binary restart files unless the issue requires -it. In the latter case you should also include an input deck to quickly +bug. Try to identify what command or combination of commands is causing +the problem and upload the complete input deck as a tar or zip archive. +Please avoid using binary restart files unless the issue requires it. +In the latter case you should also include an input deck to quickly generate this restart from a data file or a simple additional input. +This input deck can be used with tools like a debugger or `valgrind +`_ to further :doc:`debug the crash `. You may also send an email to the LAMMPS mailing list at "lammps-users at lists.sourceforge.net" describing the problem with the @@ -43,3 +46,4 @@ have looked at it. .. _lws: https://lammps.sandia.gov .. _gip: https://github.com/lammps/issues +.. _valgrind: https://valgrind.org diff --git a/doc/src/Errors_debug.rst b/doc/src/Errors_debug.rst new file mode 100644 index 0000000000000000000000000000000000000000..1e7e678602d83068645ae8fb721e74aa8417fbaf --- /dev/null +++ b/doc/src/Errors_debug.rst @@ -0,0 +1,237 @@ +Debugging crashes +================= + +If LAMMPS crashes with a "segmentation fault" or a "bus error" or +similar message, then you can use the following two methods to further +narrow down the origin of the issue. This will help the LAMMPS +developers (or yourself) to understand the reason for the crash and +apply a fix (either to the input script or the source code). +This requires that your LAMMPS executable includes the required +:ref:`debug information `. Otherwise it is not possible to +look up the names of functions or variables. + +The following patch will introduce a bug into the code for pair style +:doc:`lj/cut ` when using the ``examples/melt/in.melt`` input. +We use it to show how to identify the origin of a segmentation fault. + +.. code-block:: diff + + --- a/src/pair_lj_cut.cpp + +++ b/src/pair_lj_cut.cpp + @@ -81,6 +81,7 @@ void PairLJCut::compute(int eflag, int vflag) + int nlocal = atom->nlocal; + double *special_lj = force->special_lj; + int newton_pair = force->newton_pair; + + double comx = 0.0; + + inum = list->inum; + ilist = list->ilist; + @@ -134,8 +135,10 @@ void PairLJCut::compute(int eflag, int vflag) + evdwl,0.0,fpair,delx,dely,delz); + } + } + - } + + + comx += atom->rmass[i]*x[i][0]; /* BUG */ + + } + + printf("comx = %g\n",comx); + if (vflag_fdotr) virial_fdotr_compute(); + } + +After recompiling LAMMPS and running the input you should get something like this: + +.. code-block: + + $ ./lmp -in in.melt + LAMMPS (19 Mar 2020) + OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (src/comm.cpp:94) + using 1 OpenMP thread(s) per MPI task + Lattice spacing in x,y,z = 1.6796 1.6796 1.6796 + Created orthogonal box = (0 0 0) to (16.796 16.796 16.796) + 1 by 1 by 1 MPI processor grid + Created 4000 atoms + create_atoms CPU = 0.000432253 secs + Neighbor list info ... + update every 20 steps, delay 0 steps, check no + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 2.8 + ghost atom cutoff = 2.8 + binsize = 1.4, bins = 12 12 12 + 1 neighbor lists, perpetual/occasional/extra = 1 0 0 + (1) pair lj/cut, perpetual + attributes: half, newton on + pair build: half/bin/atomonly/newton + stencil: half/bin/3d/newton + bin: standard + Setting up Verlet run ... + Unit style : lj + Current step : 0 + Time step : 0.005 + Segmentation fault (core dumped) + + +Using the GDB debugger to get a stack trace +------------------------------------------- + +There are two options to use the GDB debugger for identifying the origin +of the segmentation fault or similar crash. The GDB debugger has many +more features and options, as can be seen for example its `online +documentation `_. + +Run LAMMPS from within the debugger +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Running LAMMPS under the control of the debugger as shown below only +works for a single MPI rank (for debugging a program running in parallel +you usually need a parallel debugger program). A simple way to launch +GDB is to prefix the LAMMPS command line with ``gdb --args`` and then +type the command "run" at the GDB prompt. This will launch the +debugger, load the LAMMPS executable and its debug info, and then run +it. When it reaches the code causing the segmentation fault, it will +stop with a message why it stopped, print the current line of code, and +drop back to the GDB prompt. + +.. code-block:: + + [...] + Setting up Verlet run ... + Unit style : lj + Current step : 0 + Time step : 0.005 + + Program received signal SIGSEGV, Segmentation fault. + 0x00000000006653ab in LAMMPS_NS::PairLJCut::compute (this=0x829740, eflag=1, vflag=) at /home/akohlmey/compile/lammps/src/pair_lj_cut.cpp:139 + 139 comx += atom->rmass[i]*x[i][0]; /* BUG */ + (gdb) + +Now typing the command "where" will show the stack of functions starting from +the current function back to "main()". + +.. code-block:: + + (gdb) where + #0 0x00000000006653ab in LAMMPS_NS::PairLJCut::compute (this=0x829740, eflag=1, vflag=) at /home/akohlmey/compile/lammps/src/pair_lj_cut.cpp:139 + #1 0x00000000004cf0a2 in LAMMPS_NS::Verlet::setup (this=0x7e6c90, flag=1) at /home/akohlmey/compile/lammps/src/verlet.cpp:131 + #2 0x000000000049db42 in LAMMPS_NS::Run::command (this=this@entry=0x7fffffffcca0, narg=narg@entry=1, arg=arg@entry=0x7e8750) + at /home/akohlmey/compile/lammps/src/run.cpp:177 + #3 0x000000000041258a in LAMMPS_NS::Input::command_creator (lmp=, narg=1, arg=0x7e8750) + at /home/akohlmey/compile/lammps/src/input.cpp:878 + #4 0x0000000000410ad3 in LAMMPS_NS::Input::execute_command (this=0x7d1410) at /home/akohlmey/compile/lammps/src/input.cpp:864 + #5 0x00000000004111fb in LAMMPS_NS::Input::file (this=0x7d1410) at /home/akohlmey/compile/lammps/src/input.cpp:229 + #6 0x000000000040933a in main (argc=, argv=) at /home/akohlmey/compile/lammps/src/main.cpp:65 + (gdb) + +You can also print the value of variables and see if there is anything +unexpected. Segmentation faults, for example, commonly happen when a +pointer variable is not assigned and still initialized to NULL. + +.. code-block:: + + (gdb) print x + $1 = (double **) 0x7ffff7ca1010 + (gdb) print i + $2 = 0 + (gdb) print x[0] + $3 = (double *) 0x7ffff6d80010 + (gdb) print x[0][0] + $4 = 0 + (gdb) print x[1][0] + $5 = 0.83979809569125363 + (gdb) print atom->rmass + $6 = (double *) 0x0 + (gdb) + + +Inspect a core dump file with the debugger +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When an executable crashes with a "core dumped" message, it creates a +file "core" or "core." which contains the information about the +current state. This file may be located in the folder where you ran +LAMMPS or in some hidden folder managed by the systemd daemon. In the +latter case, you need to "extract" the core file with the ``coredumpctl`` +utility to the current folder. Example: ``coredumpctl -o core dump lmp``. +Now you can launch the debugger to load the executable, its debug info +and the core dump and drop you to a prompt like before. + +.. code-block:: + + $ gdb lmp core + Reading symbols from lmp... + [New LWP 1928535] + [Thread debugging using libthread_db enabled] + Using host libthread_db library "/lib64/libthread_db.so.1". + Core was generated by `./lmp -in in.melt'. + Program terminated with signal SIGSEGV, Segmentation fault. + #0 0x00000000006653ab in LAMMPS_NS::PairLJCut::compute (this=0x1b10740, eflag=1, vflag=) + at /home/akohlmey/compile/lammps/src/pair_lj_cut.cpp:139 + 139 comx += atom->rmass[i]*x[i][0]; /* BUG */ + (gdb) + +From here on, you use the same commands as shown before to get a stack +trace and print current values of (pointer) variables. + + +Using valgrind to get a stack trace +----------------------------------- + +The `valgrind `_ suite of tools allows to closely +inspect the behavior of a compiled program by essentially emulating a +CPU and instrumenting the program while running. This slows down +execution quite significantly, but can also report issues that are not +resulting in a crash. The default valgrind tool is a memory checker and +you can use it by prefixing the normal command line with ``valgrind``. +Unlike GDB, this will also work for parallel execution, but it is +recommended to redirect the valgrind output to a file (e.g. with +``--log-file=crash-%p.txt``, the %p will be substituted with the +process ID) so that the messages of the multiple valgrind instances to +the console are not mixed. + +.. code-block:: + + $ valgrind ./lmp -in in.melt + ==1933642== Memcheck, a memory error detector + ==1933642== Copyright (C) 2002-2017, and GNU GPL'd, by Julian Seward et al. + ==1933642== Using Valgrind-3.15.0 and LibVEX; rerun with -h for copyright info + ==1933642== Command: ./lmp -in in.melt + ==1933642== + LAMMPS (19 Mar 2020) + OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (src/comm.cpp:94) + using 1 OpenMP thread(s) per MPI task + Lattice spacing in x,y,z = 1.6796 1.6796 1.6796 + Created orthogonal box = (0 0 0) to (16.796 16.796 16.796) + 1 by 1 by 1 MPI processor grid + Created 4000 atoms + create_atoms CPU = 0.032964 secs + Neighbor list info ... + update every 20 steps, delay 0 steps, check no + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 2.8 + ghost atom cutoff = 2.8 + binsize = 1.4, bins = 12 12 12 + 1 neighbor lists, perpetual/occasional/extra = 1 0 0 + (1) pair lj/cut, perpetual + attributes: half, newton on + pair build: half/bin/atomonly/newton + stencil: half/bin/3d/newton + bin: standard + Setting up Verlet run ... + Unit style : lj + Current step : 0 + Time step : 0.005 + ==1933642== Invalid read of size 8 + ==1933642== at 0x6653AB: LAMMPS_NS::PairLJCut::compute(int, int) (pair_lj_cut.cpp:139) + ==1933642== by 0x4CF0A1: LAMMPS_NS::Verlet::setup(int) (verlet.cpp:131) + ==1933642== by 0x49DB41: LAMMPS_NS::Run::command(int, char**) (run.cpp:177) + ==1933642== by 0x412589: void LAMMPS_NS::Input::command_creator(LAMMPS_NS::LAMMPS*, int, char**) (input.cpp:881) + ==1933642== by 0x410AD2: LAMMPS_NS::Input::execute_command() (input.cpp:864) + ==1933642== by 0x4111FA: LAMMPS_NS::Input::file() (input.cpp:229) + ==1933642== by 0x409339: main (main.cpp:65) + ==1933642== Address 0x0 is not stack'd, malloc'd or (recently) free'd + ==1933642== + +As you can see, the stack trace information is similar to that obtained +from GDB. In addition you get a more specific hint about what cause the +segmentation fault, i.e. that it is a NULL pointer dereference. To find +out which pointer exactly was NULL, you need to use the debugger, though. + diff --git a/doc/src/Errors_messages.rst b/doc/src/Errors_messages.rst index 6970cb4a9a30daa22b1e8c0987fe0a58782f8135..2140719253c95cbc519f2928fcb29b79199b628f 100644 --- a/doc/src/Errors_messages.rst +++ b/doc/src/Errors_messages.rst @@ -2783,7 +2783,7 @@ Doc page with :doc:`WARNING messages ` if this helps. *Did not find all elements in MEAM library file* - The requested elements were not found in the MEAM file. + Some requested elements were not found in the MEAM file. Check spelling etc. *Did not find fix shake partner info* Could not find bond partners implied by fix shake command. This error @@ -3135,6 +3135,9 @@ Doc page with :doc:`WARNING messages ` *Epsilon or sigma reference not set by pair style in ewald/n* The pair style is not providing the needed epsilon or sigma values. +*Error in MEAM parameter file: keyword %s (further information)* + Self-explanatory. Check the parameter file. + *Error in vdw spline: inner radius > outer radius* A pre-tabulated spline is invalid. Likely a problem with the potential parameters. @@ -4611,7 +4614,7 @@ Doc page with :doc:`WARNING messages ` *Incorrect format in COMB3 potential file* Incorrect number of words per line in the potential file. -*Incorrect format in MEAM potential file* +*Incorrect format in MEAM library file* Incorrect number of words per line in the potential file. *Incorrect format in SNAP coefficient file* @@ -5669,6 +5672,9 @@ Doc page with :doc:`WARNING messages ` A fix is referenced incorrectly or a fix that produces per-atom values is used in an equal-style variable formula. +*Mismatched parameter in MEAM library file: z!=lat* + The coordination number and lattice do not match, check that consistent values are given. + *Mismatched variable in variable formula* A variable is referenced incorrectly or an atom-style variable that produces per-atom values is used in an equal-style variable @@ -5845,6 +5851,9 @@ Doc page with :doc:`WARNING messages ` Cannot use the temper command with only one processor partition. Use the -partition command-line option. +*Must not have multiple fixes change box parameter ...* + Self-explanatory. + *Must read Atoms before Angles* The Atoms section of a data file must come before an Angles section. @@ -7764,6 +7773,9 @@ keyword to allow for additional bonds to be formed *Too many atoms to dump sort* Cannot sort when running with more than 2\^31 atoms. +*Too many elements extracted from MEAM library.* + Increase 'maxelt' in meam.h and recompile. + *Too many exponent bits for lookup table* Table size specified via pair_modify command does not work with your machine's floating point representation. @@ -7997,11 +8009,11 @@ keyword to allow for additional bonds to be formed *Unknown unit_style* Self-explanatory. Check the input script or data file. -*Unrecognized lattice type in MEAM file 1* +*Unrecognized lattice type in MEAM library file* The lattice type in an entry of the MEAM library file is not valid. -*Unrecognized lattice type in MEAM file 2* +*Unrecognized lattice type in MEAM parameter file* The lattice type in an entry of the MEAM parameter file is not valid. @@ -8017,6 +8029,9 @@ keyword to allow for additional bonds to be formed *Unsupported order in kspace_style pppm/disp, pair_style %s* Only pair styles with 1/r and 1/r\^6 dependence are currently supported. +*Unsupported parameter in MEAM library file* + Self-explanatory. + *Use cutoff keyword to set cutoff in single mode* Mode is single so cutoff/multi keyword cannot be used. diff --git a/doc/src/Howto.rst b/doc/src/Howto.rst index f4376898b1ba3940d37c6f9549267c398dbc78f6..0ca955ab86283c48c028864a69a6a23d52e8cf1c 100644 --- a/doc/src/Howto.rst +++ b/doc/src/Howto.rst @@ -16,6 +16,7 @@ Tutorials howto :name: tutorials :maxdepth: 1 + Howto_cmake Howto_github Howto_pylammps Howto_bash diff --git a/doc/src/Howto_bash.rst b/doc/src/Howto_bash.rst index 02322e5e1cf82095fa1aeaa3bab27bad87e75d28..b452f579d03804b1d4a94dbc21b051d78aaba0fe 100644 --- a/doc/src/Howto_bash.rst +++ b/doc/src/Howto_bash.rst @@ -12,6 +12,10 @@ via apt-get and all files are accessible in both the Windows Explorer and your Linux shell (bash). This avoids switching to a different operating system or installing a virtual machine. Everything runs on Windows. +.. seealso:: + + You can find more detailed information at the `Windows Subsystem for Linux Installation Guide for Windows 10 `_. + Installing Bash on Windows -------------------------- @@ -103,7 +107,7 @@ needed for various LAMMPS features: .. code-block:: bash - sudo apt install -y build-essential ccache gfortran openmpi-bin libopenmpi-dev libfftw3-dev libjpeg-dev libpng12-dev python-dev python-virtualenv libblas-dev liblapack-dev libhdf5-serial-dev hdf5-tools + sudo apt install -y build-essential ccache gfortran openmpi-bin libopenmpi-dev libfftw3-dev libjpeg-dev libpng-dev python-dev python-virtualenv libblas-dev liblapack-dev libhdf5-serial-dev hdf5-tools Files in Ubuntu on Windows ^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/doc/src/Howto_cmake.rst b/doc/src/Howto_cmake.rst new file mode 100644 index 0000000000000000000000000000000000000000..db7b88dc3dd3b79fe8a340aa68049a6f98ee0785 --- /dev/null +++ b/doc/src/Howto_cmake.rst @@ -0,0 +1,483 @@ +Using CMake with LAMMPS tutorial +================================ + +The support for building LAMMPS with CMake is a recent addition to +LAMMPS thanks to the efforts of Christoph Junghans (LANL) and Richard +Berger (Temple U). One of the key strengths of CMake is that it is not +tied to a specific platform or build system and thus generate the files +necessary to build and develop for different build systems and on +different platforms. Note, that this applies to the build system itself +not the LAMMPS code. In other words, without additional porting effort, +it is not possible - for example - to compile LAMMPS with Visual C++ on +Windows. The build system output can also include support files +necessary to program LAMMPS as a project in integrated development +environments (IDE) like Eclipse, Visual Studio, QtCreator, Xcode, +CodeBlocks, Kate and others. + +A second important feature of CMake is, that it can detect and validate +available libraries, optimal settings, available support tools and so +on, so that by default LAMMPS will take advantage of available tools +without requiring to provide the details about how to enable/integrate +them. + +The downside of this approach is, that there is some complexity +associated with running CMake itself and how to customize the building +of LAMMPS. This tutorial will show how to manage this through some +selected examples. Please see the chapter about :doc:`building LAMMPS +` for descriptions of specific flags and options for LAMMPS in +general and for specific packages. + +CMake can be used through either the command-line interface (CLI) +program ``cmake`` (or ``cmake3``), a text mode interactive user +interface (TUI) program ``ccmake`` (or ``ccmake3``), or a graphical user +interface (GUI) program ``cmake-gui``. All of them are portable +software available on all supported platforms and can be used +interchangeably. The minimum supported CMake version is 3.10 (3.12 or +later is recommended). + +All details about features and settings for CMake are in the `CMake +online documentation `_. We focus +below on the most important aspects with respect to compiling LAMMPS. + +Prerequisites +------------- + +This tutorial assumes that you are operating in a command-line environment +using a shell like Bash. + +- Linux: any Terminal window will work +- MacOS X: launch the Terminal application. +- Windows 10: install and run the :doc:`Windows subsystem for Linux ` + +We also assume that you have downloaded and unpacked a recent LAMMPS source code package +or used Git to create a clone of the LAMMPS sources on your compilation machine. + +You should change into the top level directory of the LAMMPS source tree all +paths mentioned in the tutorial are relative to that. Immediately after downloading +it should look like this: + +.. code-block:: bash + + $ ls + bench doc lib potentials README tools + cmake examples LICENSE python src + +Build versus source directory +----------------------------- + +When using CMake the build procedure is separated into multiple distinct phases: + + #. **Configuration:** detect or define which features and settings + should be enable and used and how LAMMPS should be compiled + #. **Compilation:** generate and compile all necessary source files + and build libraries and executables. + #. **Installation:** copy selected files from the compilation into + your file system, so they can be used without having to keep the + source and build tree around. + +The configuration and compilation of LAMMPS has to happen in a dedicated +*build directory* which must be different from the source directory. +Also the source directory (``src``) must remain pristine, so it is not +allowed to "install" packages using the traditional make process and +after an compilation attempt all created source files must be removed. +This can be achieved with ``make no-all purge``. + +You can pick **any** folder outside the source tree. We recommend to +create a folder ``build`` in the top-level directory, or multiple +folders in case you want to have separate builds of LAMMPS with +different options (``build-parallel``, ``build-serial``) or with +different compilers (``build-gnu``, ``build-clang``, ``build-intel``) +and so on. All the auxiliary files created by one build process +(executable, object files, log files, etc) are stored in this directory +or sub-directories within it that CMake creates. + + +Running CMake +------------- + +CLI version +^^^^^^^^^^^ + +In the (empty) ``build`` directory, we now run the command ``cmake +../cmake``, which will start the configuration phase and you will see +the progress of the configuration printed to the screen followed by a +summary of the enabled features, options and compiler settings. A typical +summary screen will look like this: + +.. code-block:: + + $ cmake ../cmake/ + -- The CXX compiler identification is GNU 8.2.0 + -- Check for working CXX compiler: /opt/tools/gcc-8.2.0/bin/c++ + -- Check for working CXX compiler: /opt/tools/gcc-8.2.0/bin/c++ - works + -- Detecting CXX compiler ABI info + -- Detecting CXX compiler ABI info - done + -- Detecting CXX compile features + -- Detecting CXX compile features - done + -- Found Git: /usr/bin/git (found version "2.25.2") + -- Running check for auto-generated files from make-based build system + -- Found MPI_CXX: /usr/lib64/mpich/lib/libmpicxx.so (found version "3.1") + -- Found MPI: TRUE (found version "3.1") + -- Looking for C++ include omp.h + -- Looking for C++ include omp.h - found + -- Found OpenMP_CXX: -fopenmp (found version "4.5") + -- Found OpenMP: TRUE (found version "4.5") + -- Found JPEG: /usr/lib64/libjpeg.so (found version "62") + -- Found PNG: /usr/lib64/libpng.so (found version "1.6.37") + -- Found ZLIB: /usr/lib64/libz.so (found version "1.2.11") + -- Found GZIP: /usr/bin/gzip + -- Found FFMPEG: /usr/bin/ffmpeg + -- Performing Test COMPILER_SUPPORTS-ffast-math + -- Performing Test COMPILER_SUPPORTS-ffast-math - Success + -- Performing Test COMPILER_SUPPORTS-march=native + -- Performing Test COMPILER_SUPPORTS-march=native - Success + -- Looking for C++ include cmath + -- Looking for C++ include cmath - found + -- Generating style_angle.h... + [...] + -- Generating lmpinstalledpkgs.h... + -- The following tools and libraries have been found and configured: + * Git + * MPI + * OpenMP + * JPEG + * PNG + * ZLIB + + -- <<< Build configuration >>> + Build type: RelWithDebInfo + Install path: /home/akohlmey/.local + Generator: Unix Makefiles using /usr/bin/gmake + -- <<< Compilers and Flags: >>> + -- C++ Compiler: /opt/tools/gcc-8.2.0/bin/c++ + Type: GNU + Version: 8.2.0 + C++ Flags: -O2 -g -DNDEBUG + Defines: LAMMPS_SMALLBIG;LAMMPS_MEMALIGN=64;LAMMPS_JPEG;LAMMPS_PNG;LAMMPS_GZIP;LAMMPS_FFMPEG + Options: -ffast-math;-march=native + -- <<< Linker flags: >>> + -- Executable name: lmp + -- Static library flags: + -- <<< MPI flags >>> + -- MPI includes: /usr/include/mpich-x86_64 + -- MPI libraries: /usr/lib64/mpich/lib/libmpicxx.so;/usr/lib64/mpich/lib/libmpi.so; + -- Configuring done + -- Generating done + -- Build files have been written to: /home/akohlmey/compile/lammps/build + +The ``cmake`` command has one mandatory argument, and that is a folder +with either the file ``CMakeLists.txt`` or ``CMakeCache.txt``. The +``CMakeCache.txt`` file is created during the CMake configuration run +and contains all active settings, thus after a first run of CMake +all future runs in the build folder can use the folder ``.`` and CMake +will know where to find the CMake scripts and reload the settings +from the previous step. This means, that one can modify an existing +configuration by re-running CMake, but only needs to provide flags +indicating the desired change, everything else will be retained. One +can also mix compilation and configuration, i.e. start with a minimal +configuration and then, if needed, enable additional features and +recompile. + +The steps above **will NOT compile the code**\ . The compilation can be +started in a portable fashion with ``cmake --build .``, or you use the +selected built tool, e.g. ``make``. + +TUI version +^^^^^^^^^^^ + +For the text mode UI CMake program the basic principle is the same. +You start the command ``ccmake ../cmake`` in the ``build`` folder. + +.. list-table:: + + * - .. figure:: JPG/ccmake-initial.png + :target: JPG/ccmake-initial.png + :align: center + + Initial ``ccmake`` screen + + - .. figure:: JPG/ccmake-config.png + :target: JPG/ccmake-config.png + :align: center + + Configure output of ``ccmake`` + + - .. figure:: JPG/ccmake-options.png + :target: JPG/ccmake-options.png + :align: center + + Options screen of ``ccmake`` + +This will show you the initial screen (left image) with the empty +configuration cache. Now you type the 'c' key to run the configuration +step. That will do a first configuration run and show the summary +(center image). You exit the summary screen with 'e' and see now the +main screen with detected options and settings. You can now make changes +by moving and down with the arrow keys of the keyboard and modify +entries. For on/off settings, the enter key will toggle the state. +For others, hitting enter will allow you to modify the value and +you commit the change by hitting the enter key again or cancel using +the escape key. All "new" settings will be marked with a star '\*' +and for as long as one setting is marked like this, you have to +re-run the configuration by hitting the 'c' key again, sometimes +multiple times unless the TUI shows the word "generate" next to the +letter 'g' and by hitting the 'g' key the build files will be written +to the folder and the TUI exits. You can quit without generating +build files by hitting 'q'. + +GUI version +^^^^^^^^^^^ + +For the graphical CMake program the steps are similar to the TUI +version. You can type the command ``cmake-gui ../cmake`` in the +``build`` folder. In this case the path to the CMake script folder is +not required, it can also be entered from the GUI. + +.. list-table:: + + * - .. figure:: JPG/cmake-gui-initial.png + :target: JPG/cmake-gui-initial.png + :align: center + + Initial ``cmake-gui`` screen + + - .. figure:: JPG/cmake-gui-popup.png + :target: JPG/cmake-gui-popup.png + :align: center + + Generator selection in ``cmake-gui`` + + - .. figure:: JPG/cmake-gui-options.png + :target: JPG/cmake-gui-options.png + :align: center + + Options screen of ``cmake-gui`` + +Again, you start with an empty configuration cache (left image) and need +to start the configuration step. For the very first configuration in a +folder, you will have a pop-up dialog (center image) asking to select +the desired build tool and some configuration settings (stick with the +default) and then you get the option screen with all new settings +highlighted in red. You can modify them (or not) and click on the +"configure" button again until satisfied and click on the "generate" +button to write out the build files. You can exit the GUI from the +"File" menu or hit "ctrl-q". + + +Setting options +--------------- + +Options that enable, disable or modify settings are modified by setting +the value of CMake variables. This is done on the command line with the +*-D* flag in the format ``-D VARIABLE=value``, e.g. ``-D +CMAKE_BUILD_TYPE=Release`` or ``-D BUILD_MPI=on``. There is one quirk: +when used before the CMake directory, there may be a space between the +*-D* flag and the variable, after it must not be. Such CMake variables +can have boolean values (on/off, yes/no, or 1/0 are all valid) or are +strings representing a choice, or a path, or are free format. If the +string would contain whitespace, it must be put in quotes, for example +``-D CMAKE_TUNE_FLAGS="-ftree-vectorize -ffast-math"``. + +CMake variables fall into two categories: 1) common CMake variables that +are used by default for any CMake configuration setup and 2) project +specific variables, i.e. settings that are specific for LAMMPS. +Also CMake variables can be flagged as *advanced*, which means they are +not shown in the text mode or graphical CMake program in the overview +of all settings by default, but only when explicitly requested (by hitting +the 't' key or clicking on the 'Advanced' check-box). + +Some common CMake variables +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :header-rows: 1 + + * - Variable + - Description + * - ``CMAKE_INSTALL_PREFIX`` + - root directory of install location for ``make install`` (default: ``$HOME/.local``) + * - ``CMAKE_BUILD_TYPE`` + - controls compilation options: + one of ``RelWithDebInfo`` (default), ``Release``, ``Debug``, ``MinSizeRel`` + * - ``BUILD_SHARED_LIBS`` + - if set to ``on`` build the LAMMPS library as shared library (default: ``off``) + * - ``CMAKE_MAKE_PROGRAM`` + - name/path of the compilation command (default depends on *-G* option, usually ``make``) + * - ``CMAKE_VERBOSE_MAKEFILE`` + - if set to ``on`` echo commands while executing during build (default: ``off``) + * - ``CMAKE_C_COMPILER`` + - C compiler to be used for compilation (default: system specific, ``gcc`` on Linux) + * - ``CMAKE_CXX_COMPILER`` + - C++ compiler to be used for compilation (default: system specific, ``g++`` on Linux) + * - ``CMAKE_Fortran_COMPILER`` + - Fortran compiler to be used for compilation (default: system specific, ``gfortran`` on Linux) + * - ``CXX_COMPILER_LAUNCHER`` + - tool to launch the C++ compiler, e.g. ``ccache`` or ``distcc`` for faster compilation (default: empty) + +Some common LAMMPS specific variables +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :header-rows: 1 + + * - Variable + - Description + * - ``BUILD_MPI`` + - build LAMMPS with MPI support (default: ``on`` if a working MPI available, else ``off``) + * - ``BUILD_OMP`` + - build LAMMPS with OpenMP support (default: ``on`` if compiler supports OpenMP fully, else ``off``) + * - ``BUILD_TOOLS`` + - compile some additional executables from the ``tools`` folder (default: ``off``) + * - ``BUILD_DOC`` + - include building the HTML format documentation for packaging/installing (default: ``off``) + * - ``CMAKE_TUNE_FLAGS`` + - common compiler flags, for optimization or instrumentation (default: compiler specific) + * - ``LAMMPS_MACHINE`` + - when set to ``name`` the LAMMPS executable and library will be called ``lmp_name`` and ``liblammps_name.a`` + * - ``LAMMPS_EXCEPTIONS`` + - when set to ``on`` errors will throw a C++ exception instead of aborting (default: ``off``) + * - ``FFT`` + - select which FFT library to use: ``FFTW3``, ``MKL``, ``KISS`` (default, unless FFTW3 is found) + * - ``FFT_SINGLE`` + - select whether to use single precision FFTs (default: ``off``) + * - ``WITH_JPEG`` + - whether to support JPEG format in :doc:`dump image ` (default: ``on`` if found) + * - ``WITH_PNG`` + - whether to support PNG format in :doc:`dump image ` (default: ``on`` if found) + * - ``WITH_GZIP`` + - whether to support reading and writing compressed files (default: ``on`` if found) + * - ``WITH_FFMPEG`` + - whether to support generating movies with :doc:`dump movie ` (default: ``on`` if found) + +Enabling or disabling LAMMPS packages +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The LAMMPS software is organized into a common core that is always +included and a large number of :doc:`add-on packages ` that +have to be enabled to be included into a LAMMPS executable. Packages +are enabled through setting variables of the kind ``PKG_`` to +``on`` and disabled by setting them to ``off`` (or using ``yes``, +``no``, ``1``, ``0`` correspondingly). ```` has to be replaced by +the name of the package, e.g. ``MOLECULE`` or ``USER-MISC``. + + +Using presets +------------- + +Since LAMMPS has a lot of optional features and packages, specifying +them all on the command line can be tedious. Or when selecting a +different compiler toolchain, multiple options have to be changed +consistently and that is rather error prone. Or when enabling certain +packages, they require consistent settings to be operated in a +particular mode. For this purpose, we are providing a selection of +"preset files" for CMake in the folder ``cmake/presets``. They +represent a way to pre-load or override the CMake configuration cache by +setting or changing CMake variables. Preset files are loaded using the +*-C* command line flag. You can combine loading multiple preset files or +change some variables later with additional *-D* flags. A few examples: + +.. code-block:: bash + + cmake -C ../cmake/preset/minimal.cmake -D PKG_MISC=on ../cmake + cmake -C ../cmake/preset/clang.cmake -C ../cmake/preset/most.cmake ../cmake + cmake -C ../cmake/preset/minimal.cmake -D BUILD_MPI=off ../cmake + +The first command will install the packages ``KSPACE``, ``MANYBODY``, +``MOLECULE``, ``RIGID`` and ``MISC``; the first four from the preset +file and the fifth from the explicit variable definition. The second +command will first switch the compiler toolchain to use the Clang +compilers and install a large number of packages that are not depending +on any special external libraries or tools and are not very unusual. +The third command will enable the first four packages like above and +then enforce compiling LAMMPS as a serial program (using the MPI STUBS +library). + +It is also possible to do this incrementally. + +.. code-block:: bash + + cmake -C ../cmake/preset/minimal.cmake ../cmake + cmake -D PKG_MISC=on . + +will achieve the same configuration like in the first example above. In +this scenario it is particularly convenient to do the second +configuration step using either the text mode or graphical user +interface (``ccmake`` or ``cmake-gui``). + +Compilation and build targets +----------------------------- + +The actual compilation will be started by running the selected build +command (on Linux this is by default ``make``, see below how to select +alternatives). You can also use the portable command ``cmake --build .`` +which will adapt to whatever the selected build command is. +This is particularly convenient, if you have set a custom build command +via the ``CMAKE_MAKE_PROGRAM`` variable. + +When calling the build program, you can also select which "target" is to +be build through appending the name of the target to the build command. +Example: ``cmake --build . all``. The following abstract targets are available: + +.. list-table:: + :header-rows: 1 + + * - Target + - Description + * - ``all`` + - build "everything" (default) + * - ``lammps`` + - build the LAMMPS library and executable + * - ``doc`` + - build the HTML documentation (if configured) + * - ``install`` + - install all target files into folders in ``CMAKE_INSTALL_PREFIX`` + * - ``test`` + - run some simple tests (if configured with ``-D ENABLE_TESTING=on``) + * - ``clean`` + - remove all generated files + + +Choosing generators +------------------- + +While CMake usually defaults to creating makefiles to compile software +with the ``make`` program, it supports multiple alternate build tools +(e.g. ``ninja-build`` which tends to be faster and more efficient in +parallelizing builds than ``make``) and can generate project files for +integrated development environments (IDEs) like VisualStudio, Eclipse or +CodeBlocks. This is specific to how the local CMake version was +configured and compiled. The list of available options can be seen at +the end of the output of ``cmake --help``. Example on Fedora 31 this is: + +.. code-block:: + + Generators + + The following generators are available on this platform (* marks default): + * Unix Makefiles = Generates standard UNIX makefiles. + Green Hills MULTI = Generates Green Hills MULTI files + (experimental, work-in-progress). + Ninja = Generates build.ninja files. + Ninja Multi-Config = Generates build-.ninja files. + Watcom WMake = Generates Watcom WMake makefiles. + CodeBlocks - Ninja = Generates CodeBlocks project files. + CodeBlocks - Unix Makefiles = Generates CodeBlocks project files. + CodeLite - Ninja = Generates CodeLite project files. + CodeLite - Unix Makefiles = Generates CodeLite project files. + Sublime Text 2 - Ninja = Generates Sublime Text 2 project files. + Sublime Text 2 - Unix Makefiles + = Generates Sublime Text 2 project files. + Kate - Ninja = Generates Kate project files. + Kate - Unix Makefiles = Generates Kate project files. + Eclipse CDT4 - Ninja = Generates Eclipse CDT 4.0 project files. + Eclipse CDT4 - Unix Makefiles= Generates Eclipse CDT 4.0 project files. + +Below is a screenshot of using the CodeBlocks IDE with the ninja build tool +after running CMake as follows: + +.. code-block:: bash + + cmake -G 'CodeBlocks - Ninja' ../cmake/presets/most.cmake ../cmake/ + +.. image:: JPG/cmake-codeblocks.png + :align: center diff --git a/doc/src/Install_mac.rst b/doc/src/Install_mac.rst index 41b53c0595c4e796c0ac99192b466b1c01321b14..642169ffe5ff1ae0a5f4324e8e9a3d77a6f69922 100644 --- a/doc/src/Install_mac.rst +++ b/doc/src/Install_mac.rst @@ -34,9 +34,9 @@ install the `openkim-models` package % brew install openkim-models If you have problems with the installation you can post issues to -`this link `_. +`this link `_. -.. _homebrew: https://github.com/Homebrew/homebrew-core/issues +.. _homebrew: https://brew.sh Thanks to Derek Thomas (derekt at cello.t.u-tokyo.ac.jp) for setting up the Homebrew capability. diff --git a/doc/src/JPG/bow_tutorial_01.png b/doc/src/JPG/bow_tutorial_01.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/bow_tutorial_01_small.png b/doc/src/JPG/bow_tutorial_01_small.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/bow_tutorial_02.png b/doc/src/JPG/bow_tutorial_02.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/bow_tutorial_02_small.png b/doc/src/JPG/bow_tutorial_02_small.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/bow_tutorial_03.png b/doc/src/JPG/bow_tutorial_03.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/bow_tutorial_03_small.png b/doc/src/JPG/bow_tutorial_03_small.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/bow_tutorial_04.png b/doc/src/JPG/bow_tutorial_04.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/bow_tutorial_04_small.png b/doc/src/JPG/bow_tutorial_04_small.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/bow_tutorial_05.png b/doc/src/JPG/bow_tutorial_05.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/bow_tutorial_06.png b/doc/src/JPG/bow_tutorial_06.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/bow_tutorial_07.png b/doc/src/JPG/bow_tutorial_07.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/bow_tutorial_08.png b/doc/src/JPG/bow_tutorial_08.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/bow_tutorial_09.png b/doc/src/JPG/bow_tutorial_09.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/bow_tutorial_10.png b/doc/src/JPG/bow_tutorial_10.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/ccmake-config.png b/doc/src/JPG/ccmake-config.png new file mode 100644 index 0000000000000000000000000000000000000000..ef7508964653c4c217f36ed39b5c4fdb93e175ce Binary files /dev/null and b/doc/src/JPG/ccmake-config.png differ diff --git a/doc/src/JPG/ccmake-initial.png b/doc/src/JPG/ccmake-initial.png new file mode 100644 index 0000000000000000000000000000000000000000..d6291afde52f96d0debf00affca6864ba821b663 Binary files /dev/null and b/doc/src/JPG/ccmake-initial.png differ diff --git a/doc/src/JPG/ccmake-options.png b/doc/src/JPG/ccmake-options.png new file mode 100644 index 0000000000000000000000000000000000000000..b05d58c1d2e87d48128fa3538d56bc935ddc3b95 Binary files /dev/null and b/doc/src/JPG/ccmake-options.png differ diff --git a/doc/src/JPG/cmake-codeblocks.png b/doc/src/JPG/cmake-codeblocks.png new file mode 100644 index 0000000000000000000000000000000000000000..d0d96fec8c5666fe3eb0721c426a435ea73c8167 Binary files /dev/null and b/doc/src/JPG/cmake-codeblocks.png differ diff --git a/doc/src/JPG/cmake-gui-initial.png b/doc/src/JPG/cmake-gui-initial.png new file mode 100644 index 0000000000000000000000000000000000000000..062bc6ba836c3ab6ecf7ea31954362e0441353b8 Binary files /dev/null and b/doc/src/JPG/cmake-gui-initial.png differ diff --git a/doc/src/JPG/cmake-gui-options.png b/doc/src/JPG/cmake-gui-options.png new file mode 100644 index 0000000000000000000000000000000000000000..abbf63494612a88e2223d28f12474d28db374613 Binary files /dev/null and b/doc/src/JPG/cmake-gui-options.png differ diff --git a/doc/src/JPG/cmake-gui-popup.png b/doc/src/JPG/cmake-gui-popup.png new file mode 100644 index 0000000000000000000000000000000000000000..d623aad30b4be2364a466fdde4af5241a0eb87dc Binary files /dev/null and b/doc/src/JPG/cmake-gui-popup.png differ diff --git a/doc/src/JPG/offload_knc.png b/doc/src/JPG/offload_knc.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/rhodo_staggered.jpg b/doc/src/JPG/rhodo_staggered.jpg old mode 100755 new mode 100644 diff --git a/doc/src/JPG/saed_ewald_intersect.jpg b/doc/src/JPG/saed_ewald_intersect.jpg old mode 100755 new mode 100644 diff --git a/doc/src/JPG/saed_ewald_intersect_small.jpg b/doc/src/JPG/saed_ewald_intersect_small.jpg old mode 100755 new mode 100644 diff --git a/doc/src/JPG/saed_mesh.jpg b/doc/src/JPG/saed_mesh.jpg old mode 100755 new mode 100644 diff --git a/doc/src/JPG/saed_mesh_small.jpg b/doc/src/JPG/saed_mesh_small.jpg old mode 100755 new mode 100644 diff --git a/doc/src/JPG/user_intel.png b/doc/src/JPG/user_intel.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/xrd_mesh.jpg b/doc/src/JPG/xrd_mesh.jpg old mode 100755 new mode 100644 diff --git a/doc/src/JPG/xrd_mesh_small.jpg b/doc/src/JPG/xrd_mesh_small.jpg old mode 100755 new mode 100644 diff --git a/doc/src/Packages_details.rst b/doc/src/Packages_details.rst index e20c2886ed4a639eddbd682c707b387e7bdcb0b8..8251c5301ea8885d7b418b15e51b870b9082fb46 100644 --- a/doc/src/Packages_details.rst +++ b/doc/src/Packages_details.rst @@ -365,12 +365,17 @@ KIM package This package contains a set of commands that serve as a wrapper on the `Open Knowledgebase of Interatomic Models (OpenKIM) `_ -repository of interatomic models (IMs) -enabling compatible ones to be used in LAMMPS simulations. -This includes :doc:`kim_init and kim_interactions ` -commands to select, initialize and instantiate the IM, and a -:doc:`kim_query ` command to perform web queries -for material property predictions of OpenKIM IMs. +repository of interatomic models (IMs) enabling compatible ones to be used in +LAMMPS simulations. + +This includes :doc:`kim_init `, and +:doc:`kim_interactions ` commands to select, initialize and +instantiate the IM, a :doc:`kim_query ` command to perform web +queries for material property predictions of OpenKIM IMs, a +:doc:`kim_param ` command to access KIM Model Parameters from +LAMMPS, and a :doc:`kim_property ` command to write material +properties computed in LAMMPS to standard KIM property instance format. + Support for KIM IMs that conform to the `KIM Application Programming Interface (API) `_ is provided by the :doc:`pair_style kim ` command. @@ -392,13 +397,16 @@ The KIM project is led by Ellad Tadmor and Ryan Elliott (U Minnesota) and is funded by the `National Science Foundation `_. **Authors:** Ryan Elliott (U Minnesota) is the main developer for the KIM -API and the *pair_style kim* command. Axel Kohlmeyer (Temple U) and -Ellad Tadmor (U Minnesota) contributed to the :doc:`kim_commands ` -interface in close collaboration with Ryan Elliott. +API and the *pair_style kim* command. Yaser Afshar (U Minnesota), +Axel Kohlmeyer (Temple U), Ellad Tadmor (U Minnesota), and +Daniel Karls (U Minnesota) contributed to the +:doc:`kim_commands ` interface in close collaboration with +Ryan Elliott. **Install:** -This package has :ref:`specific installation instructions ` on the :doc:`Build extras ` doc page. +This package has :ref:`specific installation instructions ` on the +:doc:`Build extras ` doc page. **Supporting info:** diff --git a/doc/src/Python_shlib.rst b/doc/src/Python_shlib.rst index 0c86362e1293d7fc6a7d5139bbf1e23ec33e3526..b02fcd7bb861aafeec10b8c6ed072f8dc2711ed9 100644 --- a/doc/src/Python_shlib.rst +++ b/doc/src/Python_shlib.rst @@ -1,6 +1,8 @@ Build LAMMPS as a shared library ================================ +.. TODO this is mostly redundant and should be addressed in the 'progguide' branch if it has not already + Build LAMMPS as a shared library using make ------------------------------------------- @@ -14,7 +16,7 @@ From the src directory, type .. code-block:: bash - make foo mode=shlib + make foo mode=shared where foo is the machine target name, such as mpi or serial. This should create the file liblammps_foo.so in the src directory, as @@ -39,7 +41,6 @@ shared library: .. code-block:: bash - -D BUILD_LIB=on # enable building LAMMPS as a library -D BUILD_SHARED_LIBS=on # enable building of LAMMPS shared library (both options are needed!) What this does is create a liblammps.so which contains the majority of LAMMPS @@ -59,7 +60,7 @@ CMAKE_INSTALL_PREFIX. # build library mkdir build cd build - cmake -D PKG_PYTHON=on -D BUILD_LIB=on -D BUILD_SHARED_LIBS=on -D CMAKE_INSTALL_PREFIX=$VIRTUAL_ENV ../cmake + cmake -D PKG_PYTHON=on -D BUILD_SHARED_LIBS=on -D CMAKE_INSTALL_PREFIX=$VIRTUAL_ENV ../cmake make -j 4 # install into prefix diff --git a/doc/src/Speed_compare.rst b/doc/src/Speed_compare.rst index d3947ec3ef856b4d22d6b8b2c5968974184516d5..4ab616a5dab8903fe864c0f4d5a2c1c8de271d4e 100644 --- a/doc/src/Speed_compare.rst +++ b/doc/src/Speed_compare.rst @@ -87,7 +87,7 @@ section below for examples where this has been done. * The GPU package requires neighbor lists to be built on the CPU when using exclusion lists, or a triclinic simulation box. * The GPU package can be compiled for CUDA or OpenCL and thus supports - both, Nvidia and AMD GPUs well. On Nvidia hardware, using CUDA is typically + both, NVIDIA and AMD GPUs well. On NVIDIA hardware, using CUDA is typically resulting in equal or better performance over OpenCL. * OpenCL in the GPU package does theoretically also support Intel CPUs or Intel Xeon Phi, but the native support for those in KOKKOS (or USER-INTEL) diff --git a/doc/src/Speed_kokkos.rst b/doc/src/Speed_kokkos.rst index 66589570068aaa1a595c2ccfc1e594c86c7c8831..ab8444b845d5e559c3c515765027b94abd9e0623 100644 --- a/doc/src/Speed_kokkos.rst +++ b/doc/src/Speed_kokkos.rst @@ -38,7 +38,7 @@ compatible with specific hardware. .. note:: To build with Kokkos support for NVIDIA GPUs, NVIDIA CUDA - software version 7.5 or later must be installed on your system. See + software version 9.0 or later must be installed on your system. See the discussion for the :doc:`GPU package ` for details of how to check and do this. diff --git a/doc/src/Speed_packages.rst b/doc/src/Speed_packages.rst index ab02ba7f48a20fce7afcb5b985dd46c1f5f24dff..685bd3c601e9a3ac9bdcf9d62286c2e0445a0a2a 100644 --- a/doc/src/Speed_packages.rst +++ b/doc/src/Speed_packages.rst @@ -20,7 +20,7 @@ standard or user packages: +-----------------------------------------+-------------------------------------------------------+ | :doc:`USER-INTEL Package ` | for Intel CPUs and Intel Xeon Phi | +-----------------------------------------+-------------------------------------------------------+ -| :doc:`KOKKOS Package ` | for Nvidia GPUs, Intel Xeon Phi, and OpenMP threading | +| :doc:`KOKKOS Package ` | for NVIDIA GPUs, Intel Xeon Phi, and OpenMP threading | +-----------------------------------------+-------------------------------------------------------+ | :doc:`USER-OMP Package ` | for OpenMP threading and generic CPU optimizations | +-----------------------------------------+-------------------------------------------------------+ diff --git a/doc/src/balance.rst b/doc/src/balance.rst index 5e631fdd2754941ed30c5eea7e0df04e2e0dcb45..58737249242557d5ce1e140cfc44b5c0018b73f6 100644 --- a/doc/src/balance.rst +++ b/doc/src/balance.rst @@ -175,23 +175,27 @@ a logical 3d grid of processors. They operate by changing the cutting planes (or lines) between processors in 3d (or 2d), to adjust the volume (area in 2d) assigned to each processor, as in the following 2d diagram where processor sub-domains are shown and particles are -colored by the processor that owns them. The leftmost diagram is the -default partitioning of the simulation box across processors (one -sub-box for each of 16 processors); the middle diagram is after a -"grid" method has been applied. - -.. image:: JPG/balance_uniform_small.jpg - :target: JPG/balance_uniform.jpg -.. image:: JPG/balance_nonuniform_small.jpg - :target: JPG/balance_nonuniform.jpg -.. image:: JPG/balance_rcb_small.jpg - :target: JPG/balance_rcb.jpg - -The *rcb* style is a "tiling" method which does not produce a logical -3d grid of processors. Rather it tiles the simulation domain with -rectangular sub-boxes of varying size and shape in an irregular -fashion so as to have equal numbers of particles (or weight) in each -sub-box, as in the rightmost diagram above. +colored by the processor that owns them. + +.. list-table:: + + * - .. figure:: JPG/balance_uniform_small.jpg + :target: JPG/balance_uniform.jpg + + - .. figure:: JPG/balance_nonuniform_small.jpg + :target: JPG/balance_nonuniform.jpg + + - .. figure:: JPG/balance_rcb_small.jpg + :target: JPG/balance_rcb.jpg + +The leftmost diagram is the default partitioning of the simulation box +across processors (one sub-box for each of 16 processors); the middle +diagram is after a "grid" method has been applied. The *rcb* style is a +"tiling" method which does not produce a logical 3d grid of processors. +Rather it tiles the simulation domain with rectangular sub-boxes of +varying size and shape in an irregular fashion so as to have equal +numbers of particles (or weight) in each sub-box, as in the rightmost +diagram above. The "grid" methods can be used with either of the :doc:`comm_style ` command options, *brick* or *tiled*\ . The diff --git a/doc/src/bond_oxdna.rst b/doc/src/bond_oxdna.rst index 8e69b298bf0f1dcce514ed698418a385f87c3bb2..71e510543601da1074251a26417af70be90f6fee 100644 --- a/doc/src/bond_oxdna.rst +++ b/doc/src/bond_oxdna.rst @@ -32,7 +32,7 @@ Examples bond_coeff * 2.0 0.25 0.7564 bond_style oxrna2/fene - bond_coeff \* 2.0 0.25 0.76107 + bond_coeff * 2.0 0.25 0.76107 Description """"""""""" diff --git a/doc/src/fix_adapt.rst b/doc/src/fix_adapt.rst index eab8f92639b7b31543b985640e0d968bdc3f1e9f..fbc4aa466170e447a1fbad8ecf6b294ba06220a6 100644 --- a/doc/src/fix_adapt.rst +++ b/doc/src/fix_adapt.rst @@ -14,7 +14,7 @@ Syntax * adapt = style name of this fix command * N = adapt simulation settings every this many timesteps * one or more attribute/arg pairs may be appended -* attribute = *pair* or *kspace* or *atom* +* attribute = *pair* or *bond* or *kspace* or *atom* .. parsed-literal:: @@ -86,8 +86,8 @@ the end of a simulation. Even if *reset* is specified as *yes*\ , a restart file written during a simulation will contain the modified settings. -If the *scale* keyword is set to *no*\ , then the value the parameter is -set to will be whatever the variable generates. If the *scale* +If the *scale* keyword is set to *no*\ , then the value of the altered +parameter will be whatever the variable generates. If the *scale* keyword is set to *yes*\ , then the value of the altered parameter will be the initial value of that parameter multiplied by whatever the variable generates. I.e. the variable is now a "scale factor" applied @@ -288,11 +288,19 @@ Currently *bond* does not support bond_style hybrid nor bond_style hybrid/overlay as bond styles. The only bonds that currently are working with fix_adapt are -+---------------------------------+-------+------------+ -| :doc:`gromos ` | k, r0 | type bonds | -+---------------------------------+-------+------------+ -| :doc:`harmonic ` | k,r0 | type bonds | -+---------------------------------+-------+------------+ ++------------------------------------+-------+------------+ +| :doc:`class2 ` | r0 | type bonds | ++------------------------------------+-------+------------+ +| :doc:`fene ` | k, r0 | type bonds | ++------------------------------------+-------+------------+ +| :doc:`gromos ` | k, r0 | type bonds | ++------------------------------------+-------+------------+ +| :doc:`harmonic ` | k,r0 | type bonds | ++------------------------------------+-------+------------+ +| :doc:`morse ` | r0 | type bonds | ++------------------------------------+-------+------------+ +| :doc:`nonlinear ` | r0 | type bonds | ++------------------------------------+-------+------------+ ---------- @@ -311,26 +319,23 @@ The *atom* keyword enables various atom properties to be changed. The current list of atom parameters that can be varied by this fix: * charge = charge on particle -* diameter = diameter of particle +* diameter, or, diameter/disc = diameter of particle The *v_name* argument of the *atom* keyword is the name of an :doc:`equal-style variable ` which will be evaluated each time -this fix is invoked to set the parameter to a new value. It should be -specified as v_name, where name is the variable name. See the +this fix is invoked to set, or scale the parameter to a new value. +It should be specified as v_name, where name is the variable name. See the discussion above describing the formulas associated with equal-style variables. The new value is assigned to the corresponding attribute for all atoms in the fix group. -.. note:: - - The *atom* keyword works this way whether the *scale* keyword is - set to *no* or *yes*\ . I.e. the use of scale yes is not yet supported - by the *atom* keyword. - If the atom parameter is *diameter* and per-atom density and per-atom mass are defined for particles (e.g. :doc:`atom_style granular `), then the mass of each particle is also -changed when the diameter changes (density is assumed to stay -constant). +changed when the diameter changes. The mass is set from the particle volume +for 3d systems (density is assumed to stay constant). For 2d, the default is +for LAMMPS to model particles with a radius attribute as spheres. +However, if the atom parameter is *diameter/disc*, then the mass is +set from the particle area (the density is assumed to be in mass/distance^2 units). For example, these commands would shrink the diameter of all granular particles in the "center" group from 1.0 to 0.1 in a linear fashion diff --git a/doc/src/fix_balance.rst b/doc/src/fix_balance.rst index 13fdf6b1c76cdf4ccb1ae6133eace2569564952e..b92644fcd916a96fe7b86ee30c81bed92ba5c01c 100644 --- a/doc/src/fix_balance.rst +++ b/doc/src/fix_balance.rst @@ -154,28 +154,27 @@ of processors. It operates by changing the cutting planes (or lines) between processors in 3d (or 2d), to adjust the volume (area in 2d) assigned to each processor, as in the following 2d diagram where processor sub-domains are shown and atoms are colored by the processor -that owns them. The leftmost diagram is the default partitioning of -the simulation box across processors (one sub-box for each of 16 -processors); the middle diagram is after a "grid" method has been -applied. - -.. |bal_uni| image:: JPG/balance_uniform_small.jpg - :target: JPG/balance_uniform.jpg - :width: 31% -.. |bal_non| image:: JPG/balance_nonuniform_small.jpg - :target: JPG/balance_nonuniform.jpg - :width: 31% -.. |bal_rcb| image:: JPG/balance_rcb_small.jpg - :target: JPG/balance_rcb.jpg - :width: 31% - -|bal_uni| |bal_non| |bal_rcb| - -The *rcb* style is a "tiling" method which does not produce a logical -3d grid of processors. Rather it tiles the simulation domain with -rectangular sub-boxes of varying size and shape in an irregular -fashion so as to have equal numbers of particles (or weight) in each -sub-box, as in the rightmost diagram above. +that owns them. + +.. list-table:: + + * - .. figure:: JPG/balance_uniform_small.jpg + :target: JPG/balance_uniform.jpg + + - .. figure:: JPG/balance_nonuniform_small.jpg + :target: JPG/balance_nonuniform.jpg + + - .. figure:: JPG/balance_rcb_small.jpg + :target: JPG/balance_rcb.jpg + +The leftmost diagram is the default partitioning of the simulation box +across processors (one sub-box for each of 16 processors); the middle +diagram is after a "grid" method has been applied. The *rcb* style is a +"tiling" method which does not produce a logical 3d grid of processors. +Rather it tiles the simulation domain with rectangular sub-boxes of +varying size and shape in an irregular fashion so as to have equal +numbers of particles (or weight) in each sub-box, as in the rightmost +diagram above. The "grid" methods can be used with either of the :doc:`comm_style ` command options, *brick* or *tiled*\ . The diff --git a/doc/src/fix_precession_spin.rst b/doc/src/fix_precession_spin.rst index 065f894926343b6ae3d930a2a12d0847d8c2731b..783963af7223a36c075a3cb2a7763e5368c73f65 100644 --- a/doc/src/fix_precession_spin.rst +++ b/doc/src/fix_precession_spin.rst @@ -76,6 +76,7 @@ function for the same parameters. .. image:: JPG/zeeman_langevin.jpg :align: center + :width: 600 The temperature effects are accounted for by connecting the spin :math:`i` to a thermal bath using a Langevin thermostat (see @@ -154,6 +155,11 @@ The *precession/spin* style is part of the SPIN package. This style is only enabled if LAMMPS was built with this package, and if the atom_style "spin" was declared. See the :doc:`Build package ` doc page for more info. +The *precession/spin* style can only be declared once. If more +than one precession type (for example combining an anisotropy and a Zeeman interactions) +has to be declared, they have to be chained in the same command +line (as shown in the examples above). + Related commands """""""""""""""" diff --git a/doc/src/kim_commands.rst b/doc/src/kim_commands.rst index c710ddfe5dcd3211cb568ced5c40ad355f4c8bd8..99f7efffd5dd7186270fff48da6e2aa019167cc7 100644 --- a/doc/src/kim_commands.rst +++ b/doc/src/kim_commands.rst @@ -1,16 +1,19 @@ -.. index:: kim_init, kim_interactions, kim_query, kim_param +.. index:: kim_init, kim_interactions, kim_query, kim_param, kim_property -kim_init command -================= +:ref:`kim_init` command +========================================= -kim_interactions command -========================= +:ref:`kim_interactions` command +========================================================= -kim_query command -================== +:ref:`kim_query` command +=========================================== -kim_param command -================== +:ref:`kim_param` command +=========================================== + +:ref:`kim_property` command +================================================= Syntax """""" @@ -22,6 +25,11 @@ Syntax kim_query variable formatarg query_function queryargs kim_param get param_name index_range variables formatarg kim_param set param_name index_range values + kim_property create instance_id property_id + kim_property modify instance_id key key_name key_name_key key_name_value + kim_property remove instance_id key key_name + kim_property destroy instance_id + kim_property dump file .. _formatarg_options: @@ -48,6 +56,12 @@ Syntax * param_name = name of a KIM portable model parameter * index_range = KIM portable model parameter index range (an integer for a single element, or pair of integers separated by a colon for a range of elements) * values = new value(s) to replace the current value(s) of a KIM portable model parameter +* instance_id = a positive integer identifying the KIM property instance +* property_id = identifier of a `KIM Property Definition `_, which can be (1) a property short name, (2) the full unique ID of the property (including the contributor and date), (3) a file name corresponding to a local property definition file +* key_name = one of the keys belonging to the specified KIM property definition +* key_name_key = a key belonging to a key-value pair (standardized in the `KIM Properties Framework `__) +* key_name_value = value to be associated with a key_name_key in a key-value pair +* file = name of a file to write the currently defined set of KIM property instances to Examples """""""" @@ -64,6 +78,15 @@ Examples kim_query a0 get_lattice_constant_cubic crystal=["fcc"] species=["Al"] units=["angstrom"] kim_param get gamma 1 varGamma kim_param set gamma 1 3.0 + kim_property create 1 atomic-mass + kim_property modify 1 key mass source-value 26.98154 + kim_property modify 1 key species source-value Al + kim_property remove 1 key species + kim_property destroy 1 + kim_property dump results.edn + + +.. _kim_description: Description """"""""""" @@ -157,11 +180,10 @@ See the `current list of KIM PMs and SMs archived in OpenKIM `_ to -learn how to install a pre-build binary of the OpenKIM Repository of Models. +See `Obtaining KIM Models `_ to +learn how to install a pre-built binary of the OpenKIM Repository of Models. .. note:: - It is also possible to locally install IMs not archived in OpenKIM, in which case their names do not have to conform to the KIM ID format. @@ -169,15 +191,17 @@ Using OpenKIM IMs with LAMMPS ----------------------------- Two commands are employed when using OpenKIM IMs, one to select the -IM and perform necessary initialization (*kim_init*), and the second +IM and perform necessary initialization (\ *kim_init*\ ), and the second to set up the IM for use by executing any necessary LAMMPS commands -(*kim_interactions*). Both are required. +(\ *kim_interactions*\ ). Both are required. See the *examples/kim* directory for example input scripts that use KIM PMs and KIM SMs. +.. _kim_init command: + OpenKIM IM Initialization (*kim_init*) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The *kim_init* mode command must be issued **before** the simulation box is created (normally at the top of the file). @@ -219,7 +243,7 @@ either match the required units of the IM or the IM must be able to adjust its units to match. (The latter is only possible with some KIM PMs; SMs can never adjust their units.) If a match is possible, the LAMMPS :doc:`units ` command is called to set the units to -*user_units*. If the match fails, the simulation is terminated with +*user_units*\ . If the match fails, the simulation is terminated with an error. Here is an example of a LAMMPS script to compute the cohesive energy @@ -324,8 +348,10 @@ be done to convert the box and all atomic positions to the correct units: all appropriate places in the input script. It is up to the user to do this correctly. +.. _kim_interactions command: + OpenKIM IM Execution (*kim_interactions*) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The second and final step in using an OpenKIM IM is to execute the *kim_interactions* command. This command must be preceded by a *kim_init* @@ -399,12 +425,17 @@ the *kim_interactions* command executes the following LAMMPS input commands: pair_coeff * * ffield.reax.rdx C H N O fix reaxqeq all qeq/reax 1 0.0 10.0 1.0e-6 param.qeq -Note that the files *lmp_control*, *ffield.reax.rdx* and *param.qeq* -are specific to the Strachan et al. (2003) ReaxFF parameterization -and are archived as part of the SM package in OpenKIM. -Note also that parameters like cutoff radii and charge tolerances, -which have an effect on IM predictions, are also included in the -SM definition ensuring reproducibility. +.. note:: + + The files *lmp_control*, *ffield.reax.rdx* and *param.qeq* + are specific to the Strachan et al. (2003) ReaxFF parameterization + and are archived as part of the SM package in OpenKIM. + +.. note:: + + Parameters like cutoff radii and charge tolerances, + which have an effect on IM predictions, are also included in the + SM definition ensuring reproducibility. .. note:: @@ -414,8 +445,10 @@ SM definition ensuring reproducibility. bond_coeff, fixes related to charge equilibration, etc.) should normally not appear in the input script. +.. _kim_query command: + Using OpenKIM Web Queries in LAMMPS (*kim_query*) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The *kim_query* command performs a web query to retrieve the predictions of an IM set by *kim_init* for material properties archived in @@ -427,6 +460,7 @@ of an IM set by *kim_init* for material properties archived in The syntax for the *kim_query* command is as follows: + .. code-block:: LAMMPS kim_query variable formatarg query_function queryargs @@ -442,7 +476,7 @@ individual variables of the form *prefix_I*, where *prefix* is set to the *kim_query* *variable* argument and *I* ranges from 1 to the number of returned values. The number and order of the returned values is determined by the type of query performed. (Note that the "explicit" setting of -*formatarg* is not supported by *kim_query*.) +*formatarg* is not supported by *kim_query*\ .) .. note:: @@ -452,7 +486,7 @@ by the type of query performed. (Note that the "explicit" setting of cases will generate an error. The second required argument *query_function* is the name of the -query function to be called (e.g. *get_lattice_constant_cubic*). +query function to be called (e.g. *get_lattice_constant_cubic*\ ). All following :doc:`arguments ` are parameters handed over to the web query in the format *keyword=value*\ , where *value* is always an array of one or more comma-separated items in brackets. @@ -466,7 +500,7 @@ is available on the OpenKIM webpage at All query functions require the *model* keyword, which identifies the IM whose predictions are being queried. This keyword is automatically generated by *kim_query* based on the IM set in *kim_init* and must not - be specified as an argument to *kim_query*. + be specified as an argument to *kim_query*\ . .. note:: @@ -475,11 +509,11 @@ is available on the OpenKIM webpage at used to compute this property. In cases where there are multiple methods in OpenKIM for computing a property, a *method* keyword can be provided to select the method of choice. See the - `query documentation `_ - to see which methods are available for a given *query function*\ . + `query documentation `_ + to see which methods are available for a given *query_function*\ . *kim_query* Usage Examples and Further Clarifications -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The data obtained by *kim_query* commands can be used as part of the setup or analysis phases of LAMMPS simulations. Some examples are given below. @@ -502,10 +536,12 @@ crystal. By using *kim_query*, the user is saved the trouble and possible error of tracking this value down, or of having to perform an energy minimization to find the equilibrium lattice constant. -Note that in *unit_conversion_mode* the results obtained from a -*kim_query* would need to be converted to the appropriate units system. -For example, in the above script, the lattice command would need to be -changed to: "lattice fcc ${a0}\*${_u_distance}". +.. note:: + + In *unit_conversion_mode* the results obtained from a + *kim_query* would need to be converted to the appropriate units system. + For example, in the above script, the lattice command would need to be + changed to: "lattice fcc ${a0}*${_u_distance}". **Define an equilibrium hcp crystal** @@ -524,7 +560,7 @@ changed to: "lattice fcc ${a0}\*${_u_distance}". In this case the *kim_query* returns two arguments (since the hexagonal close packed (hcp) structure has two independent lattice constants). The *formatarg* keyword "split" places the two values into -the variables *latconst_1* and *latconst_2*. (These variables are +the variables *latconst_1* and *latconst_2*\ . (These variables are created if they do not already exist.) For convenience the variables *a0* and *c0* are created in order to make the remainder of the input script more readable. @@ -555,9 +591,9 @@ potential. If no tolerance is passed a default value is used. If multiple results are returned (indicating that the tolerance is too large), *kim_query* will return an error. See the - `query documentation `_ + `query documentation `_ to see which numerical arguments and tolerances are available for a - given *query function*\ . + given *query_function*\ . **Compute defect formation energy** @@ -586,8 +622,10 @@ ideal fcc cohesive energy of the atoms in the system obtained from from these programs are queried is tracked. No other information about the nature of the query or its source is recorded. +.. _kim_param command: + Accessing KIM Model Parameters from LAMMPS (*kim_param*) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ All IMs are functional forms containing a set of parameters. The values of these parameters are typically @@ -620,7 +658,7 @@ for details). .. note:: - The *kim_param get/set* commands must be preceded by *kim_init*. + The *kim_param get/set* commands must be preceded by *kim_init*\ . The *kim_param set* command must additionally be preceded by a *kim_interactions* command (or alternatively by a *pair_style kim* and *pair_coeff* commands). The *kim_param set* command may be used wherever a *pair_coeff* command may occur. @@ -674,13 +712,13 @@ Multiple parameters can be retrieved with a single call to *kim_param get* by repeating the argument list following *get*\ . For a *set* operation, the *values* argument contains the new value(s) -for the element(s) of the parameter specified by *index_range*. For the case +for the element(s) of the parameter specified by *index_range*\ . For the case where multiple values are being set, *values* contains a set of values separated by spaces. Multiple parameters can be set with a single call to *kim_param set* by repeating the argument list following *set*\ . *kim_param* Usage Examples and Further Clarifications -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Examples of getting and setting KIM PM parameters with further clarifications are provided below. @@ -722,7 +760,7 @@ determined by the *formatarg* argument. In this case, *formatarg* is not specified and therefore the default "explicit" mode is used. (The behavior would be the same if the word -*explicit* were added after *LAM_TeSe*.) Elements 7, 8 and 9 of parameter +*explicit* were added after *LAM_TeSe*\ .) Elements 7, 8 and 9 of parameter lambda retrieved by the *get* operation are placed in the LAMMPS variables *LAM_TeTe*, *LAM_TeZn* and *LAM_TeSe*, respectively. @@ -765,7 +803,7 @@ contains the current value of lambda. In this case, the "split" mode of *formatarg* is used. The three values retrieved by the *get* operation are stored in -the three LAMMPS variables *LAM_15*, *LAM_16* and *LAM_17*. +the three LAMMPS variables *LAM_15*, *LAM_16* and *LAM_17*\ . The provided name "LAM" is used as prefix and the location in the lambda array is appended to create the variable names. @@ -797,7 +835,7 @@ potential, while *NEW_GAMMA* will contain the value 2.6. **Setting multiple scalar parameters with a single call** -.. parsed-literal:: +.. code-block:: LAMMPS kim_init SW_ZhouWardMartin_2013_CdTeZnSeHgS__MO_503261197030_002 metal ... @@ -824,6 +862,421 @@ In this case, elements 2 through 6 of the parameter *sigma* are set to the values 2.35214, 2.23869, 2.04516, 2.43269 and 1.80415 in order. +.. _kim_property command: + +Writing material properties computed in LAMMPS to standard KIM property instance format (*kim_property*) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +As explained :ref:`above`, +The OpenKIM system includes a collection of Tests (material property calculation codes), +Models (interatomic potentials), Predictions, and Reference Data (DFT or experiments). +Specifically, a KIM Test is a computation that when coupled with a KIM Model generates +the prediction of that model for a specific material property rigorously defined +by a KIM Property Definition (see the +`KIM Properties Framework `__ +for further details). A prediction of a material property for a given model is a specific +numerical realization of a property definition, referred to as a "Property +Instance." The objective of the *kim_property* command is to make it easy to +output material properties in a standardized, machine readable, format that can be easily +ingested by other programs. +Additionally, it aims to make it as easy as possible to convert a LAMMPS script that computes a +material property into a KIM Test that can then be uploaded to `openkim.org `_ + +A developer interested in creating a KIM Test using a LAMMPS script should +first determine whether a property definition that applies to their calculation +already exists in OpenKIM by searching the `properties page +`_. If none exists, it is possible to use a +locally defined property definition contained in a file until it can be +uploaded to the official repository (see below). Once one or more applicable +property definitions have been identified, the *kim_property create*, +*kim_property modify*, *kim_property remove*, and *kim_property destroy*, +commands provide an interface to create, set, modify, remove, and destroy +instances of them within a LAMMPS script. Their general syntax is as follows: + +.. code-block:: LAMMPS + + kim_property create instance_id property_id + kim_property modify instance_id key key_name key_name_key key_name_value + kim_property remove instance_id key key_name + kim_property destroy instance_id + kim_property dump file + +Here, *instance_id* is a positive integer used to uniquely identify each +property instance; (note that the results file can contain multiple property +instances). A property_id is an identifier of a +`KIM Property Definition `_, +which can be (1) a property short name, (2) the full unique ID of the property +(including the contributor and date), (3) a file name corresponding to a local +property definition file. Examples of each of these cases are shown below: + +.. code-block:: LAMMPS + + kim_property create 1 atomic-mass + kim_property create 2 cohesive-energy-relation-cubic-crystal + +.. code-block:: LAMMPS + + kim_property create 1 tag:brunnels@noreply.openkim.org,2016-05-11:property/atomic-mass + kim_property create 2 tag:staff@noreply.openkim.org,2014-04-15:property/cohesive-energy-relation-cubic-crystal + +.. code-block:: LAMMPS + + kim_property create 1 new-property.edn + kim_property create 2 /home/mary/marys-kim-properties/dissociation-energy.edn + +In the last example, "new-property.edn" and "/home/mary/marys-kim-properties/dissociation-energy.edn" are the +names of files that contain user-defined (local) property definitions. + +A KIM property instance takes the form of a "map," i.e. a set of key-value +pairs akin to Perl's hash, Python's dictionary, or Java's Hashtable. It +consists of a set of property key names, each of which is referred to here by +the *key_name* argument, that are defined as part of the relevant KIM Property +Definition and include only lowercase alphanumeric characters and dashes. The +value paired with each property key is itself a map whose possible keys are +defined as part of the `KIM Properties Framework +`__; these keys are +referred to by the *key_name_key* argument and their associated values by the +*key_name_value* argument. These values may either be scalars or arrays, +as stipulated in the property definition. + +.. note:: + + Each map assigned to a *key_name* must contain the *key_name_key* + "source-value" and an associated *key_name_value* of the appropriate + type (as defined in the relevant KIM Property Definition). For keys that are + defined as having physical units, the + "source-unit" *key_name_key* must also be given a string value recognized + by `GNU units `_. + +Once a *kim_property create* command has been given to instantiate a property +instance, maps associated with the property's keys can be edited using the +*kim_property modify* command. In using this command, the special keyword +"key" should be given, followed by the property key name and the key-value pair +in the map associated with the key that is to be set. For example, the +`atomic-mass `_ +property definition consists of two property keys named "mass" and "species." +An instance of this property could be created like so: + +.. code-block:: LAMMPS + + kim_property create 1 atomic-mass + kim_property modify 1 key species source-value Al + kim_property modify 1 key mass source-value 26.98154 + kim_property modify 1 key mass source-unit amu + +or, equivalently, + +.. code-block:: LAMMPS + + kim_property create 1 atomic-mass + kim_property modify 1 key species source-value Al & + key mass source-value 26.98154 & + source-unit amu + +*kim_property* Usage Examples and Further Clarifications +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +**Create** + +.. code-block:: LAMMPS + + kim_property create instance_id property_id + +The *kim_property create* command takes as input a property instance ID and the +property definition name, and creates an initial empty property instance data +structure. For example, + +.. code-block:: LAMMPS + + kim_property create 1 atomic-mass + kim_property create 2 cohesive-energy-relation-cubic-crystal + +creates an empty property instance of the "atomic-mass" property definition +with instance ID 1 and an empty instance of the +"cohesive-energy-relation-cubic-crystal" property with ID 2. A list of +published property definitions in OpenKIM can be found on the `properties page +`_. + +One can also provide the name of a file in the current working directory or the +path of a file containing a valid property definition. For example, + +.. code-block:: LAMMPS + + kim_property create 1 new-property.edn + +where "new-property.edn" refers to a file name containing a new property +definition that does not exist in OpenKIM. + +If the *property_id* given cannot be found in OpenKIM and no file of this name +containing a valid property definition can be found, this command will produce +an error with an appropriate message. Calling *kim_property create* with the +same instance ID multiple times will also produce an error. + +**Modify** + +.. code-block:: LAMMPS + + kim_property modify instance_id key key_name key_name_key key_name_value + +The *kim_property modify* command incrementally builds the property instance +by receiving property definition keys along with associated arguments. Each +*key_name* is associated with a map containing one or more key-value pairs (in +the form of *key_name_key*-*key_name_value* pairs). For example, + +.. code-block:: LAMMPS + + kim_property modify 1 key species source-value Al + kim_property modify 1 key mass source-value 26.98154 + kim_property modify 1 key mass source-unit amu + +where the special keyword "key" is followed by a *key_name* ("species" or +"mass" in the above) and one or more key-value pairs. These key-value pairs +may continue until either another "key" keyword is given or the end of the +command line is reached. Thus, the above could equivalently be written as + +.. code-block:: LAMMPS + + kim_property modify 1 key species source-value Al & + key mass source-value 26.98154 & + key mass source-unit amu + +As an example of modifying multiple key-value pairs belonging to the map of a +single property key, the following command modifies the map of the +"cohesive-potential-energy" property key to contain the key "source-unit" which +is assigned a value of "eV" and the key "digits" which is assigned a value of +5: + +.. code-block:: LAMMPS + + kim_property modify 2 key cohesive-potential-energy source-unit eV digits 5 + +.. note:: + + The relevant data types of the values in the map are handled + automatically based on the specification of the key in the + KIM Property Definition. In the example above, + this means that the value "eV" will automatically be interpreted as a string + while the value 5 will be interpreted as an integer. + +The values contained in maps can either be scalars, as in all of the examples +above, or arrays depending on which is stipulated in the corresponding Property +Definition. For one-dimensional arrays, a single one-based index must be +supplied that indicates which element of the array is to be modified. For +multidimensional arrays, multiple indices must be given depending on the +dimensionality of the array. + +.. note:: + + All array indexing used by *kim_property modify* is one-based, i.e. the + indices are enumerated 1, 2, 3, ... + +.. note:: + + The dimensionality of arrays are defined in the the corresponding Property + Definition. The extent of each dimension of an array can either be a + specific finite number or indefinite and determined at run time. If + an array has a fixed extent, attempting to modify an out-of-range index will + fail with an error message. + +For example, the "species" property key of the +`cohesive-energy-relation-cubic-crystal +`_ +property is a one-dimensional array that can contain any number of entries +based on the number of atoms in the unit cell of a given cubic crystal. To +assign an array containing the string "Al" four times to the "source-value" key +of the "species" property key, we can do so by issuing: + +.. code-block:: LAMMPS + + kim_property modify 2 key species source-value 1 Al + kim_property modify 2 key species source-value 2 Al + kim_property modify 2 key species source-value 3 Al + kim_property modify 2 key species source-value 4 Al + +.. note:: + + No declaration of the number of elements in this array was given; + *kim_property modify* will automatically handle memory management to allow + an arbitrary number of elements to be added to the array. + +.. note:: + + In the event that *kim_property modify* is used to set the value of an + array index without having set the values of all lesser indices, they will + be assigned default values based on the data type associated with the key in + the map: + + .. table_from_list:: + :columns: 2 + + * Data type + * Default value + * int + * 0 + * float + * 0.0 + * string + * \"\" + * file + * \"\" + + For example, doing the following: + + .. code-block:: LAMMPS + + kim_property create 2 cohesive-energy-relation-cubic-crystal + kim_property modify 2 key species source-value 4 Al + + will result in the "source-value" key in the map for the property key + "species" being assigned the array ["", "", "", "Al"]. + +For convenience, the index argument provided may refer to an inclusive range of +indices by specifying two integers separated by a colon (the first integer must +be less than or equal to the second integer, and no whitespace should be +included). Thus, the snippet above could equivalently be written: + +.. code-block:: LAMMPS + + kim_property modify 2 key species source-value 1:4 Al Al Al Al + +Calling this command with a non-positive index, e.g. +``kim_property modify 2 key species source-value 0 Al``, or an incorrect +number of input arguments, e.g. +``kim_property modify 2 key species source-value 1:4 Al Al``, will result in an +error. + +As an example of modifying multidimensional arrays, consider the "basis-atoms" +key in the `cohesive-energy-relation-cubic-crystal +`_ +property definition. This is a two-dimensional array containing the fractional +coordinates of atoms in the unit cell of the cubic crystal. In the case of, +e.g. a conventional fcc unit cell, the "source-value" key in the map associated +with this key should be assigned the following value: + +.. code-block:: LAMMPS + + [[0.0, 0.0, 0.0], + [0.5, 0.5, 0.0], + [0.5, 0.0, 0.5], + [0.0, 0.5, 0.5]] + +While each of the twelve components could be set individually, we can instead set +each row at a time using colon notation: + +.. code-block:: LAMMPS + + kim_property modify 2 key basis-atom-coordinates source-value 1 1:3 0.0 0.0 0.0 + kim_property modify 2 key basis-atom-coordinates source-value 2 1:3 0.5 0.5 0.0 + kim_property modify 2 key basis-atom-coordinates source-value 3 1:3 0.5 0.0 0.5 + kim_property modify 2 key basis-atom-coordinates source-value 4 1:3 0.0 0.5 0.5 + +Where the first index given refers to a row and the second index refers to a +column. We could, instead, choose to set each column at a time like so: + +.. code-block:: LAMMPS + + kim_property modify 2 key basis-atom-coordinates source-value 1:4 1 0.0 0.5 0.5 0.0 & + key basis-atom-coordinates source-value 1:4 2 0.0 0.5 0.0 0.5 & + key basis-atom-coordinates source-value 1:4 3 0.0 0.0 0.5 0.5 + +.. note:: + + Multiple calls of *kim_property modify* made for the same instance ID + can be combined into a single invocation, meaning the following are + both valid: + + .. code-block:: LAMMPS + + kim_property modify 2 key basis-atom-coordinates source-value 1 1:3 0.0 0.0 0.0 & + key basis-atom-coordinates source-value 2 1:3 0.5 0.5 0.0 & + key basis-atom-coordinates source-value 3 1:3 0.5 0.0 0.5 & + key basis-atom-coordinates source-value 4 1:3 0.0 0.5 0.5 + + .. code-block:: LAMMPS + + kim_property modify 2 key short-name source-value 1 fcc & + key species source-value 1:4 Al Al Al Al & + key a source-value 1:5 3.9149 4.0000 4.032 4.0817 4.1602 & + source-unit angstrom & + digits 5 & + key basis-atom-coordinates source-value 1 1:3 0.0 0.0 0.0 & + key basis-atom-coordinates source-value 2 1:3 0.5 0.5 0.0 & + key basis-atom-coordinates source-value 3 1:3 0.5 0.0 0.5 & + key basis-atom-coordinates source-value 4 1:3 0.0 0.5 0.5 + +.. note:: + + For multidimensional arrays, only one colon-separated range is allowed + in the index listing. Therefore, + + .. code-block:: LAMMPS + + kim_property modify 2 key basis-atom-coordinates 1 1:3 0.0 0.0 0.0 + + is valid but + + .. code-block:: LAMMPS + + kim_property modify 2 key basis-atom-coordinates 1:2 1:3 0.0 0.0 0.0 0.0 0.0 0.0 + + is not. + +.. note:: + + After one sets a value in a map with the *kim_property modify* command, + additional calls will overwrite the previous value. + +**Remove** + +.. code-block:: LAMMPS + + kim_property remove instance_id key key_name + +The *kim_property remove* command can be used to remove a property key from a +property instance. For example, + +.. code-block:: LAMMPS + + kim_property remove 2 key basis-atom-coordinates + +**Destroy** + +.. code-block:: LAMMPS + + kim_property destroy instance_id + +The *kim_property destroy* command deletes a previously created property +instance ID. For example, + +.. code-block:: LAMMPS + + kim_property destroy 2 + +.. note:: + + If this command is called with an instance ID that does not exist, no + error is raised. + +**Dump** + +The *kim_property dump* command can be used to write the content of all +currently defined property instances to a file: + +.. code-block:: LAMMPS + + kim_property dump file + +For example, + +.. code-block:: LAMMPS + + kim_property dump results.edn + +.. note:: + + Issuing the *kim_property dump* command clears all existing property + instances from memory. + Citation of OpenKIM IMs ----------------------- @@ -847,8 +1300,11 @@ LAMMPS is built with that package. A requirement for the KIM package, is the KIM API library that must be downloaded from the `OpenKIM website `_ and installed before LAMMPS is compiled. When installing LAMMPS from binary, the kim-api package -is a dependency that is automatically downloaded and installed. See the KIM -section of the :doc:`Packages details ` for details. +is a dependency that is automatically downloaded and installed. The *kim_query* +command requires the *libcurl* library to be installed. The *kim_property* +command requires *Python* 3.6 or later and the *kim-property* python package to +be installed. See the KIM section of the :doc:`Packages details ` +for details. Furthermore, when using *kim_commands* to run KIM SMs, any packages required by the native potential being used or other commands or fixes that it invokes diff --git a/doc/src/min_modify.rst b/doc/src/min_modify.rst index a01aab173d1ac1b70ecdd4d3ffd9b738913e13bb..e541571b7bd5c105693d0446c6813d0ecc36b735 100644 --- a/doc/src/min_modify.rst +++ b/doc/src/min_modify.rst @@ -87,8 +87,7 @@ The choice of a norm can be modified for the min styles *cg*\ , *sd*\ the 2-norm (Euclidean length) of the global force vector: .. math:: - - || \vec{F} ||_{2} = \sqrt{\vec{F}_1+ \cdots + \vec{F}_N} + || \vec{F} ||_{2} = \sqrt{\vec{F}_1^2+ \cdots + \vec{F}_N^2} The *max* norm computes the length of the 3-vector force for each atom (2-norm), and takes the maximum value of those across diff --git a/doc/src/minimize.rst b/doc/src/minimize.rst index dde5284b7ab3efd2e517c4cc243c6d2daa05085a..81234ae80e62eb124badc97b8b81fca36ba911e5 100644 --- a/doc/src/minimize.rst +++ b/doc/src/minimize.rst @@ -64,7 +64,7 @@ performed using a line search algorithm. The line search typically evaluates forces and energies several times to set new coordinates. Currently, a backtracking algorithm is used which may not be optimal in terms of the number of force evaluations performed, but appears to -be more robust than previous line searches we've tried. The +be more robust than previous line searches we have tried. The backtracking method is described in Nocedal and Wright's Numerical Optimization (Procedure 3.1 on p 41). diff --git a/doc/src/pair_adp.rst b/doc/src/pair_adp.rst index 6c4796fb85f1f8338ed1406b6c7b665a7bdd9bb2..bbfd98f677cd9676759b1ae70dd7ca199621f6c8 100644 --- a/doc/src/pair_adp.rst +++ b/doc/src/pair_adp.rst @@ -56,8 +56,10 @@ command to specify them. * The NIST WWW site at http://www.ctcms.nist.gov/potentials. Note that ADP potentials obtained from NIST must be converted into the extended DYNAMO *setfl* format discussed below. -* The OpenKIM Project at https://openkim.org/browse/models/by-type provides - ADP potentials that can be used directly in LAMMPS with the :doc:`kim_commands interface `. +* The OpenKIM Project at + `https://openkim.org/browse/models/by-type `_ + provides ADP potentials that can be used directly in LAMMPS with the + :doc:`kim_commands ` interface. ---------- diff --git a/doc/src/pair_bop.rst b/doc/src/pair_bop.rst index ce7b69f41f3cf1bf98f7b139f40fbfedb07912fd..5cd045931a31499cf53a50f276f22922a8328168 100644 --- a/doc/src/pair_bop.rst +++ b/doc/src/pair_bop.rst @@ -132,9 +132,9 @@ and Te. If your LAMMPS simulation has 4 atoms types and you want the 1st 3 to be Cd, and the 4th to be Te, you would use the following pair_coeff command: -.. parsed-literal:: +.. code-block:: LAMMPS - pair_coeff \* \* CdTe Cd Cd Cd Te + pair_coeff * * CdTe Cd Cd Cd Te The 1st 2 arguments must be \* \* so as to span all LAMMPS atom types. The first three Cd arguments map LAMMPS atom types 1,2,3 to the Cd diff --git a/doc/src/pair_class2.rst b/doc/src/pair_class2.rst index d1c673ab977f0bcc6c278d576bf2ffbc82cb9260..8131799181047ecf180d6afc58097352db134f44 100644 --- a/doc/src/pair_class2.rst +++ b/doc/src/pair_class2.rst @@ -60,18 +60,18 @@ Examples .. code-block:: LAMMPS pair_style lj/class2 10.0 - pair_coeff \* \* 100.0 2.5 - pair_coeff 1 2\* 100.0 2.5 9.0 + pair_coeff * * 100.0 2.5 + pair_coeff 1 2* 100.0 2.5 9.0 pair_style lj/class2/coul/cut 10.0 pair_style lj/class2/coul/cut 10.0 8.0 - pair_coeff \* \* 100.0 3.0 + pair_coeff * * 100.0 3.0 pair_coeff 1 1 100.0 3.5 9.0 pair_coeff 1 1 100.0 3.5 9.0 9.0 pair_style lj/class2/coul/long 10.0 pair_style lj/class2/coul/long 10.0 8.0 - pair_coeff \* \* 100.0 3.0 + pair_coeff * * 100.0 3.0 pair_coeff 1 1 100.0 3.5 9.0 Description diff --git a/doc/src/pair_coeff.rst b/doc/src/pair_coeff.rst index 26910c17469c9301e2f14d6516615c3d8fe48b79..1886ce111838d08704100bc61c48a242b7e5eb16 100644 --- a/doc/src/pair_coeff.rst +++ b/doc/src/pair_coeff.rst @@ -19,11 +19,11 @@ Examples .. code-block:: LAMMPS pair_coeff 1 2 1.0 1.0 2.5 - pair_coeff 2 \* 1.0 1.0 - pair_coeff 3\* 1\*2 1.0 1.0 2.5 - pair_coeff \* \* 1.0 1.0 - pair_coeff \* \* nialhjea 1 1 2 - pair_coeff \* 3 morse.table ENTRY1 + pair_coeff 2 * 1.0 1.0 + pair_coeff 3* 1*2 1.0 1.0 2.5 + pair_coeff * * 1.0 1.0 + pair_coeff * * nialhjea 1 1 2 + pair_coeff * 3 morse.table ENTRY1 pair_coeff 1 2 lj/cut 1.0 1.0 2.5 (for pair_style hybrid) Description @@ -55,7 +55,7 @@ pairs, then overwrite the coeffs for just the I,J = 2,3 pair: .. code-block:: LAMMPS - pair_coeff \* \* 1.0 1.0 2.5 + pair_coeff * * 1.0 1.0 2.5 pair_coeff 2 3 2.0 1.0 1.12 A line in a data file that specifies pair coefficients uses the exact diff --git a/doc/src/pair_cosine_squared.rst b/doc/src/pair_cosine_squared.rst index 4f19ddd1e4dad21d9f2e57dbb99286006f77b181..a8cf206135f1f41bfa67cb17ce87a5c4a490e8db 100644 --- a/doc/src/pair_cosine_squared.rst +++ b/doc/src/pair_cosine_squared.rst @@ -31,7 +31,7 @@ Examples .. code-block:: LAMMPS pair_style cosine/squared 3.0 - pair_coeff \* \* 1.0 1.3 + pair_coeff * * 1.0 1.3 pair_coeff 1 3 1.0 1.3 2.0 pair_coeff 1 3 1.0 1.3 wca pair_coeff 1 3 1.0 1.3 2.0 wca @@ -46,7 +46,7 @@ Style *cosine/squared* computes a potential of the form E = \begin{cases} -\epsilon& \quad r < \sigma \\ - -\epsilon\cos\left(\frac{\pi\left(r - \sigma\right)}{2\left(r_c - \sigma\right)}\right)&\quad \sigma \leq r < r_c \\ + -\epsilon\cos\left(\frac{\pi\left(r - \sigma\right)}{2\left(r_c - \sigma\right)}\right)^2&\quad \sigma \leq r < r_c \\ 0& \quad r \geq r_c \end{cases} diff --git a/doc/src/pair_cs.rst b/doc/src/pair_cs.rst index 72332a87b2172bfba2d1d73ed78d7643eb561150..9790c2defd1115efb8b4a4d2458a95ddcd277ecb 100644 --- a/doc/src/pair_cs.rst +++ b/doc/src/pair_cs.rst @@ -30,6 +30,9 @@ pair_style coul/wolf/cs command pair_style lj/cut/coul/long/cs command ======================================= +pair_style lj/class2/coul/long/cs command +========================================== + Syntax """""" @@ -37,7 +40,7 @@ Syntax pair_style style args -* style = *born/coul/dsf/cs* or *born/coul/long/cs* or *born/coul/wolf/cs* or *buck/coul/long/cs* or *coul/long/cs* or *coul/wolf/cs* or *lj/cut/coul/long/cs* +* style = *born/coul/dsf/cs* or *born/coul/long/cs* or *born/coul/wolf/cs* or *buck/coul/long/cs* or *coul/long/cs* or *coul/wolf/cs* or *lj/cut/coul/long/cs* or *lj/class2/coul/long/cs* * args = list of arguments for a particular style .. parsed-literal:: @@ -64,6 +67,9 @@ Syntax *lj/cut/coul/long/cs* args = cutoff (cutoff2) cutoff = global cutoff for LJ (and Coulombic if only 1 arg) (distance units) cutoff2 = global cutoff for Coulombic (optional) (distance units) + *lj/class2/coul/long/cs* args = cutoff (cutoff2) + cutoff = global cutoff for LJ (and Coulombic if only 1 arg) (distance units) + cutoff2 = global cutoff for Coulombic (optional) (distance units) Examples """""""" @@ -115,6 +121,7 @@ the "/cs" in the name: * :doc:`pair_style coul/long ` * :doc:`pair_style coul/wolf ` * :doc:`pair_style lj/cut/coul/long ` +* :doc:`pair_style lj/class2/coul/long ` except that they correctly treat the special case where the distance between two charged core and shell atoms in the same core/shell pair diff --git a/doc/src/pair_eam.rst b/doc/src/pair_eam.rst index 4710f8a9a1a9d2a8198979ae61f0934eff1d4ade..7384a4d54cf3f8f72b78eb844fc255719aea6e9c 100644 --- a/doc/src/pair_eam.rst +++ b/doc/src/pair_eam.rst @@ -149,6 +149,7 @@ potentials stored in DYNAMO or other formats: http://www.ctcms.nist.gov/potentials http://cst-www.nrl.navy.mil/ccm6/ap http://enpub.fulton.asu.edu/cms/potentials/main/main.htm + https://openkim.org These potentials should be usable with LAMMPS, though the alternate formats would need to be converted to the DYNAMO format used by LAMMPS @@ -156,6 +157,11 @@ and described on this page. The NIST site is maintained by Chandler Becker (cbecker at nist.gov) who is good resource for info on interatomic potentials and file formats. +The OpenKIM Project at +`https://openkim.org/browse/models/by-type `_ +provides EAM potentials that can be used directly in LAMMPS with the +:doc:`kim_commands ` interface. + ---------- For style *eam*\ , potential values are read from a file that is in the diff --git a/doc/src/pair_meam_sw_spline.rst b/doc/src/pair_meam_sw_spline.rst index ebb51795c213294ef64c700b766a3153db1fceeb..0cf4c5797558dcd0f4459e32395eac5fc615e0aa 100644 --- a/doc/src/pair_meam_sw_spline.rst +++ b/doc/src/pair_meam_sw_spline.rst @@ -75,7 +75,9 @@ If your LAMMPS simulation has 3 atoms types and they are all to be treated with this potential, you would use the following pair_coeff command: -pair_coeff \* \* Ti.meam.sw.spline Ti Ti Ti +.. code-block:: LAMMPS + + pair_coeff * * Ti.meam.sw.spline Ti Ti Ti The 1st 2 arguments must be \* \* so as to span all LAMMPS atom types. The three Ti arguments map LAMMPS atom types 1,2,3 to the Ti element diff --git a/doc/src/pair_meamc.rst b/doc/src/pair_meamc.rst index 850be5b583e88c3bc48b84ec01922818c2f36cd0..f227af8b452856dbc96796c0ab3fca5aa889b2e4 100644 --- a/doc/src/pair_meamc.rst +++ b/doc/src/pair_meamc.rst @@ -93,12 +93,13 @@ and the 4th to be C, you would use the following pair_coeff command: pair_coeff * * library.meam Si C sic.meam Si Si Si C The 1st 2 arguments must be \* \* so as to span all LAMMPS atom types. -The two filenames are for the library and parameter file respectively. -The Si and C arguments (between the file names) are the two elements -for which info will be extracted from the library file. The first -three trailing Si arguments map LAMMPS atom types 1,2,3 to the MEAM Si -element. The final C argument maps LAMMPS atom type 4 to the MEAM C -element. +The first filename is the element library file. The list of elements following +it extracts lines from the library file and assigns numeric indices to these +elements. The second filename is the alloy parameter file, which refers to +elements using the numeric indices assigned before. +The arguments after the parameter file map LAMMPS atom types to elements, i.e. +LAMMPS atom types 1,2,3 to the MEAM Si element. The final C argument maps +LAMMPS atom type 4 to the MEAM C element. If the 2nd filename is specified as NULL, no parameter file is read, which simply means the generic parameters in the library file are @@ -140,7 +141,7 @@ not required. The other numeric parameters match values in the formulas above. The value of the "elt" string is what is used in the pair_coeff command to identify which settings from the library file you wish to read in. There can be multiple entries in the library -file with the same "elt" value; LAMMPS reads the 1st matching entry it +file with the same "elt" value; LAMMPS reads the first matching entry it finds and ignores the rest. Other parameters in the MEAM library file correspond to single-element @@ -192,7 +193,7 @@ trailing comment (starting with #) which is ignored. The indices I, J, K correspond to the elements selected from the MEAM library file numbered in the order of how those elements were -selected starting from 1. Thus for the example given below +selected starting from 1. Thus for the example given before .. code-block:: LAMMPS @@ -202,11 +203,6 @@ an index of 1 would refer to Si and an index of 2 to C. The recognized keywords for the parameter file are as follows: -Ec, alpha, rho0, delta, lattce, attrac, repuls, nn2, Cmin, Cmax, rc, delr, -augt1, gsmooth_factor, re - -where - .. parsed-literal:: rc = cutoff radius for cutoff function; default = 4.0 diff --git a/doc/src/pair_nb3b_harmonic.rst b/doc/src/pair_nb3b_harmonic.rst index c4f3f3c3e8c42b08b506da2ed8373da453fb17e1..9d63df65d50769ff79f23a91dbec759a148296b3 100644 --- a/doc/src/pair_nb3b_harmonic.rst +++ b/doc/src/pair_nb3b_harmonic.rst @@ -64,7 +64,9 @@ NULL values are placeholders for atom types that will be used with other potentials. An example of a pair_coeff command for use with the *hybrid* pair style is: -pair_coeff \* \* nb3b/harmonic MgOH.nb3b.harmonic Mg O H +.. code-block:: LAMMPS + + pair_coeff * * nb3b/harmonic MgOH.nb3b.harmonic Mg O H Three-body non-bonded harmonic files in the *potentials* directory of the LAMMPS distribution have a ".nb3b.harmonic" suffix. Lines that diff --git a/doc/src/pair_polymorphic.rst b/doc/src/pair_polymorphic.rst index e06e9e78555b81a78527d6968bb8fcfc93e65d59..c0db3f10a4b4730249b7ec584bed67917a5dd524 100644 --- a/doc/src/pair_polymorphic.rst +++ b/doc/src/pair_polymorphic.rst @@ -180,9 +180,9 @@ functions for Si-C tersoff potential. If your LAMMPS simulation has 4 atoms types and you want the 1st 3 to be Si, and the 4th to be C, you would use the following pair_coeff command: -.. parsed-literal:: +.. code-block:: LAMMPS - pair_coeff \* \* SiC_tersoff.poly Si Si Si C + pair_coeff * * SiC_tersoff.poly Si Si Si C The 1st 2 arguments must be \* \* so as to span all LAMMPS atom types. The first three Si arguments map LAMMPS atom types 1,2,3 to the diff --git a/doc/src/pair_python.rst b/doc/src/pair_python.rst index fa76d4c16c32d4ad96b91593c16cf6192dcb0d2c..e654a6025f9bd6119e5126d93b75e7d86b1cf380 100644 --- a/doc/src/pair_python.rst +++ b/doc/src/pair_python.rst @@ -113,8 +113,8 @@ which the parameters epsilon and sigma are both 1.0: class LJCutMelt(LAMMPSPairPotential): def __init__(self): super(LJCutMelt,self).__init__() - # set coeffs: 48\*eps\*sig\*\*12, 24\*eps\*sig\*\*6, - # 4\*eps\*sig\*\*12, 4\*eps\*sig\*\*6 + # set coeffs: 48*eps*sig**12, 24*eps*sig**6, + # 4*eps*sig**12, 4*eps*sig**6 self.units = 'lj' self.coeff = {'lj' : {'lj' : (48.0,24.0,4.0,4.0)}} @@ -137,18 +137,18 @@ the *LJCutMelt* example, here are the two functions: def compute_force(self,rsq,itype,jtype): coeff = self.coeff[self.pmap[itype]][self.pmap[jtype]] r2inv = 1.0/rsq - r6inv = r2inv\*r2inv\*r2inv + r6inv = r2inv*r2inv*r2inv lj1 = coeff[0] lj2 = coeff[1] - return (r6inv \* (lj1\*r6inv - lj2))\*r2inv + return (r6inv * (lj1*r6inv - lj2))*r2inv def compute_energy(self,rsq,itype,jtype): coeff = self.coeff[self.pmap[itype]][self.pmap[jtype]] r2inv = 1.0/rsq - r6inv = r2inv\*r2inv\*r2inv + r6inv = r2inv*r2inv*r2inv lj3 = coeff[2] lj4 = coeff[3] - return (r6inv \* (lj3\*r6inv - lj4)) + return (r6inv * (lj3*r6inv - lj4)) .. note:: diff --git a/doc/src/pair_spin_magelec.rst b/doc/src/pair_spin_magelec.rst index a220299b07dec5e3ecd4727969f96997aee37063..e4d6b81b8a28f26f390c6f210a799cfb417b218b 100644 --- a/doc/src/pair_spin_magelec.rst +++ b/doc/src/pair_spin_magelec.rst @@ -18,7 +18,7 @@ Examples .. code-block:: LAMMPS pair_style spin/magelec 4.5 - pair_coeff \* \* magelec 4.5 0.00109 1.0 1.0 1.0 + pair_coeff * * magelec 4.5 0.00109 1.0 1.0 1.0 Description """"""""""" diff --git a/doc/src/pair_style.rst b/doc/src/pair_style.rst index 5b5e6e6fe8d224b4015c55f5677f86a9b9b24ec2..48dd2dde0fcc5ff5fd3b371d039dbbf23f901513 100644 --- a/doc/src/pair_style.rst +++ b/doc/src/pair_style.rst @@ -111,41 +111,41 @@ accelerated styles exist. * :doc:`born ` - Born-Mayer-Huggins potential * :doc:`born/coul/dsf ` - Born with damped-shifted-force model * :doc:`born/coul/dsf/cs ` - Born with damped-shifted-force and core/shell model -* :doc:`born/coul/long ` - Born with long-range Coulombics -* :doc:`born/coul/long/cs ` - Born with long-range Coulombics and core/shell -* :doc:`born/coul/msm ` - Born with long-range MSM Coulombics -* :doc:`born/coul/wolf ` - Born with Wolf potential for Coulombics -* :doc:`born/coul/wolf/cs ` - Born with Wolf potential for Coulombics and core/shell model +* :doc:`born/coul/long ` - Born with long-range Coulomb +* :doc:`born/coul/long/cs ` - Born with long-range Coulomb and core/shell +* :doc:`born/coul/msm ` - Born with long-range MSM Coulomb +* :doc:`born/coul/wolf ` - Born with Wolf potential for Coulomb +* :doc:`born/coul/wolf/cs ` - Born with Wolf potential for Coulomb and core/shell model * :doc:`brownian ` - Brownian potential for Fast Lubrication Dynamics * :doc:`brownian/poly ` - Brownian potential for Fast Lubrication Dynamics with polydispersity * :doc:`buck ` - Buckingham potential * :doc:`buck/coul/cut ` - Buckingham with cutoff Coulomb -* :doc:`buck/coul/long ` - Buckingham with long-range Coulombics -* :doc:`buck/coul/long/cs ` - Buckingham with long-range Coulombics and core/shell -* :doc:`buck/coul/msm ` - Buckingham with long-range MSM Coulombics -* :doc:`buck/long/coul/long ` - long-range Buckingham with long-range Coulombics +* :doc:`buck/coul/long ` - Buckingham with long-range Coulomb +* :doc:`buck/coul/long/cs ` - Buckingham with long-range Coulomb and core/shell +* :doc:`buck/coul/msm ` - Buckingham with long-range MSM Coulomb +* :doc:`buck/long/coul/long ` - long-range Buckingham with long-range Coulomb * :doc:`buck/mdf ` - Buckingham with a taper function * :doc:`buck6d/coul/gauss/dsf ` - dispersion-damped Buckingham with damped-shift-force model -* :doc:`buck6d/coul/gauss/long ` - dispersion-damped Buckingham with long-range Coulombics +* :doc:`buck6d/coul/gauss/long ` - dispersion-damped Buckingham with long-range Coulomb * :doc:`colloid ` - integrated colloidal potential * :doc:`comb ` - charge-optimized many-body (COMB) potential * :doc:`comb3 ` - charge-optimized many-body (COMB3) potential * :doc:`cosine/squared ` - Cooke-Kremer-Deserno membrane model potential -* :doc:`coul/cut ` - cutoff Coulombic potential -* :doc:`coul/cut/soft ` - Coulombic potential with a soft core -* :doc:`coul/debye ` - cutoff Coulombic potential with Debye screening +* :doc:`coul/cut ` - cutoff Coulomb potential +* :doc:`coul/cut/soft ` - Coulomb potential with a soft core +* :doc:`coul/debye ` - cutoff Coulomb potential with Debye screening * :doc:`coul/diel ` - Coulomb potential with dielectric permittivity -* :doc:`coul/dsf ` - Coulombics with damped-shifted-force model -* :doc:`coul/long ` - long-range Coulombic potential -* :doc:`coul/long/cs ` - long-range Coulombic potential and core/shell -* :doc:`coul/long/soft ` - long-range Coulombic potential with a soft core -* :doc:`coul/msm ` - long-range MSM Coulombics -* :doc:`coul/slater/cut ` - smeared out Coulombics -* :doc:`coul/slater/long ` - long-range smeared out Coulombics -* :doc:`coul/shield ` - Coulombics for boron nitride for use with :doc:`ilp/graphene/hbn ` potential -* :doc:`coul/streitz ` - Coulombics via Streitz/Mintmire Slater orbitals -* :doc:`coul/wolf ` - Coulombics via Wolf potential -* :doc:`coul/wolf/cs ` - ditto with core/shell adjustments +* :doc:`coul/dsf ` - Coulomb with damped-shifted-force model +* :doc:`coul/long ` - long-range Coulomb potential +* :doc:`coul/long/cs ` - long-range Coulomb potential and core/shell +* :doc:`coul/long/soft ` - long-range Coulomb potential with a soft core +* :doc:`coul/msm ` - long-range MSM Coulomb +* :doc:`coul/slater/cut ` - smeared out Coulomb +* :doc:`coul/slater/long ` - long-range smeared out Coulomb +* :doc:`coul/shield ` - Coulomb for boron nitride for use with :doc:`ilp/graphene/hbn ` potential +* :doc:`coul/streitz ` - Coulomb via Streitz/Mintmire Slater orbitals +* :doc:`coul/wolf ` - Coulomb via Wolf potential +* :doc:`coul/wolf/cs ` - Coulomb via Wolf potential with core/shell adjustments * :doc:`dpd ` - dissipative particle dynamics (DPD) * :doc:`dpd/fdt ` - DPD for constant temperature and pressure * :doc:`dpd/fdt/energy ` - DPD for constant energy and enthalpy @@ -189,44 +189,45 @@ accelerated styles exist. * :doc:`lj/charmm/coul/charmm/implicit ` - CHARMM for implicit solvent * :doc:`lj/charmm/coul/long ` - CHARMM with long-range Coulomb * :doc:`lj/charmm/coul/long/soft ` - CHARMM with long-range Coulomb and a soft core -* :doc:`lj/charmm/coul/msm ` - CHARMM with long-range MSM Coulombics +* :doc:`lj/charmm/coul/msm ` - CHARMM with long-range MSM Coulomb * :doc:`lj/charmmfsw/coul/charmmfsh ` - CHARMM with force switching and shifting -* :doc:`lj/charmmfsw/coul/long ` - CHARMM with force switching and long-rnage Coulombics -* :doc:`lj/class2 ` - COMPASS (class 2) force field with no Coulomb +* :doc:`lj/charmmfsw/coul/long ` - CHARMM with force switching and long-rnage Coulomb +* :doc:`lj/class2 ` - COMPASS (class 2) force field without Coulomb * :doc:`lj/class2/coul/cut ` - COMPASS with cutoff Coulomb * :doc:`lj/class2/coul/cut/soft ` - COMPASS with cutoff Coulomb with a soft core * :doc:`lj/class2/coul/long ` - COMPASS with long-range Coulomb +* :doc:`lj/class2/coul/long/cs ` - COMPASS with long-range Coulomb with core/shell adjustments * :doc:`lj/class2/coul/long/soft ` - COMPASS with long-range Coulomb with a soft core * :doc:`lj/class2/soft ` - COMPASS (class 2) force field with no Coulomb with a soft core * :doc:`lj/cubic ` - LJ with cubic after inflection point -* :doc:`lj/cut ` - cutoff Lennard-Jones potential with no Coulomb +* :doc:`lj/cut ` - cutoff Lennard-Jones potential without Coulomb * :doc:`lj/cut/coul/cut ` - LJ with cutoff Coulomb * :doc:`lj/cut/coul/cut/soft ` - LJ with cutoff Coulomb with a soft core * :doc:`lj/cut/coul/debye ` - LJ with Debye screening added to Coulomb -* :doc:`lj/cut/coul/dsf ` - LJ with Coulombics via damped shifted forces -* :doc:`lj/cut/coul/long ` - LJ with long-range Coulombics -* :doc:`lj/cut/coul/long/cs ` - ditto with core/shell adjustments -* :doc:`lj/cut/coul/long/soft ` - LJ with long-range Coulombics with a soft core -* :doc:`lj/cut/coul/msm ` - LJ with long-range MSM Coulombics -* :doc:`lj/cut/coul/wolf ` - LJ with Coulombics via Wolf potential +* :doc:`lj/cut/coul/dsf ` - LJ with Coulomb via damped shifted forces +* :doc:`lj/cut/coul/long ` - LJ with long-range Coulomb +* :doc:`lj/cut/coul/long/cs ` - LJ with long-range Coulomb with core/shell adjustments +* :doc:`lj/cut/coul/long/soft ` - LJ with long-range Coulomb with a soft core +* :doc:`lj/cut/coul/msm ` - LJ with long-range MSM Coulomb +* :doc:`lj/cut/coul/wolf ` - LJ with Coulomb via Wolf potential * :doc:`lj/cut/dipole/cut ` - point dipoles with cutoff * :doc:`lj/cut/dipole/long ` - point dipoles with long-range Ewald * :doc:`lj/cut/soft ` - LJ with a soft core -* :doc:`lj/cut/thole/long ` - LJ with Coulombics with thole damping +* :doc:`lj/cut/thole/long ` - LJ with Coulomb with thole damping * :doc:`lj/cut/tip4p/cut ` - LJ with cutoff Coulomb for TIP4P water * :doc:`lj/cut/tip4p/long ` - LJ with long-range Coulomb for TIP4P water * :doc:`lj/cut/tip4p/long/soft ` - LJ with cutoff Coulomb for TIP4P water with a soft core * :doc:`lj/expand ` - Lennard-Jones for variable size particles -* :doc:`lj/expand/coul/long ` - Lennard-Jones for variable size particles with long-range Coulombics +* :doc:`lj/expand/coul/long ` - Lennard-Jones for variable size particles with long-range Coulomb * :doc:`lj/gromacs ` - GROMACS-style Lennard-Jones potential -* :doc:`lj/gromacs/coul/gromacs ` - GROMACS-style LJ and Coulombic potential -* :doc:`lj/long/coul/long ` - long-range LJ and long-range Coulombics +* :doc:`lj/gromacs/coul/gromacs ` - GROMACS-style LJ and Coulomb potential +* :doc:`lj/long/coul/long ` - long-range LJ and long-range Coulomb * :doc:`lj/long/dipole/long ` - long-range LJ and long-range point dipoles -* :doc:`lj/long/tip4p/long ` - long-range LJ and long-range Coulombics for TIP4P water +* :doc:`lj/long/tip4p/long ` - long-range LJ and long-range Coulomb for TIP4P water * :doc:`lj/mdf ` - LJ potential with a taper function * :doc:`lj/sdk ` - LJ for SDK coarse-graining -* :doc:`lj/sdk/coul/long ` - LJ for SDK coarse-graining with long-range Coulombics -* :doc:`lj/sdk/coul/msm ` - LJ for SDK coarse-graining with long-range Coulombics via MSM +* :doc:`lj/sdk/coul/long ` - LJ for SDK coarse-graining with long-range Coulomb +* :doc:`lj/sdk/coul/msm ` - LJ for SDK coarse-graining with long-range Coulomb via MSM * :doc:`lj/sf/dipole/sf ` - LJ with dipole interaction with shifted forces * :doc:`lj/smooth ` - smoothed Lennard-Jones potential * :doc:`lj/smooth/linear ` - linear smoothed LJ potential @@ -255,7 +256,7 @@ accelerated styles exist. * :doc:`nb3b/harmonic ` - non-bonded 3-body harmonic potential * :doc:`nm/cut ` - N-M potential * :doc:`nm/cut/coul/cut ` - N-M potential with cutoff Coulomb -* :doc:`nm/cut/coul/long ` - N-M potential with long-range Coulombics +* :doc:`nm/cut/coul/long ` - N-M potential with long-range Coulomb * :doc:`oxdna/coaxstk ` - * :doc:`oxdna/excv ` - * :doc:`oxdna/hbond ` - @@ -315,7 +316,7 @@ accelerated styles exist. * :doc:`tersoff/zbl ` - Tersoff/ZBL 3-body potential * :doc:`thole ` - Coulomb interactions with thole damping * :doc:`tip4p/cut ` - Coulomb for TIP4P water w/out LJ -* :doc:`tip4p/long ` - long-range Coulombics for TIP4P water w/out LJ +* :doc:`tip4p/long ` - long-range Coulomb for TIP4P water w/out LJ * :doc:`tip4p/long/soft ` - * :doc:`tri/lj ` - LJ potential between triangles * :doc:`ufm ` - diff --git a/doc/utils/check-packages.py b/doc/utils/check-packages.py index 99968f9a2204d2077d05773192c5ce13c279f68a..52d527fd9510c58e173e8ee0f05cfa92c6e1f859 100755 --- a/doc/utils/check-packages.py +++ b/doc/utils/check-packages.py @@ -45,7 +45,7 @@ usrpkg = [] # folder, and is not called 'MAKE' is a package for d in pkgdirs: - pkg = dirs.match(d)[1] + pkg = dirs.match(d).group(1) if not os.path.isdir(os.path.join(src,pkg)): continue if pkg in ['DEPEND','MAKE','STUBS']: continue if user.match(pkg): diff --git a/doc/utils/sphinx-config/false_positives.txt b/doc/utils/sphinx-config/false_positives.txt index e63cf1f65ac5edf426059db459e4c5876bf60b0a..c19349f3e67c77c95947d420a987dbdea98f25f1 100644 --- a/doc/utils/sphinx-config/false_positives.txt +++ b/doc/utils/sphinx-config/false_positives.txt @@ -170,6 +170,7 @@ ba Babadi Babaei backcolor +backends Baczewski Bagi Bagnold @@ -262,6 +263,7 @@ Boltzman BondAngle BondBond bondchk +BondingIDs bondmax bondtype Bonet @@ -367,6 +369,8 @@ Chemnitz Cheng Chenoweth chiral +ChiralIDs +chiralIDs chirality Cho chris @@ -433,6 +437,7 @@ cond conda Conda Condens +Connor conf config configfile @@ -487,6 +492,7 @@ cstdlib cstring cstyle csvr +ctrl Ctypes ctypes cuda @@ -564,6 +570,7 @@ defn deformable del delaystep +DeleteIDs deleteIDs Dellago delocalization @@ -578,6 +585,7 @@ dephasing dequidt Dequidt der +dereference derekt Derjagin Derjaguin @@ -713,10 +721,12 @@ Ec ecoul ecp Ecut +EdgeIDs edgeIDs edihed edim edip +edn edpd eDPD edu @@ -1049,6 +1059,7 @@ gpu gpuID gpus gradV +GradVidottan graining Graining Grama @@ -1102,6 +1113,7 @@ Harting Hartree Hartrees Hasan +Hashtable Haswell Haugk Hayoun @@ -1155,6 +1167,7 @@ Houlle howto Howto Hoyt +Hs hstyle html hTST @@ -1363,6 +1376,7 @@ Kai Kalia Kamberaj Kapfer +Karls Karlsruhe Karniadakis Karplus @@ -1385,6 +1399,7 @@ KDevelop ke KE Keblinski +Keefe keflag Keir Kelchner @@ -1407,6 +1422,7 @@ Klapp Kloss kmax Kmax +KMP Knizhnik knl Kofke @@ -1575,6 +1591,7 @@ lmpqst lmpsdata Lmpsdata lmptype +LMT ln localTemp localvectors @@ -1590,6 +1607,7 @@ Lookups LoopVar Lorant lorenz +Los lossless lossy Lozovik @@ -1651,6 +1669,8 @@ Marroquin Marsaglia Marseille Martyna +mary +marys Masaglia Mashayak Massimilliano @@ -1717,6 +1737,7 @@ Melchor Meloni Melrose Mem +mem memalign MEMALIGN membered @@ -2087,7 +2108,6 @@ nvc nvcc nve nvidia -Nvidia nvk nvt Nwait @@ -2652,6 +2672,8 @@ Shi Shiga Shinoda shlib +SHM +shm shockvel si SiC @@ -2819,6 +2841,7 @@ Synechococcus sys sysdim Syst +systemd Sz Tabbernor tabinner @@ -2940,6 +2963,7 @@ traceless tradeoff traj Tranchida +transactional transferability translational Translational @@ -3171,6 +3195,7 @@ Westview wget Whelan whitesmoke +whitespace Wi Wicaksono Wijk @@ -3238,6 +3263,7 @@ xzhou yaff YAFF Yamada +Yaser Yazdani Ybar ybox diff --git a/examples/COUPLE/simple/CMakeLists.txt b/examples/COUPLE/simple/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..4112eaa4e769752054e2c20c1e5d367460ea6ccc --- /dev/null +++ b/examples/COUPLE/simple/CMakeLists.txt @@ -0,0 +1,17 @@ +cmake_minimum_required(VERSION 3.10) +project(simple CXX) + +set(LAMMPS_SRC_DIRECTORY "" CACHE PATH "Path for lammps source") +if(NOT LAMMPS_SRC_DIRECTORY STREQUAL "" AND EXISTS ${LAMMPS_SRC_DIRECTORY}/cmake/CMakeLists.txt) + option(BUILD_LIB "Build LAMMPS library" ON) + add_subdirectory(${LAMMPS_SRC_DIRECTORY}/cmake lammps) +else() + find_package(LAMMPS REQUIRED) +endif() + +add_executable(simpleCC simple.cpp) +target_link_libraries(simpleCC LAMMPS::lammps) + +enable_language(C) +add_executable(simpleC simple.c) +target_link_libraries(simpleC LAMMPS::lammps) diff --git a/examples/COUPLE/simple/simple.c b/examples/COUPLE/simple/simple.c index ad603e09a6b637385d2edbed5a1b49ce36d5530d..f4623470b74be7f0baec19407c3950d8d2e2e0f2 100644 --- a/examples/COUPLE/simple/simple.c +++ b/examples/COUPLE/simple/simple.c @@ -23,7 +23,7 @@ #include "stdlib.h" #include "string.h" #include "mpi.h" -#include "library.h" /* this is a LAMMPS include file */ +#include "lammps/library.h" /* this is a LAMMPS include file */ int main(int narg, char **arg) { diff --git a/examples/SPIN/bfo/in.spin.bfo b/examples/SPIN/bfo/in.spin.bfo index b97f7e2d619766f37298e1022943a8d646fa6ea9..47ba535ab6069626651defa38bce7b21ed55b4cc 100644 --- a/examples/SPIN/bfo/in.spin.bfo +++ b/examples/SPIN/bfo/in.spin.bfo @@ -27,7 +27,8 @@ pair_coeff * * spin/dmi dmi 4.5 0.00005 1.0 1.0 1.0 neighbor 0.1 bin neigh_modify every 10 check yes delay 20 -fix 1 all precession/spin anisotropy 0.0000033 0.0 0.0 1.0 +fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 anisotropy 0.00033 0.0 0.0 1.0 +fix_modify 1 energy yes fix 2 all langevin/spin 0.0 0.1 21 fix 3 all nve/spin lattice frozen @@ -43,9 +44,8 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -#thermo_style custom step time v_magnorm v_emag temp etotal -thermo_style custom step time v_magnorm pe ke v_emag temp etotal -thermo 10 +thermo_style custom step time v_magnorm pe v_emag temp etotal +thermo 50 compute outsp all property/atom spx spy spz sp fmx fmy fmz dump 1 all custom 100 dump_bfo.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] diff --git a/examples/SPIN/bfo/log.14Apr20.spin.bfo.g++.1 b/examples/SPIN/bfo/log.14Apr20.spin.bfo.g++.1 new file mode 100644 index 0000000000000000000000000000000000000000..c5ad4e70ac3e7c55bd3aee60e5549ef987a0c2c9 --- /dev/null +++ b/examples/SPIN/bfo/log.14Apr20.spin.bfo.g++.1 @@ -0,0 +1,129 @@ +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# layer sc iron atoms (in the [001] plane) in bismuth oxide + +units metal +atom_style spin +dimension 3 +boundary p p f + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice sc 3.96 +Lattice spacing in x,y,z = 3.96 3.96 3.96 +region box block 0.0 34.0 0.0 34.0 0.0 5.0 +create_box 1 box +Created orthogonal box = (0 0 0) to (134.64 134.64 19.8) + 1 by 1 by 1 MPI processor grid +create_atoms 1 box +Created 5780 atoms + create_atoms CPU = 0.00263691 secs + +# setting mass, mag. moments, and interactions for bfo + +mass 1 1.0 +set group all spin/random 11 2.50 + 5780 settings made for spin/random + +#pair_style hybrid/overlay spin/exchange 6.0 spin/magelec 4.5 +pair_style hybrid/overlay spin/exchange 6.0 spin/magelec 4.5 spin/dmi 4.5 +pair_coeff * * spin/exchange exchange 6.0 -0.01575 0.0 1.965 +pair_coeff * * spin/magelec magelec 4.5 0.000109 1.0 1.0 1.0 +pair_coeff * * spin/dmi dmi 4.5 0.00005 1.0 1.0 1.0 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 anisotropy 0.00033 0.0 0.0 1.0 +fix_modify 1 energy yes +fix 2 all langevin/spin 0.0 0.1 21 +fix 3 all nve/spin lattice frozen + +timestep 0.0002 + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable magz equal c_out_mag[3] +variable magnorm equal c_out_mag[4] +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_magnorm pe v_emag temp etotal +thermo 50 + +compute outsp all property/atom spx spy spz sp fmx fmy fmz +dump 1 all custom 100 dump_bfo.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] + +run 500 +Neighbor list info ... + update every 10 steps, delay 20 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 6.1 + ghost atom cutoff = 6.1 + binsize = 3.05, bins = 45 45 7 + 3 neighbor lists, perpetual/occasional/extra = 3 0 0 + (1) pair spin/exchange, perpetual + attributes: full, newton on + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard + (2) pair spin/magelec, perpetual, copy from (1) + attributes: full, newton on + pair build: copy + stencil: none + bin: none + (3) pair spin/dmi, perpetual, copy from (1) + attributes: full, newton on + pair build: copy + stencil: none + bin: none +Per MPI rank memory allocation (min/avg/max) = 8.154 | 8.154 | 8.154 Mbytes +Step Time v_magnorm PotEng v_emag Temp TotEng + 0 0 0.010071723 -0.69133656 -0.69133656 0 -0.69133656 + 50 0.01 0.0098309931 -1.3200811 -1.3200811 0 -1.3200811 + 100 0.02 0.0095903925 -1.9446727 -1.9446727 0 -1.9446727 + 150 0.03 0.0093489139 -2.5653446 -2.5653446 0 -2.5653446 + 200 0.04 0.0091051407 -3.1824298 -3.1824298 0 -3.1824298 + 250 0.05 0.0088575394 -3.7962506 -3.7962506 0 -3.7962506 + 300 0.06 0.0086053184 -4.4070501 -4.4070501 0 -4.4070501 + 350 0.07 0.0083490959 -5.0149813 -5.0149813 0 -5.0149813 + 400 0.08 0.0080907742 -5.6201417 -5.6201417 0 -5.6201417 + 450 0.09 0.0078327124 -6.222622 -6.222622 0 -6.222622 + 500 0.1 0.0075768488 -6.8225497 -6.8225497 0 -6.8225497 +Loop time of 12.9141 on 1 procs for 500 steps with 5780 atoms + +Performance: 0.669 ns/day, 35.873 hours/ns, 38.717 timesteps/s +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 3.1844 | 3.1844 | 3.1844 | 0.0 | 24.66 +Neigh | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.012542 | 0.012542 | 0.012542 | 0.0 | 0.10 +Output | 0.048014 | 0.048014 | 0.048014 | 0.0 | 0.37 +Modify | 9.6569 | 9.6569 | 9.6569 | 0.0 | 74.78 +Other | | 0.01233 | | | 0.10 + +Nlocal: 5780 ave 5780 max 5780 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 1065 ave 1065 max 1065 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +FullNghs: 92480 ave 92480 max 92480 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 92480 +Ave neighs/atom = 16 +Neighbor list builds = 0 +Dangerous builds = 0 + +Please see the log.cite file for references relevant to this simulation + +Total wall time: 0:00:12 diff --git a/examples/SPIN/bfo/log.14Apr20.spin.bfo.g++.4 b/examples/SPIN/bfo/log.14Apr20.spin.bfo.g++.4 new file mode 100644 index 0000000000000000000000000000000000000000..655bbaef8e5c71f6974048a12e2752fca87714a2 --- /dev/null +++ b/examples/SPIN/bfo/log.14Apr20.spin.bfo.g++.4 @@ -0,0 +1,129 @@ +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# layer sc iron atoms (in the [001] plane) in bismuth oxide + +units metal +atom_style spin +dimension 3 +boundary p p f + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice sc 3.96 +Lattice spacing in x,y,z = 3.96 3.96 3.96 +region box block 0.0 34.0 0.0 34.0 0.0 5.0 +create_box 1 box +Created orthogonal box = (0 0 0) to (134.64 134.64 19.8) + 2 by 2 by 1 MPI processor grid +create_atoms 1 box +Created 5780 atoms + create_atoms CPU = 0.000847816 secs + +# setting mass, mag. moments, and interactions for bfo + +mass 1 1.0 +set group all spin/random 11 2.50 + 5780 settings made for spin/random + +#pair_style hybrid/overlay spin/exchange 6.0 spin/magelec 4.5 +pair_style hybrid/overlay spin/exchange 6.0 spin/magelec 4.5 spin/dmi 4.5 +pair_coeff * * spin/exchange exchange 6.0 -0.01575 0.0 1.965 +pair_coeff * * spin/magelec magelec 4.5 0.000109 1.0 1.0 1.0 +pair_coeff * * spin/dmi dmi 4.5 0.00005 1.0 1.0 1.0 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 anisotropy 0.00033 0.0 0.0 1.0 +fix_modify 1 energy yes +fix 2 all langevin/spin 0.0 0.1 21 +fix 3 all nve/spin lattice frozen + +timestep 0.0002 + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable magz equal c_out_mag[3] +variable magnorm equal c_out_mag[4] +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_magnorm pe v_emag temp etotal +thermo 50 + +compute outsp all property/atom spx spy spz sp fmx fmy fmz +dump 1 all custom 100 dump_bfo.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] + +run 500 +Neighbor list info ... + update every 10 steps, delay 20 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 6.1 + ghost atom cutoff = 6.1 + binsize = 3.05, bins = 45 45 7 + 3 neighbor lists, perpetual/occasional/extra = 3 0 0 + (1) pair spin/exchange, perpetual + attributes: full, newton on + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard + (2) pair spin/magelec, perpetual, copy from (1) + attributes: full, newton on + pair build: copy + stencil: none + bin: none + (3) pair spin/dmi, perpetual, copy from (1) + attributes: full, newton on + pair build: copy + stencil: none + bin: none +Per MPI rank memory allocation (min/avg/max) = 7.744 | 7.744 | 7.744 Mbytes +Step Time v_magnorm PotEng v_emag Temp TotEng + 0 0 0.010071723 -0.69133656 -0.69133656 0 -0.69133656 + 50 0.01 0.009830993 -1.3200811 -1.3200811 0 -1.3200811 + 100 0.02 0.0095903924 -1.9446727 -1.9446727 0 -1.9446727 + 150 0.03 0.0093489138 -2.5653446 -2.5653446 0 -2.5653446 + 200 0.04 0.0091051405 -3.1824298 -3.1824298 0 -3.1824298 + 250 0.05 0.0088575392 -3.7962506 -3.7962506 0 -3.7962506 + 300 0.06 0.0086053183 -4.4070501 -4.4070501 0 -4.4070501 + 350 0.07 0.0083490958 -5.0149813 -5.0149813 0 -5.0149813 + 400 0.08 0.008090774 -5.6201417 -5.6201417 0 -5.6201417 + 450 0.09 0.0078327123 -6.222622 -6.222622 0 -6.222622 + 500 0.1 0.0075768487 -6.8225497 -6.8225497 0 -6.8225497 +Loop time of 3.92182 on 4 procs for 500 steps with 5780 atoms + +Performance: 2.203 ns/day, 10.894 hours/ns, 127.492 timesteps/s +99.9% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0.80837 | 0.82745 | 0.8485 | 1.6 | 21.10 +Neigh | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.045699 | 0.067487 | 0.090503 | 6.1 | 1.72 +Output | 0.01387 | 0.0139 | 0.01394 | 0.0 | 0.35 +Modify | 3.0065 | 3.0105 | 3.0138 | 0.2 | 76.76 +Other | | 0.002516 | | | 0.06 + +Nlocal: 1445 ave 1445 max 1445 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +Nghost: 555 ave 555 max 555 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +Neighs: 0 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +FullNghs: 23120 ave 23120 max 23120 min +Histogram: 4 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 92480 +Ave neighs/atom = 16 +Neighbor list builds = 0 +Dangerous builds = 0 + +Please see the log.cite file for references relevant to this simulation + +Total wall time: 0:00:03 diff --git a/examples/SPIN/bfo/log.19Nov19.spin.bfo.g++.1 b/examples/SPIN/bfo/log.19Nov19.spin.bfo.g++.1 deleted file mode 100644 index b14210e9d0cddbeb1cb15e8aee3d5520b334cf36..0000000000000000000000000000000000000000 --- a/examples/SPIN/bfo/log.19Nov19.spin.bfo.g++.1 +++ /dev/null @@ -1,167 +0,0 @@ -LAMMPS (30 Oct 2019) -# layer sc iron atoms (in the [001] plane) in bismuth oxide - -units metal -atom_style spin -dimension 3 -boundary p p f - -# necessary for the serial algorithm (sametag) -atom_modify map array - -lattice sc 3.96 -Lattice spacing in x,y,z = 3.96 3.96 3.96 -region box block 0.0 34.0 0.0 34.0 0.0 5.0 -create_box 1 box -Created orthogonal box = (0 0 0) to (134.64 134.64 19.8) - 1 by 1 by 1 MPI processor grid -create_atoms 1 box -Created 5780 atoms - create_atoms CPU = 0.00226784 secs - -# setting mass, mag. moments, and interactions for bfo - -mass 1 1.0 -set group all spin/random 11 2.50 - 5780 settings made for spin/random - -#pair_style hybrid/overlay spin/exchange 6.0 spin/magelec 4.5 -pair_style hybrid/overlay spin/exchange 6.0 spin/magelec 4.5 spin/dmi 4.5 -pair_coeff * * spin/exchange exchange 6.0 -0.01575 0.0 1.965 -pair_coeff * * spin/magelec magelec 4.5 0.000109 1.0 1.0 1.0 -pair_coeff * * spin/dmi dmi 4.5 0.00005 1.0 1.0 1.0 - -neighbor 0.1 bin -neigh_modify every 10 check yes delay 20 - -fix 1 all precession/spin anisotropy 0.0000033 0.0 0.0 1.0 -fix 2 all langevin/spin 0.0 0.1 21 -fix 3 all nve/spin lattice frozen - -timestep 0.0002 - -compute out_mag all spin -compute out_pe all pe -compute out_ke all ke -compute out_temp all temp - -variable magz equal c_out_mag[3] -variable magnorm equal c_out_mag[4] -variable emag equal c_out_mag[5] -variable tmag equal c_out_mag[6] - -#thermo_style custom step time v_magnorm v_emag temp etotal -thermo_style custom step time v_magnorm pe ke v_emag temp etotal -thermo 10 - -compute outsp all property/atom spx spy spz sp fmx fmy fmz -dump 1 all custom 100 dump_bfo.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] - -run 500 -Neighbor list info ... - update every 10 steps, delay 20 steps, check yes - max neighbors/atom: 2000, page size: 100000 - master list distance cutoff = 6.1 - ghost atom cutoff = 6.1 - binsize = 3.05, bins = 45 45 7 - 3 neighbor lists, perpetual/occasional/extra = 3 0 0 - (1) pair spin/exchange, perpetual - attributes: full, newton on - pair build: full/bin/atomonly - stencil: full/bin/3d - bin: standard - (2) pair spin/magelec, perpetual, copy from (1) - attributes: full, newton on - pair build: copy - stencil: none - bin: none - (3) pair spin/dmi, perpetual, copy from (1) - attributes: full, newton on - pair build: copy - stencil: none - bin: none -Per MPI rank memory allocation (min/avg/max) = 8.154 | 8.154 | 8.154 Mbytes -Step Time v_magnorm PotEng KinEng v_emag Temp TotEng - 0 0 0.010071723 -0.059343109 0 -0.13132609 0 -0.059343109 - 10 0.002 0.01003044 -0.18537022 0 -0.38338861 0 -0.18537022 - 20 0.004 0.0099890716 -0.31121926 0 -0.63509581 0 -0.31121926 - 30 0.006 0.0099475919 -0.43689013 0 -0.88644739 0 -0.43689013 - 40 0.008 0.0099059782 -0.5623833 0 -1.1374442 0 -0.5623833 - 50 0.01 0.0098642085 -0.68769978 0 -1.388088 0 -0.68769978 - 60 0.012 0.0098222618 -0.81284106 0 -1.6383818 0 -0.81284106 - 70 0.014 0.0097801186 -0.93780907 0 -1.8883294 0 -0.93780907 - 80 0.016 0.0097377603 -1.0626062 0 -2.1379352 0 -1.0626062 - 90 0.018 0.0096951693 -1.187235 0 -2.3872045 0 -1.187235 - 100 0.02 0.0096523288 -1.3116986 0 -2.6361432 0 -1.3116986 - 110 0.022 0.0096092227 -1.4360002 0 -2.8847577 0 -1.4360002 - 120 0.024 0.009565836 -1.5601431 0 -3.1330547 0 -1.5601431 - 130 0.026 0.0095221542 -1.6841309 0 -3.3810411 0 -1.6841309 - 140 0.028 0.0094781635 -1.8079673 0 -3.6287241 0 -1.8079673 - 150 0.03 0.0094338509 -1.9316557 0 -3.8761109 0 -1.9316557 - 160 0.032 0.0093892044 -2.0551997 0 -4.1232085 0 -2.0551997 - 170 0.034 0.0093442126 -2.178603 0 -4.370024 0 -2.178603 - 180 0.036 0.0092988654 -2.3018687 0 -4.6165639 0 -2.3018687 - 190 0.038 0.0092531537 -2.4250002 0 -4.8628348 0 -2.4250002 - 200 0.04 0.0092070698 -2.5480003 0 -5.1088426 0 -2.5480003 - 210 0.042 0.0091606073 -2.670872 0 -5.3545929 0 -2.670872 - 220 0.044 0.0091137617 -2.7936178 0 -5.6000909 0 -2.7936178 - 230 0.046 0.0090665298 -2.9162399 0 -5.8453412 0 -2.9162399 - 240 0.048 0.0090189108 -3.0387405 0 -6.0903478 0 -3.0387405 - 250 0.05 0.0089709056 -3.1611214 0 -6.3351146 0 -3.1611214 - 260 0.052 0.0089225173 -3.2833841 0 -6.5796445 0 -3.2833841 - 270 0.054 0.0088737511 -3.4055299 0 -6.8239403 0 -3.4055299 - 280 0.056 0.0088246147 -3.52756 0 -7.0680043 0 -3.52756 - 290 0.058 0.0087751176 -3.6494754 0 -7.3118383 0 -3.6494754 - 300 0.06 0.008725272 -3.7712768 0 -7.5554438 0 -3.7712768 - 310 0.062 0.0086750916 -3.8929648 0 -7.7988222 0 -3.8929648 - 320 0.064 0.0086245927 -4.0145399 0 -8.0419744 0 -4.0145399 - 330 0.066 0.0085737928 -4.1360026 0 -8.2849013 0 -4.1360026 - 340 0.068 0.0085227116 -4.2573532 0 -8.5276035 0 -4.2573532 - 350 0.07 0.0084713698 -4.378592 0 -8.7700818 0 -4.378592 - 360 0.072 0.0084197895 -4.4997194 0 -9.0123367 0 -4.4997194 - 370 0.074 0.0083679936 -4.6207358 0 -9.2543688 0 -4.6207358 - 380 0.076 0.0083160058 -4.7416414 0 -9.496179 0 -4.7416414 - 390 0.078 0.0082638503 -4.8624367 0 -9.7377681 0 -4.8624367 - 400 0.08 0.0082115512 -4.9831222 0 -9.9791371 0 -4.9831222 - 410 0.082 0.0081591329 -5.1036986 0 -10.220287 0 -5.1036986 - 420 0.084 0.0081066195 -5.2241665 0 -10.46122 0 -5.2241665 - 430 0.086 0.0080540347 -5.3445267 0 -10.701936 0 -5.3445267 - 440 0.088 0.008001402 -5.4647802 0 -10.942439 0 -5.4647802 - 450 0.09 0.0079487439 -5.5849281 0 -11.18273 0 -5.5849281 - 460 0.092 0.0078960829 -5.7049716 0 -11.422811 0 -5.7049716 - 470 0.094 0.0078434404 -5.824912 0 -11.662686 0 -5.824912 - 480 0.096 0.0077908378 -5.9447508 0 -11.902357 0 -5.9447508 - 490 0.098 0.0077382955 -6.0644896 0 -12.141828 0 -6.0644896 - 500 0.1 0.0076858338 -6.1841301 0 -12.381101 0 -6.1841301 -Loop time of 13.543 on 1 procs for 500 steps with 5780 atoms - -Performance: 0.638 ns/day, 37.619 hours/ns, 36.919 timesteps/s -100.0% CPU use with 1 MPI tasks x no OpenMP threads - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Pair | 3.8138 | 3.8138 | 3.8138 | 0.0 | 28.16 -Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.011875 | 0.011875 | 0.011875 | 0.0 | 0.09 -Output | 0.049726 | 0.049726 | 0.049726 | 0.0 | 0.37 -Modify | 9.655 | 9.655 | 9.655 | 0.0 | 71.29 -Other | | 0.01262 | | | 0.09 - -Nlocal: 5780 ave 5780 max 5780 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Nghost: 1065 ave 1065 max 1065 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Neighs: 0 ave 0 max 0 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -FullNghs: 92480 ave 92480 max 92480 min -Histogram: 1 0 0 0 0 0 0 0 0 0 - -Total # of neighbors = 92480 -Ave neighs/atom = 16 -Neighbor list builds = 0 -Dangerous builds = 0 - -Please see the log.cite file for references relevant to this simulation - -Total wall time: 0:00:13 diff --git a/examples/SPIN/bfo/log.19Nov19.spin.bfo.g++.4 b/examples/SPIN/bfo/log.19Nov19.spin.bfo.g++.4 deleted file mode 100644 index 8c701c93c13cd9c294d141c5c1ff752ec17dea5f..0000000000000000000000000000000000000000 --- a/examples/SPIN/bfo/log.19Nov19.spin.bfo.g++.4 +++ /dev/null @@ -1,167 +0,0 @@ -LAMMPS (30 Oct 2019) -# layer sc iron atoms (in the [001] plane) in bismuth oxide - -units metal -atom_style spin -dimension 3 -boundary p p f - -# necessary for the serial algorithm (sametag) -atom_modify map array - -lattice sc 3.96 -Lattice spacing in x,y,z = 3.96 3.96 3.96 -region box block 0.0 34.0 0.0 34.0 0.0 5.0 -create_box 1 box -Created orthogonal box = (0 0 0) to (134.64 134.64 19.8) - 2 by 2 by 1 MPI processor grid -create_atoms 1 box -Created 5780 atoms - create_atoms CPU = 0.00149798 secs - -# setting mass, mag. moments, and interactions for bfo - -mass 1 1.0 -set group all spin/random 11 2.50 - 5780 settings made for spin/random - -#pair_style hybrid/overlay spin/exchange 6.0 spin/magelec 4.5 -pair_style hybrid/overlay spin/exchange 6.0 spin/magelec 4.5 spin/dmi 4.5 -pair_coeff * * spin/exchange exchange 6.0 -0.01575 0.0 1.965 -pair_coeff * * spin/magelec magelec 4.5 0.000109 1.0 1.0 1.0 -pair_coeff * * spin/dmi dmi 4.5 0.00005 1.0 1.0 1.0 - -neighbor 0.1 bin -neigh_modify every 10 check yes delay 20 - -fix 1 all precession/spin anisotropy 0.0000033 0.0 0.0 1.0 -fix 2 all langevin/spin 0.0 0.1 21 -fix 3 all nve/spin lattice frozen - -timestep 0.0002 - -compute out_mag all spin -compute out_pe all pe -compute out_ke all ke -compute out_temp all temp - -variable magz equal c_out_mag[3] -variable magnorm equal c_out_mag[4] -variable emag equal c_out_mag[5] -variable tmag equal c_out_mag[6] - -#thermo_style custom step time v_magnorm v_emag temp etotal -thermo_style custom step time v_magnorm pe ke v_emag temp etotal -thermo 10 - -compute outsp all property/atom spx spy spz sp fmx fmy fmz -dump 1 all custom 100 dump_bfo.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] - -run 500 -Neighbor list info ... - update every 10 steps, delay 20 steps, check yes - max neighbors/atom: 2000, page size: 100000 - master list distance cutoff = 6.1 - ghost atom cutoff = 6.1 - binsize = 3.05, bins = 45 45 7 - 3 neighbor lists, perpetual/occasional/extra = 3 0 0 - (1) pair spin/exchange, perpetual - attributes: full, newton on - pair build: full/bin/atomonly - stencil: full/bin/3d - bin: standard - (2) pair spin/magelec, perpetual, copy from (1) - attributes: full, newton on - pair build: copy - stencil: none - bin: none - (3) pair spin/dmi, perpetual, copy from (1) - attributes: full, newton on - pair build: copy - stencil: none - bin: none -Per MPI rank memory allocation (min/avg/max) = 7.744 | 7.744 | 7.744 Mbytes -Step Time v_magnorm PotEng KinEng v_emag Temp TotEng - 0 0 0.010071723 -0.11868622 0 -0.12966919 0 -0.11868622 - 10 0.002 0.010030399 -0.37068593 0 -0.38171598 0 -0.37068593 - 20 0.004 0.0099889925 -0.6223216 0 -0.6334048 0 -0.6223216 - 30 0.006 0.0099474775 -0.87359359 0 -0.8847354 0 -0.87359359 - 40 0.008 0.0099058307 -1.1245034 0 -1.1357086 0 -1.1245034 - 50 0.01 0.0098640297 -1.3750538 0 -1.3863265 0 -1.3750538 - 60 0.012 0.0098220535 -1.6252482 0 -1.6365919 0 -1.6252482 - 70 0.014 0.0097798823 -1.8750914 0 -1.8865086 0 -1.8750914 - 80 0.016 0.0097374973 -2.1245886 0 -2.1360814 0 -2.1245886 - 90 0.018 0.0096948808 -2.3737458 0 -2.3853155 0 -2.3737458 - 100 0.02 0.0096520159 -2.6225698 0 -2.6342168 0 -2.6225698 - 110 0.022 0.0096088866 -2.8710677 0 -2.8827919 0 -2.8710677 - 120 0.024 0.0095654776 -3.1192469 0 -3.1310475 0 -3.1192469 - 130 0.026 0.0095217746 -3.367115 0 -3.3789906 0 -3.367115 - 140 0.028 0.0094777638 -3.61468 0 -3.6266285 0 -3.61468 - 150 0.03 0.0094334323 -3.8619496 0 -3.8739683 0 -3.8619496 - 160 0.032 0.0093887679 -4.1089316 0 -4.1210173 0 -4.1089316 - 170 0.034 0.0093437596 -4.3556335 0 -4.3677824 0 -4.3556335 - 180 0.036 0.0092983972 -4.6020625 0 -4.6142704 0 -4.6020625 - 190 0.038 0.0092526717 -4.8482255 0 -4.8604877 0 -4.8482255 - 200 0.04 0.0092065755 -5.0941291 0 -5.1064403 0 -5.0941291 - 210 0.042 0.0091601024 -5.3397792 0 -5.3521339 0 -5.3397792 - 220 0.044 0.0091132478 -5.5851813 0 -5.5975736 0 -5.5851813 - 230 0.046 0.0090660089 -5.8303404 0 -5.842764 0 -5.8303404 - 240 0.048 0.0090183847 -6.0752609 0 -6.0877092 0 -6.0752609 - 250 0.05 0.0089703764 -6.3199467 0 -6.3324129 0 -6.3199467 - 260 0.052 0.0089219873 -6.5644011 0 -6.5768782 0 -6.5644011 - 270 0.054 0.0088732228 -6.808627 0 -6.8211078 0 -6.808627 - 280 0.056 0.0088240906 -7.0526266 0 -7.0651038 0 -7.0526266 - 290 0.058 0.0087746006 -7.296402 0 -7.3088682 0 -7.296402 - 300 0.06 0.0087247648 -7.5399545 0 -7.5524024 0 -7.5399545 - 310 0.062 0.0086745976 -7.7832854 0 -7.7957077 0 -7.7832854 - 320 0.064 0.0086241149 -8.0263956 0 -8.038785 0 -8.0263956 - 330 0.066 0.008573335 -8.2692858 0 -8.281635 0 -8.2692858 - 340 0.068 0.0085222772 -8.5119564 0 -8.5242586 0 -8.5119564 - 350 0.07 0.0084709627 -8.7544078 0 -8.7666562 0 -8.7544078 - 360 0.072 0.0084194136 -8.9966403 0 -9.0088285 0 -8.9966403 - 370 0.074 0.008367653 -9.2386543 0 -9.2507761 0 -9.2386543 - 380 0.076 0.0083157046 -9.4804501 0 -9.4924997 0 -9.4804501 - 390 0.078 0.0082635925 -9.7220281 0 -9.7340001 0 -9.7220281 - 400 0.08 0.0082113412 -9.9633888 0 -9.9752784 0 -9.9633888 - 410 0.082 0.0081589747 -10.204533 0 -10.216336 0 -10.204533 - 420 0.084 0.0081065173 -10.445462 0 -10.457173 0 -10.445462 - 430 0.086 0.0080539925 -10.686176 0 -10.697793 0 -10.686176 - 440 0.088 0.0080014235 -10.926676 0 -10.938197 0 -10.926676 - 450 0.09 0.0079488329 -11.166966 0 -11.178387 0 -11.166966 - 460 0.092 0.0078962427 -11.407045 0 -11.418366 0 -11.407045 - 470 0.094 0.0078436743 -11.646917 0 -11.658136 0 -11.646917 - 480 0.096 0.0077911486 -11.886583 0 -11.8977 0 -11.886583 - 490 0.098 0.007738686 -12.126047 0 -12.137063 0 -12.126047 - 500 0.1 0.0076863062 -12.365311 0 -12.376226 0 -12.365311 -Loop time of 3.94852 on 4 procs for 500 steps with 5780 atoms - -Performance: 2.188 ns/day, 10.968 hours/ns, 126.630 timesteps/s -99.9% CPU use with 4 MPI tasks x no OpenMP threads - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Pair | 0.97416 | 0.98668 | 1.0022 | 1.0 | 24.99 -Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.032618 | 0.04948 | 0.062614 | 5.0 | 1.25 -Output | 0.014166 | 0.014229 | 0.014374 | 0.1 | 0.36 -Modify | 2.8947 | 2.8957 | 2.8965 | 0.0 | 73.34 -Other | | 0.002385 | | | 0.06 - -Nlocal: 1445 ave 1445 max 1445 min -Histogram: 4 0 0 0 0 0 0 0 0 0 -Nghost: 555 ave 555 max 555 min -Histogram: 4 0 0 0 0 0 0 0 0 0 -Neighs: 0 ave 0 max 0 min -Histogram: 4 0 0 0 0 0 0 0 0 0 -FullNghs: 23120 ave 23120 max 23120 min -Histogram: 4 0 0 0 0 0 0 0 0 0 - -Total # of neighbors = 92480 -Ave neighs/atom = 16 -Neighbor list builds = 0 -Dangerous builds = 0 - -Please see the log.cite file for references relevant to this simulation - -Total wall time: 0:00:03 diff --git a/examples/SPIN/cobalt_fcc/in.spin.cobalt_fcc b/examples/SPIN/cobalt_fcc/in.spin.cobalt_fcc index dd9ed890ee8b3be6ff1ba9d1ffab132fee743337..f81a962e3f6c33e5734e23b63baff451f735870b 100644 --- a/examples/SPIN/cobalt_fcc/in.spin.cobalt_fcc +++ b/examples/SPIN/cobalt_fcc/in.spin.cobalt_fcc @@ -45,8 +45,6 @@ compute out_pe all pe compute out_ke all ke compute out_temp all temp -thermo_style custom f_1 - variable magx equal c_out_mag[1] variable magy equal c_out_mag[2] variable magz equal c_out_mag[3] @@ -54,7 +52,7 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -thermo_style custom step time f_1 v_magx v_magy v_magnorm v_emag temp etotal +thermo_style custom step time v_magx v_magy v_magnorm pe v_emag temp etotal thermo 50 # compute outsp all property/atom spx spy spz sp fmx fmy fmz diff --git a/examples/SPIN/cobalt_fcc/log.14Apr20.spin.cobalt_fcc.g++.1 b/examples/SPIN/cobalt_fcc/log.14Apr20.spin.cobalt_fcc.g++.1 new file mode 100644 index 0000000000000000000000000000000000000000..f233c5e9ea6794d0846704b2294426be3fcc6a4e --- /dev/null +++ b/examples/SPIN/cobalt_fcc/log.14Apr20.spin.cobalt_fcc.g++.1 @@ -0,0 +1,144 @@ +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# fcc cobalt in a 3d periodic box + +clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +units metal +atom_style spin + +dimension 3 +boundary p p p + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice fcc 3.54 +Lattice spacing in x,y,z = 3.54 3.54 3.54 +region box block 0.0 5.0 0.0 5.0 0.0 5.0 +create_box 1 box +Created orthogonal box = (0 0 0) to (17.7 17.7 17.7) + 1 by 1 by 1 MPI processor grid +create_atoms 1 box +Created 500 atoms + create_atoms CPU = 0.00057292 secs + +# setting mass, mag. moments, and interactions for fcc cobalt + +mass 1 58.93 + +#set group all spin/random 31 1.72 +set group all spin 1.72 0.0 0.0 1.0 + 500 settings made for spin +velocity all create 100 4928459 rot yes dist gaussian + +pair_style hybrid/overlay eam/alloy spin/exchange 4.0 +pair_coeff * * eam/alloy Co_PurjaPun_2012.eam.alloy Co +pair_coeff * * spin/exchange exchange 4.0 0.0446928 0.003496 1.4885 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin zeeman 1.0 0.0 0.0 1.0 +fix_modify 1 energy yes + +fix 2 all langevin/spin 0.0 0.0 21 + +fix 3 all nve/spin lattice moving +timestep 0.0001 + +# compute and output options + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable magx equal c_out_mag[1] +variable magy equal c_out_mag[2] +variable magz equal c_out_mag[3] +variable magnorm equal c_out_mag[4] +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_magx v_magy v_magnorm pe v_emag temp etotal +thermo 50 + +# compute outsp all property/atom spx spy spz sp fmx fmy fmz +# dump 1 all custom 100 dump_cobalt_fcc.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] + +run 1000 +Neighbor list info ... + update every 10 steps, delay 20 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 6.59954 + ghost atom cutoff = 6.59954 + binsize = 3.29977, bins = 6 6 6 + 2 neighbor lists, perpetual/occasional/extra = 2 0 0 + (1) pair eam/alloy, perpetual, half/full from (2) + attributes: half, newton on + pair build: halffull/newton + stencil: none + bin: none + (2) pair spin/exchange, perpetual + attributes: full, newton on + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 5.718 | 5.718 | 5.718 Mbytes +Step Time v_magx v_magy v_magnorm PotEng v_emag Temp TotEng + 0 0 0 0 1 -2285.0679 -94.095041 100.00543 -2278.6175 + 50 0.005 0 0 1 -2284.7512 -94.095025 95.095146 -2278.6175 + 100 0.01 0 0 1 -2283.8798 -94.094815 81.584879 -2278.6175 + 150 0.015 0 0 1 -2282.6701 -94.09401 62.831647 -2278.6175 + 200 0.02 0 0 1 -2281.4185 -94.092175 43.426664 -2278.6174 + 250 0.025 0 0 1 -2280.4165 -94.089084 27.892653 -2278.6174 + 300 0.03 0 0 1 -2279.8662 -94.084906 19.36049 -2278.6174 + 350 0.035 0 0 1 -2279.8238 -94.080244 18.702888 -2278.6174 + 400 0.04 0 0 1 -2280.1962 -94.076013 24.47725 -2278.6174 + 450 0.045 0 0 1 -2280.7898 -94.073181 33.678697 -2278.6175 + 500 0.05 0 0 1 -2281.3871 -94.072475 42.940446 -2278.6175 + 550 0.055 0 0 1 -2281.8215 -94.074138 49.675032 -2278.6175 + 600 0.06 0 0 1 -2282.0209 -94.077829 52.765336 -2278.6175 + 650 0.065 0 0 1 -2282.0136 -94.082703 52.652501 -2278.6175 + 700 0.07 0 0 1 -2281.8997 -94.087644 50.88749 -2278.6174 + 750 0.075 0 0 1 -2281.8013 -94.091583 49.361723 -2278.6174 + 800 0.08 0 0 1 -2281.8108 -94.093792 49.508108 -2278.6174 + 850 0.085 0 0 1 -2281.9558 -94.094064 51.756992 -2278.6174 + 900 0.09 0 0 1 -2282.1927 -94.092723 55.428834 -2278.6175 + 950 0.095 0 0 1 -2282.4277 -94.090469 59.072774 -2278.6175 + 1000 0.1 0 0 1 -2282.5578 -94.088121 61.090565 -2278.6175 +Loop time of 4.4001 on 1 procs for 1000 steps with 500 atoms + +Performance: 1.964 ns/day, 12.222 hours/ns, 227.268 timesteps/s +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 2.0815 | 2.0815 | 2.0815 | 0.0 | 47.31 +Neigh | 0.011835 | 0.011835 | 0.011835 | 0.0 | 0.27 +Comm | 0.033331 | 0.033331 | 0.033331 | 0.0 | 0.76 +Output | 0.00045395 | 0.00045395 | 0.00045395 | 0.0 | 0.01 +Modify | 2.2677 | 2.2677 | 2.2677 | 0.0 | 51.54 +Other | | 0.005309 | | | 0.12 + +Nlocal: 500 ave 500 max 500 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 1956 ave 1956 max 1956 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 24088 ave 24088 max 24088 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +FullNghs: 48176 ave 48176 max 48176 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 48176 +Ave neighs/atom = 96.352 +Neighbor list builds = 6 +Dangerous builds = 0 + +Please see the log.cite file for references relevant to this simulation + +Total wall time: 0:00:04 diff --git a/examples/SPIN/cobalt_fcc/log.14Apr20.spin.cobalt_fcc.g++.4 b/examples/SPIN/cobalt_fcc/log.14Apr20.spin.cobalt_fcc.g++.4 new file mode 100644 index 0000000000000000000000000000000000000000..3cbcd942686ef00d9c80b46ede39d141d033b652 --- /dev/null +++ b/examples/SPIN/cobalt_fcc/log.14Apr20.spin.cobalt_fcc.g++.4 @@ -0,0 +1,144 @@ +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# fcc cobalt in a 3d periodic box + +clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +units metal +atom_style spin + +dimension 3 +boundary p p p + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice fcc 3.54 +Lattice spacing in x,y,z = 3.54 3.54 3.54 +region box block 0.0 5.0 0.0 5.0 0.0 5.0 +create_box 1 box +Created orthogonal box = (0 0 0) to (17.7 17.7 17.7) + 1 by 2 by 2 MPI processor grid +create_atoms 1 box +Created 500 atoms + create_atoms CPU = 0.000688791 secs + +# setting mass, mag. moments, and interactions for fcc cobalt + +mass 1 58.93 + +#set group all spin/random 31 1.72 +set group all spin 1.72 0.0 0.0 1.0 + 500 settings made for spin +velocity all create 100 4928459 rot yes dist gaussian + +pair_style hybrid/overlay eam/alloy spin/exchange 4.0 +pair_coeff * * eam/alloy Co_PurjaPun_2012.eam.alloy Co +pair_coeff * * spin/exchange exchange 4.0 0.0446928 0.003496 1.4885 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin zeeman 1.0 0.0 0.0 1.0 +fix_modify 1 energy yes + +fix 2 all langevin/spin 0.0 0.0 21 + +fix 3 all nve/spin lattice moving +timestep 0.0001 + +# compute and output options + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable magx equal c_out_mag[1] +variable magy equal c_out_mag[2] +variable magz equal c_out_mag[3] +variable magnorm equal c_out_mag[4] +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_magx v_magy v_magnorm pe v_emag temp etotal +thermo 50 + +# compute outsp all property/atom spx spy spz sp fmx fmy fmz +# dump 1 all custom 100 dump_cobalt_fcc.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] + +run 1000 +Neighbor list info ... + update every 10 steps, delay 20 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 6.59954 + ghost atom cutoff = 6.59954 + binsize = 3.29977, bins = 6 6 6 + 2 neighbor lists, perpetual/occasional/extra = 2 0 0 + (1) pair eam/alloy, perpetual, half/full from (2) + attributes: half, newton on + pair build: halffull/newton + stencil: none + bin: none + (2) pair spin/exchange, perpetual + attributes: full, newton on + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 5.664 | 5.664 | 5.664 Mbytes +Step Time v_magx v_magy v_magnorm PotEng v_emag Temp TotEng + 0 0 0 0 1 -2285.0679 -94.095041 100.00543 -2278.6175 + 50 0.005 0 0 1 -2284.7564 -94.094967 95.176149 -2278.6175 + 100 0.01 0 0 1 -2283.8977 -94.094606 81.863661 -2278.6175 + 150 0.015 0 0 1 -2282.7006 -94.09362 63.303924 -2278.6175 + 200 0.02 0 0 1 -2281.452 -94.091648 43.94572 -2278.6174 + 250 0.025 0 0 1 -2280.4374 -94.08853 28.216468 -2278.6174 + 300 0.03 0 0 1 -2279.859 -94.084466 19.248729 -2278.6174 + 350 0.035 0 0 1 -2279.78 -94.080048 18.024631 -2278.6174 + 400 0.04 0 0 1 -2280.1196 -94.076134 23.288369 -2278.6174 + 450 0.045 0 0 1 -2280.697 -94.073612 32.240137 -2278.6175 + 500 0.05 0 0 1 -2281.305 -94.073124 41.666374 -2278.6175 + 550 0.055 0 0 1 -2281.7792 -94.074853 49.019175 -2278.6175 + 600 0.06 0 0 1 -2282.0409 -94.078438 53.075365 -2278.6175 + 650 0.065 0 0 1 -2282.1031 -94.083056 54.040647 -2278.6175 + 700 0.07 0 0 1 -2282.0468 -94.087642 53.167271 -2278.6175 + 750 0.075 0 0 1 -2281.9765 -94.09119 52.076935 -2278.6174 + 800 0.08 0 0 1 -2281.9742 -94.093023 52.041607 -2278.6174 + 850 0.085 0 0 1 -2282.0669 -94.09296 53.478909 -2278.6174 + 900 0.09 0 0 1 -2282.2193 -94.091331 55.842481 -2278.6175 + 950 0.095 0 0 1 -2282.354 -94.088827 57.930866 -2278.6175 + 1000 0.1 0 0 1 -2282.3885 -94.086262 58.464579 -2278.6175 +Loop time of 2.75 on 4 procs for 1000 steps with 500 atoms + +Performance: 3.142 ns/day, 7.639 hours/ns, 363.637 timesteps/s +99.7% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0.57529 | 0.60652 | 0.63664 | 3.2 | 22.06 +Neigh | 0.0026889 | 0.0029447 | 0.0030761 | 0.3 | 0.11 +Comm | 0.10943 | 0.14471 | 0.17847 | 7.5 | 5.26 +Output | 0.00045419 | 0.00050056 | 0.00060511 | 0.0 | 0.02 +Modify | 1.9865 | 1.9926 | 1.998 | 0.3 | 72.46 +Other | | 0.002738 | | | 0.10 + +Nlocal: 125 ave 132 max 116 min +Histogram: 1 0 0 0 0 1 1 0 0 1 +Nghost: 1099 ave 1108 max 1092 min +Histogram: 1 0 0 1 1 0 0 0 0 1 +Neighs: 6033 ave 6372 max 5495 min +Histogram: 1 0 0 0 0 0 0 2 0 1 +FullNghs: 12066 ave 12964 max 10977 min +Histogram: 1 0 0 0 0 1 1 0 0 1 + +Total # of neighbors = 48264 +Ave neighs/atom = 96.528 +Neighbor list builds = 6 +Dangerous builds = 0 + +Please see the log.cite file for references relevant to this simulation + +Total wall time: 0:00:02 diff --git a/examples/SPIN/cobalt_fcc/log.19Nov19.spin.cobalt_fcc.g++.1 b/examples/SPIN/cobalt_fcc/log.19Nov19.spin.cobalt_fcc.g++.1 deleted file mode 100644 index 06774d08580242bac91bffff8d89150bcf1f80f6..0000000000000000000000000000000000000000 --- a/examples/SPIN/cobalt_fcc/log.19Nov19.spin.cobalt_fcc.g++.1 +++ /dev/null @@ -1,142 +0,0 @@ -LAMMPS (30 Oct 2019) -# fcc cobalt in a 3d periodic box - -clear -units metal -atom_style spin - -dimension 3 -boundary p p p - -# necessary for the serial algorithm (sametag) -atom_modify map array - -lattice fcc 3.54 -Lattice spacing in x,y,z = 3.54 3.54 3.54 -region box block 0.0 5.0 0.0 5.0 0.0 5.0 -create_box 1 box -Created orthogonal box = (0 0 0) to (17.7 17.7 17.7) - 1 by 1 by 1 MPI processor grid -create_atoms 1 box -Created 500 atoms - create_atoms CPU = 0.000470161 secs - -# setting mass, mag. moments, and interactions for fcc cobalt - -mass 1 58.93 - -#set group all spin/random 31 1.72 -set group all spin 1.72 0.0 0.0 1.0 - 500 settings made for spin -velocity all create 100 4928459 rot yes dist gaussian - -pair_style hybrid/overlay eam/alloy spin/exchange 4.0 -pair_coeff * * eam/alloy Co_PurjaPun_2012.eam.alloy Co -pair_coeff * * spin/exchange exchange 4.0 0.0446928 0.003496 1.4885 - -neighbor 0.1 bin -neigh_modify every 10 check yes delay 20 - -fix 1 all precession/spin zeeman 1.0 0.0 0.0 1.0 -fix_modify 1 energy yes - -fix 2 all langevin/spin 0.0 0.0 21 - -fix 3 all nve/spin lattice moving -timestep 0.0001 - -# compute and output options - -compute out_mag all spin -compute out_pe all pe -compute out_ke all ke -compute out_temp all temp - -thermo_style custom f_1 - -variable magx equal c_out_mag[1] -variable magy equal c_out_mag[2] -variable magz equal c_out_mag[3] -variable magnorm equal c_out_mag[4] -variable emag equal c_out_mag[5] -variable tmag equal c_out_mag[6] - -thermo_style custom step time f_1 v_magx v_magy v_magnorm v_emag temp etotal -thermo 50 - -# compute outsp all property/atom spx spy spz sp fmx fmy fmz -# dump 1 all custom 100 dump_cobalt_fcc.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] - -run 1000 -Neighbor list info ... - update every 10 steps, delay 20 steps, check yes - max neighbors/atom: 2000, page size: 100000 - master list distance cutoff = 6.59954 - ghost atom cutoff = 6.59954 - binsize = 3.29977, bins = 6 6 6 - 2 neighbor lists, perpetual/occasional/extra = 2 0 0 - (1) pair eam/alloy, perpetual, half/full from (2) - attributes: half, newton on - pair build: halffull/newton - stencil: none - bin: none - (2) pair spin/exchange, perpetual - attributes: full, newton on - pair build: full/bin/atomonly - stencil: full/bin/3d - bin: standard -Per MPI rank memory allocation (min/avg/max) = 5.718 | 5.718 | 5.718 Mbytes -Step Time f_1 v_magx v_magy v_magnorm v_emag Temp TotEng - 0 0 -0.099570972 0 0 1 -188.09051 100.00543 -2278.6175 - 50 0.005 -0.099570972 0 0 1 -188.09048 95.094679 -2278.6175 - 100 0.01 -0.099570972 0 0 1 -188.09007 81.578321 -2278.6177 - 150 0.015 -0.099570972 0 0 1 -188.08848 62.802727 -2278.6185 - 200 0.02 -0.099570972 0 0 1 -188.08487 43.35108 -2278.6203 - 250 0.025 -0.099570972 0 0 1 -188.07877 27.749821 -2278.6233 - 300 0.03 -0.099570972 0 0 1 -188.07054 19.149389 -2278.6274 - 350 0.035 -0.099570972 0 0 1 -188.06135 18.453387 -2278.632 - 400 0.04 -0.099570972 0 0 1 -188.053 24.249423 -2278.6362 - 450 0.045 -0.099570972 0 0 1 -188.04742 33.548008 -2278.639 - 500 0.05 -0.099570972 0 0 1 -188.04604 42.973172 -2278.6397 - 550 0.055 -0.099570972 0 0 1 -188.04935 49.902539 -2278.638 - 600 0.06 -0.099570972 0 0 1 -188.0567 53.166772 -2278.6344 - 650 0.065 -0.099570972 0 0 1 -188.06642 53.153416 -2278.6295 - 700 0.07 -0.099570972 0 0 1 -188.07628 51.377187 -2278.6246 - 750 0.075 -0.099570972 0 0 1 -188.08415 49.725449 -2278.6206 - 800 0.08 -0.099570972 0 0 1 -188.08857 49.663576 -2278.6184 - 850 0.085 -0.099570972 0 0 1 -188.0891 51.681567 -2278.6182 - 900 0.09 -0.099570972 0 0 1 -188.08636 55.166554 -2278.6195 - 950 0.095 -0.099570972 0 0 1 -188.08174 58.718232 -2278.6218 - 1000 0.1 -0.099570972 0 0 1 -188.0769 60.75567 -2278.6243 -Loop time of 4.6196 on 1 procs for 1000 steps with 500 atoms - -Performance: 1.870 ns/day, 12.832 hours/ns, 216.469 timesteps/s -99.9% CPU use with 1 MPI tasks x no OpenMP threads - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Pair | 2.3116 | 2.3116 | 2.3116 | 0.0 | 50.04 -Neigh | 0.011227 | 0.011227 | 0.011227 | 0.0 | 0.24 -Comm | 0.032837 | 0.032837 | 0.032837 | 0.0 | 0.71 -Output | 0.00039411 | 0.00039411 | 0.00039411 | 0.0 | 0.01 -Modify | 2.2584 | 2.2584 | 2.2584 | 0.0 | 48.89 -Other | | 0.005152 | | | 0.11 - -Nlocal: 500 ave 500 max 500 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Nghost: 1956 ave 1956 max 1956 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Neighs: 24065 ave 24065 max 24065 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -FullNghs: 48130 ave 48130 max 48130 min -Histogram: 1 0 0 0 0 0 0 0 0 0 - -Total # of neighbors = 48130 -Ave neighs/atom = 96.26 -Neighbor list builds = 6 -Dangerous builds = 0 - -Please see the log.cite file for references relevant to this simulation - -Total wall time: 0:00:04 diff --git a/examples/SPIN/cobalt_fcc/log.19Nov19.spin.cobalt_fcc.g++.4 b/examples/SPIN/cobalt_fcc/log.19Nov19.spin.cobalt_fcc.g++.4 deleted file mode 100644 index 331ed60315a408c4cc2d221e13b18348b540b87b..0000000000000000000000000000000000000000 --- a/examples/SPIN/cobalt_fcc/log.19Nov19.spin.cobalt_fcc.g++.4 +++ /dev/null @@ -1,142 +0,0 @@ -LAMMPS (30 Oct 2019) -# fcc cobalt in a 3d periodic box - -clear -units metal -atom_style spin - -dimension 3 -boundary p p p - -# necessary for the serial algorithm (sametag) -atom_modify map array - -lattice fcc 3.54 -Lattice spacing in x,y,z = 3.54 3.54 3.54 -region box block 0.0 5.0 0.0 5.0 0.0 5.0 -create_box 1 box -Created orthogonal box = (0 0 0) to (17.7 17.7 17.7) - 1 by 2 by 2 MPI processor grid -create_atoms 1 box -Created 500 atoms - create_atoms CPU = 0.000808001 secs - -# setting mass, mag. moments, and interactions for fcc cobalt - -mass 1 58.93 - -#set group all spin/random 31 1.72 -set group all spin 1.72 0.0 0.0 1.0 - 500 settings made for spin -velocity all create 100 4928459 rot yes dist gaussian - -pair_style hybrid/overlay eam/alloy spin/exchange 4.0 -pair_coeff * * eam/alloy Co_PurjaPun_2012.eam.alloy Co -pair_coeff * * spin/exchange exchange 4.0 0.0446928 0.003496 1.4885 - -neighbor 0.1 bin -neigh_modify every 10 check yes delay 20 - -fix 1 all precession/spin zeeman 1.0 0.0 0.0 1.0 -fix_modify 1 energy yes - -fix 2 all langevin/spin 0.0 0.0 21 - -fix 3 all nve/spin lattice moving -timestep 0.0001 - -# compute and output options - -compute out_mag all spin -compute out_pe all pe -compute out_ke all ke -compute out_temp all temp - -thermo_style custom f_1 - -variable magx equal c_out_mag[1] -variable magy equal c_out_mag[2] -variable magz equal c_out_mag[3] -variable magnorm equal c_out_mag[4] -variable emag equal c_out_mag[5] -variable tmag equal c_out_mag[6] - -thermo_style custom step time f_1 v_magx v_magy v_magnorm v_emag temp etotal -thermo 50 - -# compute outsp all property/atom spx spy spz sp fmx fmy fmz -# dump 1 all custom 100 dump_cobalt_fcc.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] - -run 1000 -Neighbor list info ... - update every 10 steps, delay 20 steps, check yes - max neighbors/atom: 2000, page size: 100000 - master list distance cutoff = 6.59954 - ghost atom cutoff = 6.59954 - binsize = 3.29977, bins = 6 6 6 - 2 neighbor lists, perpetual/occasional/extra = 2 0 0 - (1) pair eam/alloy, perpetual, half/full from (2) - attributes: half, newton on - pair build: halffull/newton - stencil: none - bin: none - (2) pair spin/exchange, perpetual - attributes: full, newton on - pair build: full/bin/atomonly - stencil: full/bin/3d - bin: standard -Per MPI rank memory allocation (min/avg/max) = 5.664 | 5.664 | 5.664 Mbytes -Step Time f_1 v_magx v_magy v_magnorm v_emag Temp TotEng - 0 0 -0.099570972 0 0 1 -188.09051 100.00543 -2372.6129 - 50 0.005 -0.099570972 0 0 1 -188.09036 95.174807 -2372.6129 - 100 0.01 -0.099570972 0 0 1 -188.08965 81.854304 -2372.6129 - 150 0.015 -0.099570972 0 0 1 -188.0877 63.270938 -2372.6129 - 200 0.02 -0.099570972 0 0 1 -188.08381 43.867262 -2372.6129 - 250 0.025 -0.099570972 0 0 1 -188.07767 28.075261 -2372.6129 - 300 0.03 -0.099570972 0 0 1 -188.06966 19.046222 -2372.6129 - 350 0.035 -0.099570972 0 0 1 -188.06096 17.79071 -2372.6129 - 400 0.04 -0.099570972 0 0 1 -188.05326 23.079994 -2372.6129 - 450 0.045 -0.099570972 0 0 1 -188.04831 32.127316 -2372.6129 - 500 0.05 -0.099570972 0 0 1 -188.04737 41.709644 -2372.6129 - 550 0.055 -0.099570972 0 0 1 -188.05082 49.246292 -2372.6129 - 600 0.06 -0.099570972 0 0 1 -188.05795 53.465535 -2372.6129 - 650 0.065 -0.099570972 0 0 1 -188.06713 54.522857 -2372.6129 - 700 0.07 -0.099570972 0 0 1 -188.07626 53.635521 -2372.6129 - 750 0.075 -0.099570972 0 0 1 -188.08332 52.419678 -2372.6129 - 800 0.08 -0.099570972 0 0 1 -188.08696 52.176558 -2372.6129 - 850 0.085 -0.099570972 0 0 1 -188.0868 53.380592 -2372.6129 - 900 0.09 -0.099570972 0 0 1 -188.08348 55.551378 -2372.6129 - 950 0.095 -0.099570972 0 0 1 -188.07838 57.540047 -2372.6129 - 1000 0.1 -0.099570972 0 0 1 -188.07314 58.088674 -2372.6129 -Loop time of 2.54753 on 4 procs for 1000 steps with 500 atoms - -Performance: 3.392 ns/day, 7.076 hours/ns, 392.538 timesteps/s -100.0% CPU use with 4 MPI tasks x no OpenMP threads - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Pair | 0.62017 | 0.6485 | 0.66275 | 2.1 | 25.46 -Neigh | 0.0027115 | 0.0029724 | 0.0030868 | 0.3 | 0.12 -Comm | 0.095047 | 0.1102 | 0.13819 | 5.0 | 4.33 -Output | 0.00039029 | 0.00042999 | 0.00049996 | 0.0 | 0.02 -Modify | 1.7801 | 1.7834 | 1.7852 | 0.1 | 70.01 -Other | | 0.002028 | | | 0.08 - -Nlocal: 125 ave 133 max 116 min -Histogram: 1 0 0 0 0 2 0 0 0 1 -Nghost: 1099 ave 1108 max 1091 min -Histogram: 1 0 0 0 2 0 0 0 0 1 -Neighs: 6032.5 ave 6417 max 5489 min -Histogram: 1 0 0 0 0 0 1 1 0 1 -FullNghs: 12065 ave 13062 max 10970 min -Histogram: 1 0 0 0 0 2 0 0 0 1 - -Total # of neighbors = 48260 -Ave neighs/atom = 96.52 -Neighbor list builds = 6 -Dangerous builds = 0 - -Please see the log.cite file for references relevant to this simulation - -Total wall time: 0:00:02 diff --git a/examples/SPIN/cobalt_hcp/in.spin.cobalt_hcp b/examples/SPIN/cobalt_hcp/in.spin.cobalt_hcp index dd114202cb7dce70f07dedf74077da08e36d2c90..2bfa8393f387c1d5b5646f0deff19fa930a0543c 100644 --- a/examples/SPIN/cobalt_hcp/in.spin.cobalt_hcp +++ b/examples/SPIN/cobalt_hcp/in.spin.cobalt_hcp @@ -32,10 +32,9 @@ pair_coeff * * spin/exchange exchange 4.0 -0.3593 1.135028015e-05 1.064568567 neighbor 0.1 bin neigh_modify every 10 check yes delay 20 -#fix 1 all precession/spin zeeman 1.0 0.0 0.0 1.0 fix 1 all precession/spin anisotropy 0.01 0.0 0.0 1.0 -#fix 2 all langevin/spin 0.0 0.0 21 -fix 2 all langevin/spin 0.0 0.1 21 +fix_modify 1 energy yes +fix 2 all langevin/spin 0.0 0.0 21 fix 3 all nve/spin lattice moving timestep 0.0001 @@ -51,8 +50,8 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -thermo_style custom step time v_magnorm v_emag temp press etotal -thermo 10 +thermo_style custom step time v_magnorm pe v_emag temp press etotal +thermo 50 compute outsp all property/atom spx spy spz sp fmx fmy fmz dump 1 all custom 100 dump_cobalt_hcp.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] diff --git a/examples/SPIN/cobalt_hcp/log.14Apr20.spin.cobalt_hcp.g++.1 b/examples/SPIN/cobalt_hcp/log.14Apr20.spin.cobalt_hcp.g++.1 new file mode 100644 index 0000000000000000000000000000000000000000..b26892f6e9831524682bd1314917ce2ced478975 --- /dev/null +++ b/examples/SPIN/cobalt_hcp/log.14Apr20.spin.cobalt_hcp.g++.1 @@ -0,0 +1,142 @@ +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# hcp cobalt in a 3d periodic box + +clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +units metal +atom_style spin + +dimension 3 +boundary p p p + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice hcp 2.5071 +Lattice spacing in x,y,z = 2.5071 4.34242 4.09408 +region box block 0.0 5.0 0.0 5.0 0.0 5.0 +create_box 1 box +Created orthogonal box = (0 0 0) to (12.5355 21.7121 20.4704) + 1 by 1 by 1 MPI processor grid +create_atoms 1 box +Created 500 atoms + create_atoms CPU = 0.00121403 secs + +# setting mass, mag. moments, and interactions for hcp cobalt + +mass 1 58.93 + +set group all spin/random 31 1.72 + 500 settings made for spin/random +#set group all spin 1.72 0.0 0.0 1.0 +velocity all create 100 4928459 rot yes dist gaussian + +#pair_style hybrid/overlay eam/alloy spin/exchange 4.0 spin/neel 4.0 +pair_style hybrid/overlay eam/alloy spin/exchange 4.0 +pair_coeff * * eam/alloy Co_PurjaPun_2012.eam.alloy Co +pair_coeff * * spin/exchange exchange 4.0 -0.3593 1.135028015e-05 1.064568567 +#pair_coeff * * spin/neel neel 4.0 0.0048 0.234 1.168 2.6905 0.705 0.652 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin anisotropy 0.01 0.0 0.0 1.0 +fix_modify 1 energy yes +fix 2 all langevin/spin 0.0 0.0 21 +fix 3 all nve/spin lattice moving + +timestep 0.0001 + + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable magz equal c_out_mag[3] +variable magnorm equal c_out_mag[4] +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_magnorm pe v_emag temp press etotal +thermo 50 + +compute outsp all property/atom spx spy spz sp fmx fmy fmz +dump 1 all custom 100 dump_cobalt_hcp.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] + +run 1000 +Neighbor list info ... + update every 10 steps, delay 20 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 6.59954 + ghost atom cutoff = 6.59954 + binsize = 3.29977, bins = 4 7 7 + 2 neighbor lists, perpetual/occasional/extra = 2 0 0 + (1) pair eam/alloy, perpetual, half/full from (2) + attributes: half, newton on + pair build: halffull/newton + stencil: none + bin: none + (2) pair spin/exchange, perpetual + attributes: full, newton on + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 7.902 | 7.902 | 7.902 Mbytes +Step Time v_magnorm PotEng v_emag Temp Press TotEng + 0 0 0.076558814 -2197.5536 -2.5536882 100.00543 -552.75983 -2191.1032 + 50 0.005 0.079437931 -2197.3113 -2.6177795 96.18776 -337.75504 -2191.1071 + 100 0.01 0.079575823 -2196.5768 -2.7375927 84.740309 571.91195 -2191.1109 + 150 0.015 0.078526145 -2195.4996 -2.8719243 67.984081 1845.185 -2191.1146 + 200 0.02 0.077001318 -2194.3502 -3.019723 50.107518 3008.9709 -2191.1182 + 250 0.025 0.077628454 -2193.403 -3.1832392 35.364524 4018.0217 -2191.122 + 300 0.03 0.077407462 -2192.8618 -3.3557644 26.910583 4535.9542 -2191.126 + 350 0.035 0.078090775 -2192.8052 -3.5305639 25.971561 4733.0322 -2191.13 + 400 0.04 0.078594494 -2193.135 -3.6772939 31.026665 4309.2088 -2191.1338 + 450 0.045 0.079898162 -2193.6965 -3.8044501 39.672566 3591.9593 -2191.1376 + 500 0.05 0.079885039 -2194.3293 -3.9435795 49.423774 2698.4519 -2191.1414 + 550 0.055 0.077196547 -2194.8554 -4.0868278 57.523322 1934.2444 -2191.1451 + 600 0.06 0.075430904 -2195.1797 -4.2355252 62.494025 1597.2543 -2191.1488 + 650 0.065 0.076510964 -2195.2854 -4.3752086 64.080496 1656.2312 -2191.1522 + 700 0.07 0.07649426 -2195.2723 -4.5226349 63.825926 1521.7541 -2191.1555 + 750 0.075 0.076254777 -2195.2746 -4.6937954 63.804162 1505.1323 -2191.1592 + 800 0.08 0.074211447 -2195.3567 -4.8567561 65.022623 1203.5409 -2191.1627 + 850 0.085 0.072034236 -2195.5531 -5.0007443 68.003461 900.03381 -2191.1668 + 900 0.09 0.071097702 -2195.8563 -5.1391578 72.641879 548.08834 -2191.1709 + 950 0.095 0.072642434 -2196.2007 -5.2853353 77.926596 194.45928 -2191.1743 + 1000 0.1 0.07306537 -2196.46 -5.4169261 81.891897 -404.70871 -2191.1779 +Loop time of 4.78824 on 1 procs for 1000 steps with 500 atoms + +Performance: 1.804 ns/day, 13.301 hours/ns, 208.845 timesteps/s +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 2.3284 | 2.3284 | 2.3284 | 0.0 | 48.63 +Neigh | 0.01273 | 0.01273 | 0.01273 | 0.0 | 0.27 +Comm | 0.042432 | 0.042432 | 0.042432 | 0.0 | 0.89 +Output | 0.0083201 | 0.0083201 | 0.0083201 | 0.0 | 0.17 +Modify | 2.3895 | 2.3895 | 2.3895 | 0.0 | 49.90 +Other | | 0.006823 | | | 0.14 + +Nlocal: 500 ave 500 max 500 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 2444 ave 2444 max 2444 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 27036 ave 27036 max 27036 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +FullNghs: 54072 ave 54072 max 54072 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 54072 +Ave neighs/atom = 108.144 +Neighbor list builds = 6 +Dangerous builds = 0 + +Please see the log.cite file for references relevant to this simulation + +Total wall time: 0:00:04 diff --git a/examples/SPIN/cobalt_hcp/log.14Apr20.spin.cobalt_hcp.g++.4 b/examples/SPIN/cobalt_hcp/log.14Apr20.spin.cobalt_hcp.g++.4 new file mode 100644 index 0000000000000000000000000000000000000000..ad8bf1807f0925c3e74a9697392bf7d44376ce82 --- /dev/null +++ b/examples/SPIN/cobalt_hcp/log.14Apr20.spin.cobalt_hcp.g++.4 @@ -0,0 +1,142 @@ +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# hcp cobalt in a 3d periodic box + +clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +units metal +atom_style spin + +dimension 3 +boundary p p p + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice hcp 2.5071 +Lattice spacing in x,y,z = 2.5071 4.34242 4.09408 +region box block 0.0 5.0 0.0 5.0 0.0 5.0 +create_box 1 box +Created orthogonal box = (0 0 0) to (12.5355 21.7121 20.4704) + 1 by 2 by 2 MPI processor grid +create_atoms 1 box +Created 500 atoms + create_atoms CPU = 0.000710964 secs + +# setting mass, mag. moments, and interactions for hcp cobalt + +mass 1 58.93 + +set group all spin/random 31 1.72 + 500 settings made for spin/random +#set group all spin 1.72 0.0 0.0 1.0 +velocity all create 100 4928459 rot yes dist gaussian + +#pair_style hybrid/overlay eam/alloy spin/exchange 4.0 spin/neel 4.0 +pair_style hybrid/overlay eam/alloy spin/exchange 4.0 +pair_coeff * * eam/alloy Co_PurjaPun_2012.eam.alloy Co +pair_coeff * * spin/exchange exchange 4.0 -0.3593 1.135028015e-05 1.064568567 +#pair_coeff * * spin/neel neel 4.0 0.0048 0.234 1.168 2.6905 0.705 0.652 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin anisotropy 0.01 0.0 0.0 1.0 +fix_modify 1 energy yes +fix 2 all langevin/spin 0.0 0.0 21 +fix 3 all nve/spin lattice moving + +timestep 0.0001 + + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable magz equal c_out_mag[3] +variable magnorm equal c_out_mag[4] +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_magnorm pe v_emag temp press etotal +thermo 50 + +compute outsp all property/atom spx spy spz sp fmx fmy fmz +dump 1 all custom 100 dump_cobalt_hcp.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] + +run 1000 +Neighbor list info ... + update every 10 steps, delay 20 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 6.59954 + ghost atom cutoff = 6.59954 + binsize = 3.29977, bins = 4 7 7 + 2 neighbor lists, perpetual/occasional/extra = 2 0 0 + (1) pair eam/alloy, perpetual, half/full from (2) + attributes: half, newton on + pair build: halffull/newton + stencil: none + bin: none + (2) pair spin/exchange, perpetual + attributes: full, newton on + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 7.814 | 7.814 | 7.815 Mbytes +Step Time v_magnorm PotEng v_emag Temp Press TotEng + 0 0 0.076558814 -2197.5536 -2.5536882 100.00543 -552.75983 -2191.1032 + 50 0.005 0.079452711 -2197.3115 -2.6266704 96.190558 -328.47868 -2191.1071 + 100 0.01 0.079678568 -2196.5823 -2.7759025 84.826338 585.21827 -2191.1109 + 150 0.015 0.078665787 -2195.5034 -2.9426881 68.043637 1872.5465 -2191.1146 + 200 0.02 0.076875906 -2194.3466 -3.1163511 50.052941 2986.3962 -2191.1181 + 250 0.025 0.076865073 -2193.3857 -3.2792912 35.095608 4003.4846 -2191.122 + 300 0.03 0.07675751 -2192.8611 -3.4452905 26.902302 4561.1429 -2191.1259 + 350 0.035 0.077351833 -2192.8579 -3.6193072 26.792986 4523.2641 -2191.1297 + 400 0.04 0.077672952 -2193.275 -3.7845654 33.199609 4108.3226 -2191.1336 + 450 0.045 0.077553541 -2193.9028 -3.9247064 42.874729 3267.3626 -2191.1373 + 500 0.05 0.076992612 -2194.5433 -4.0593738 52.743363 2317.5276 -2191.1413 + 550 0.055 0.074971927 -2195.0364 -4.1961092 60.332059 1620.5766 -2191.145 + 600 0.06 0.072652113 -2195.3154 -4.3458839 64.600641 1265.4418 -2191.1486 + 650 0.065 0.071405665 -2195.392 -4.5049778 65.734457 1221.4637 -2191.1521 + 700 0.07 0.072030336 -2195.337 -4.6535106 64.831697 1224.9583 -2191.1553 + 750 0.075 0.072468553 -2195.2702 -4.7829549 63.746912 1220.2392 -2191.1585 + 800 0.08 0.071546019 -2195.3271 -4.9161885 64.581676 1134.3858 -2191.1616 + 850 0.085 0.071414723 -2195.5618 -5.0652271 68.168922 864.52044 -2191.1648 + 900 0.09 0.073428472 -2195.9219 -5.219789 73.702937 556.14868 -2191.168 + 950 0.095 0.0745891 -2196.3065 -5.3782699 79.616238 -64.458151 -2191.1711 + 1000 0.1 0.074027925 -2196.6049 -5.5446493 84.189424 -640.80166 -2191.1746 +Loop time of 2.86028 on 4 procs for 1000 steps with 500 atoms + +Performance: 3.021 ns/day, 7.945 hours/ns, 349.616 timesteps/s +100.0% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0.6266 | 0.66621 | 0.68577 | 3.0 | 23.29 +Neigh | 0.0030921 | 0.0031813 | 0.0032392 | 0.1 | 0.11 +Comm | 0.1184 | 0.14023 | 0.18062 | 6.6 | 4.90 +Output | 0.0031779 | 0.0032207 | 0.0033176 | 0.1 | 0.11 +Modify | 2.0432 | 2.0445 | 2.0467 | 0.1 | 71.48 +Other | | 0.002987 | | | 0.10 + +Nlocal: 125 ave 128 max 121 min +Histogram: 1 0 1 0 0 0 0 0 0 2 +Nghost: 1324 ave 1331 max 1318 min +Histogram: 2 0 0 0 0 0 0 0 1 1 +Neighs: 6756.5 ave 6978 max 6543 min +Histogram: 1 1 0 0 0 0 0 0 1 1 +FullNghs: 13513 ave 13915 max 13042 min +Histogram: 1 1 0 0 0 0 0 0 0 2 + +Total # of neighbors = 54052 +Ave neighs/atom = 108.104 +Neighbor list builds = 6 +Dangerous builds = 0 + +Please see the log.cite file for references relevant to this simulation + +Total wall time: 0:00:02 diff --git a/examples/SPIN/cobalt_hcp/log.19Nov19.spin.cobalt_hcp.g++.1 b/examples/SPIN/cobalt_hcp/log.19Nov19.spin.cobalt_hcp.g++.1 deleted file mode 100644 index 781e2264a73438a7f71bbacbca8ec749fa19403e..0000000000000000000000000000000000000000 --- a/examples/SPIN/cobalt_hcp/log.19Nov19.spin.cobalt_hcp.g++.1 +++ /dev/null @@ -1,219 +0,0 @@ -LAMMPS (30 Oct 2019) -# hcp cobalt in a 3d periodic box - -clear -units metal -atom_style spin - -dimension 3 -boundary p p p - -# necessary for the serial algorithm (sametag) -atom_modify map array - -lattice hcp 2.5071 -Lattice spacing in x,y,z = 2.5071 4.34242 4.09408 -region box block 0.0 5.0 0.0 5.0 0.0 5.0 -create_box 1 box -Created orthogonal box = (0 0 0) to (12.5355 21.7121 20.4704) - 1 by 1 by 1 MPI processor grid -create_atoms 1 box -Created 500 atoms - create_atoms CPU = 0.00105 secs - -# setting mass, mag. moments, and interactions for hcp cobalt - -mass 1 58.93 - -set group all spin/random 31 1.72 - 500 settings made for spin/random -#set group all spin 1.72 0.0 0.0 1.0 -velocity all create 100 4928459 rot yes dist gaussian - -#pair_style hybrid/overlay eam/alloy spin/exchange 4.0 spin/neel 4.0 -pair_style hybrid/overlay eam/alloy spin/exchange 4.0 -pair_coeff * * eam/alloy Co_PurjaPun_2012.eam.alloy Co -pair_coeff * * spin/exchange exchange 4.0 -0.3593 1.135028015e-05 1.064568567 -#pair_coeff * * spin/neel neel 4.0 0.0048 0.234 1.168 2.6905 0.705 0.652 - -neighbor 0.1 bin -neigh_modify every 10 check yes delay 20 - -#fix 1 all precession/spin zeeman 1.0 0.0 0.0 1.0 -fix 1 all precession/spin anisotropy 0.01 0.0 0.0 1.0 -#fix 2 all langevin/spin 0.0 0.0 21 -fix 2 all langevin/spin 0.0 0.1 21 -fix 3 all nve/spin lattice moving - -timestep 0.0001 - - -compute out_mag all spin -compute out_pe all pe -compute out_ke all ke -compute out_temp all temp - -variable magz equal c_out_mag[3] -variable magnorm equal c_out_mag[4] -variable emag equal c_out_mag[5] -variable tmag equal c_out_mag[6] - -thermo_style custom step time v_magnorm v_emag temp press etotal -thermo 10 - -compute outsp all property/atom spx spy spz sp fmx fmy fmz -dump 1 all custom 100 dump_cobalt_hcp.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] - -run 1000 -Neighbor list info ... - update every 10 steps, delay 20 steps, check yes - max neighbors/atom: 2000, page size: 100000 - master list distance cutoff = 6.59954 - ghost atom cutoff = 6.59954 - binsize = 3.29977, bins = 4 7 7 - 2 neighbor lists, perpetual/occasional/extra = 2 0 0 - (1) pair eam/alloy, perpetual, half/full from (2) - attributes: half, newton on - pair build: halffull/newton - stencil: none - bin: none - (2) pair spin/exchange, perpetual - attributes: full, newton on - pair build: full/bin/atomonly - stencil: full/bin/3d - bin: standard -Per MPI rank memory allocation (min/avg/max) = 7.902 | 7.902 | 7.902 Mbytes -Step Time v_magnorm v_emag Temp Press TotEng - 0 0 0.076558814 -5.1073764 100.00543 -552.75983 -2189.4486 - 10 0.001 0.074494403 -6.2746901 100.01038 -1571.7966 -2190.0317 - 20 0.002 0.072366265 -7.4280779 99.885587 -2535.9845 -2190.5874 - 30 0.003 0.070127018 -8.5667999 99.611653 -3445.9872 -2191.1169 - 40 0.004 0.067755946 -9.6899272 99.164813 -4302.5715 -2191.6215 - 50 0.005 0.065261592 -10.79648 98.520535 -5107.2841 -2192.103 - 60 0.006 0.062676613 -11.885341 97.657148 -5862.7198 -2192.5638 - 70 0.007 0.060046709 -12.955115 96.558718 -6572.0571 -2193.0064 - 80 0.008 0.057417313 -14.004096 95.216748 -7238.1396 -2193.4327 - 90 0.009 0.054822275 -15.030416 93.630634 -7862.5226 -2193.8437 - 100 0.01 0.052277835 -16.032345 91.80711 -8445.2646 -2194.2391 - 110 0.011 0.049783153 -17.008652 89.759163 -8985.5937 -2194.6181 - 120 0.012 0.047326373 -17.958895 87.504922 -9483.1141 -2194.98 - 130 0.013 0.044893289 -18.883574 85.066818 -9938.8838 -2195.3247 - 140 0.014 0.042474822 -19.784052 82.471014 -10355.911 -2195.6533 - 150 0.015 0.040070404 -20.662271 79.746901 -10739.081 -2195.9677 - 160 0.016 0.037686856 -21.520294 76.926428 -11094.793 -2196.2709 - 170 0.017 0.035334961 -22.359822 74.043181 -11430.247 -2196.5659 - 180 0.018 0.033026799 -23.181822 71.131269 -11752.268 -2196.8556 - 190 0.019 0.030775544 -23.986406 68.224204 -12065.774 -2197.1412 - 200 0.02 0.028597121 -24.773013 65.353995 -12372.712 -2197.4226 - 210 0.021 0.026511775 -25.540835 62.55053 -12672.055 -2197.6975 - 220 0.022 0.02454383 -26.289327 59.841288 -12961.112 -2197.9631 - 230 0.023 0.02271918 -27.018625 57.251361 -13237.544 -2198.2165 - 240 0.024 0.021061271 -27.729714 54.80373 -13501.028 -2198.4564 - 250 0.025 0.019587072 -28.42449 52.519717 -13754.325 -2198.6833 - 260 0.026 0.018304494 -29.105398 50.419388 -14002.718 -2198.8994 - 270 0.027 0.017211977 -29.775134 48.521812 -14253.089 -2199.1079 - 280 0.028 0.016300002 -30.436204 46.845075 -14512.437 -2199.3119 - 290 0.029 0.015553519 -31.090499 45.405985 -14786.53 -2199.5143 - 300 0.03 0.014954102 -31.739026 44.219544 -15079.165 -2199.7168 - 310 0.031 0.014481189 -32.381585 43.298175 -15391.531 -2199.9198 - 320 0.032 0.014112494 -33.016984 42.650874 -15722.828 -2200.1226 - 330 0.033 0.013824206 -33.643289 42.282535 -16070.874 -2200.324 - 340 0.034 0.013591568 -34.258323 42.19365 -16433.065 -2200.5226 - 350 0.035 0.013390035 -34.860184 42.380506 -16807.186 -2200.7174 - 360 0.036 0.01319679 -35.447655 42.835832 -17191.816 -2200.9077 - 370 0.037 0.012992271 -36.020512 43.549656 -17586.676 -2201.0935 - 380 0.038 0.012761486 -36.579332 44.510078 -17991.857 -2201.2754 - 390 0.039 0.012494918 -37.125414 45.703757 -18407.738 -2201.4538 - 400 0.04 0.0121888 -37.660321 47.115967 -18834.276 -2201.6292 - 410 0.041 0.011844474 -38.185489 48.730291 -19270.674 -2201.8019 - 420 0.042 0.011466715 -38.70192 50.528119 -19715.276 -2201.9716 - 430 0.043 0.011061388 -39.21005 52.488204 -20165.66 -2202.1377 - 440 0.044 0.010633241 -39.709778 54.586528 -20618.997 -2202.2998 - 450 0.045 0.010184696 -40.200724 56.79654 -21072.538 -2202.4571 - 460 0.046 0.0097161044 -40.682449 59.089699 -21523.873 -2202.6094 - 470 0.047 0.0092271788 -41.154614 61.436133 -21970.922 -2202.7565 - 480 0.048 0.0087187266 -41.617256 63.805414 -22412.32 -2202.8989 - 490 0.049 0.0081937768 -42.070708 66.167399 -22847.061 -2203.037 - 500 0.05 0.0076576327 -42.51563 68.493235 -23274.619 -2203.172 - 510 0.051 0.0071170477 -42.952841 70.756444 -23694.559 -2203.3046 - 520 0.052 0.006579078 -43.383338 72.933996 -24106.717 -2203.4358 - 530 0.053 0.006050144 -43.807962 75.007131 -24510.338 -2203.5662 - 540 0.054 0.0055354475 -44.227552 76.961803 -24904.495 -2203.6957 - 550 0.055 0.0050386503 -44.64268 78.788647 -25287.341 -2203.8241 - 560 0.056 0.0045617699 -45.053996 80.4825 -25657.11 -2203.9504 - 570 0.057 0.0041054334 -45.461923 82.041527 -26011.443 -2204.0737 - 580 0.058 0.003669689 -45.866895 83.466142 -26348.265 -2204.1931 - 590 0.059 0.0032553824 -46.269219 84.757926 -26665.834 -2204.3077 - 600 0.06 0.0028655752 -46.669125 85.918711 -26963.24 -2204.4173 - 610 0.061 0.0025060765 -47.066641 86.95 -27240.331 -2204.5218 - 620 0.062 0.0021839971 -47.461566 87.852838 -27497.728 -2204.6218 - 630 0.063 0.0019039581 -47.853462 88.628142 -27736.503 -2204.7177 - 640 0.064 0.0016633855 -48.241747 89.277364 -27957.91 -2204.81 - 650 0.065 0.0014502904 -48.625803 89.803307 -28163.11 -2204.899 - 660 0.066 0.0012463786 -49.005026 90.210807 -28352.881 -2204.9847 - 670 0.067 0.0010345087 -49.378935 90.507107 -28527.721 -2205.0668 - 680 0.068 0.00080788134 -49.747325 90.701795 -28688.395 -2205.1453 - 690 0.069 0.000586442 -50.110227 90.80636 -28836.094 -2205.22 - 700 0.07 0.00046855102 -50.467799 90.833539 -28972.361 -2205.2911 - 710 0.071 0.00061091693 -50.82044 90.796649 -29099.44 -2205.3592 - 720 0.072 0.00094960177 -51.168606 90.709122 -29219.676 -2205.4249 - 730 0.073 0.0013742455 -51.512913 90.584346 -29335.643 -2205.4887 - 740 0.074 0.0018397629 -51.853957 90.435783 -29449.521 -2205.5511 - 750 0.075 0.0023216474 -52.192407 90.277231 -29563.316 -2205.6124 - 760 0.076 0.0028000512 -52.528883 90.123061 -29678.726 -2205.6729 - 770 0.077 0.0032569295 -52.863859 89.98824 -29797.079 -2205.7329 - 780 0.078 0.0036765431 -53.197843 89.888047 -29919.964 -2205.7925 - 790 0.079 0.0040467094 -53.530921 89.837568 -30048.271 -2205.8521 - 800 0.08 0.0043597837 -53.862938 89.850978 -30182.622 -2205.9119 - 810 0.081 0.0046129296 -54.193489 89.940884 -30323.293 -2205.9718 - 820 0.082 0.0048076151 -54.522077 90.117797 -30470.468 -2206.0321 - 830 0.083 0.004948533 -54.84813 90.389814 -30624.056 -2206.0926 - 840 0.084 0.0050423324 -55.171024 90.762454 -30783.658 -2206.1532 - 850 0.085 0.0050965581 -55.490357 91.238681 -30949.141 -2206.2139 - 860 0.086 0.0051190641 -55.805904 91.818973 -31120.5 -2206.2745 - 870 0.087 0.0051180301 -56.117429 92.501449 -31297.412 -2206.3349 - 880 0.088 0.0051024116 -56.424751 93.281992 -31479.436 -2206.3949 - 890 0.089 0.005082454 -56.727832 94.154367 -31666.293 -2206.4544 - 900 0.09 0.0050697645 -57.026442 95.110386 -31857.043 -2206.513 - 910 0.091 0.0050765431 -57.320291 96.140056 -32050.436 -2206.5703 - 920 0.092 0.0051139309 -57.609075 97.231838 -32245.079 -2206.6257 - 930 0.093 0.0051899535 -57.89236 98.372982 -32439.141 -2206.6788 - 940 0.094 0.0053078572 -58.169742 99.54995 -32630.727 -2206.7288 - 950 0.095 0.0054654923 -58.44083 100.74893 -32817.882 -2206.7752 - 960 0.096 0.0056558757 -58.705483 101.95638 -32999.116 -2206.8176 - 970 0.097 0.0058685513 -58.963698 103.15953 -33173.159 -2206.8557 - 980 0.098 0.0060912487 -59.215624 104.34681 -33338.961 -2206.8893 - 990 0.099 0.0063114886 -59.461806 105.50819 -33496.345 -2206.9188 - 1000 0.1 0.0065179843 -59.702883 106.63524 -33645.259 -2206.9444 -Loop time of 5.20295 on 1 procs for 1000 steps with 500 atoms - -Performance: 1.661 ns/day, 14.453 hours/ns, 192.199 timesteps/s -100.0% CPU use with 1 MPI tasks x no OpenMP threads - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Pair | 2.6241 | 2.6241 | 2.6241 | 0.0 | 50.43 -Neigh | 0.01424 | 0.01424 | 0.01424 | 0.0 | 0.27 -Comm | 0.041207 | 0.041207 | 0.041207 | 0.0 | 0.79 -Output | 0.0090086 | 0.0090086 | 0.0090086 | 0.0 | 0.17 -Modify | 2.5084 | 2.5084 | 2.5084 | 0.0 | 48.21 -Other | | 0.006008 | | | 0.12 - -Nlocal: 500 ave 500 max 500 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Nghost: 2442 ave 2442 max 2442 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Neighs: 27581 ave 27581 max 27581 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -FullNghs: 55162 ave 55162 max 55162 min -Histogram: 1 0 0 0 0 0 0 0 0 0 - -Total # of neighbors = 55162 -Ave neighs/atom = 110.324 -Neighbor list builds = 7 -Dangerous builds = 0 - -Please see the log.cite file for references relevant to this simulation - -Total wall time: 0:00:05 diff --git a/examples/SPIN/cobalt_hcp/log.19Nov19.spin.cobalt_hcp.g++.4 b/examples/SPIN/cobalt_hcp/log.19Nov19.spin.cobalt_hcp.g++.4 deleted file mode 100644 index de00d5be320fbd1d24663e2c1c303edcb428326c..0000000000000000000000000000000000000000 --- a/examples/SPIN/cobalt_hcp/log.19Nov19.spin.cobalt_hcp.g++.4 +++ /dev/null @@ -1,219 +0,0 @@ -LAMMPS (30 Oct 2019) -# hcp cobalt in a 3d periodic box - -clear -units metal -atom_style spin - -dimension 3 -boundary p p p - -# necessary for the serial algorithm (sametag) -atom_modify map array - -lattice hcp 2.5071 -Lattice spacing in x,y,z = 2.5071 4.34242 4.09408 -region box block 0.0 5.0 0.0 5.0 0.0 5.0 -create_box 1 box -Created orthogonal box = (0 0 0) to (12.5355 21.7121 20.4704) - 1 by 2 by 2 MPI processor grid -create_atoms 1 box -Created 500 atoms - create_atoms CPU = 0.00101995 secs - -# setting mass, mag. moments, and interactions for hcp cobalt - -mass 1 58.93 - -set group all spin/random 31 1.72 - 500 settings made for spin/random -#set group all spin 1.72 0.0 0.0 1.0 -velocity all create 100 4928459 rot yes dist gaussian - -#pair_style hybrid/overlay eam/alloy spin/exchange 4.0 spin/neel 4.0 -pair_style hybrid/overlay eam/alloy spin/exchange 4.0 -pair_coeff * * eam/alloy Co_PurjaPun_2012.eam.alloy Co -pair_coeff * * spin/exchange exchange 4.0 -0.3593 1.135028015e-05 1.064568567 -#pair_coeff * * spin/neel neel 4.0 0.0048 0.234 1.168 2.6905 0.705 0.652 - -neighbor 0.1 bin -neigh_modify every 10 check yes delay 20 - -#fix 1 all precession/spin zeeman 1.0 0.0 0.0 1.0 -fix 1 all precession/spin anisotropy 0.01 0.0 0.0 1.0 -#fix 2 all langevin/spin 0.0 0.0 21 -fix 2 all langevin/spin 0.0 0.1 21 -fix 3 all nve/spin lattice moving - -timestep 0.0001 - - -compute out_mag all spin -compute out_pe all pe -compute out_ke all ke -compute out_temp all temp - -variable magz equal c_out_mag[3] -variable magnorm equal c_out_mag[4] -variable emag equal c_out_mag[5] -variable tmag equal c_out_mag[6] - -thermo_style custom step time v_magnorm v_emag temp press etotal -thermo 10 - -compute outsp all property/atom spx spy spz sp fmx fmy fmz -dump 1 all custom 100 dump_cobalt_hcp.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] - -run 1000 -Neighbor list info ... - update every 10 steps, delay 20 steps, check yes - max neighbors/atom: 2000, page size: 100000 - master list distance cutoff = 6.59954 - ghost atom cutoff = 6.59954 - binsize = 3.29977, bins = 4 7 7 - 2 neighbor lists, perpetual/occasional/extra = 2 0 0 - (1) pair eam/alloy, perpetual, half/full from (2) - attributes: half, newton on - pair build: halffull/newton - stencil: none - bin: none - (2) pair spin/exchange, perpetual - attributes: full, newton on - pair build: full/bin/atomonly - stencil: full/bin/3d - bin: standard -Per MPI rank memory allocation (min/avg/max) = 7.814 | 7.814 | 7.815 Mbytes -Step Time v_magnorm v_emag Temp Press TotEng - 0 0 0.076558814 -5.1073764 100.00543 -552.75983 -2190.3478 - 10 0.001 0.074494512 -6.2728301 99.980769 -1570.0726 -2191.5261 - 20 0.002 0.072367013 -7.4259977 99.847801 -2531.5119 -2192.6655 - 30 0.003 0.070129365 -8.566306 99.586282 -3438.1309 -2193.7672 - 40 0.004 0.067761178 -9.6929189 99.171132 -4291.017 -2194.8323 - 50 0.005 0.065270916 -10.8048 98.575397 -5091.9111 -2195.8628 - 60 0.006 0.062690557 -11.900573 97.773618 -5843.4528 -2196.8612 - 70 0.007 0.060064592 -12.978381 96.745047 -6548.726 -2197.8306 - 80 0.008 0.05743694 -14.035923 95.476292 -7210.2954 -2198.773 - 90 0.009 0.054839883 -15.07074 93.963026 -7829.4252 -2199.689 - 100 0.01 0.052288504 -16.08066 92.210482 -8405.9983 -2200.5773 - 110 0.011 0.049782155 -17.064251 90.232741 -8939.3051 -2201.4357 - 120 0.012 0.047311759 -18.021135 88.051042 -9429.1353 -2202.2626 - 130 0.013 0.044869196 -18.952065 85.691573 -9876.5628 -2203.0575 - 140 0.014 0.042453961 -19.858739 83.18315 -10284.249 -2203.8215 - 150 0.015 0.040074171 -20.743348 80.555177 -10656.417 -2204.5569 - 160 0.016 0.037742459 -21.608 77.836156 -10998.818 -2205.2677 - 170 0.017 0.035470168 -22.454209 75.052994 -11318.525 -2205.9587 - 180 0.018 0.033263447 -23.282658 72.231211 -11623.118 -2206.6354 - 190 0.019 0.031122821 -24.093311 69.395936 -11919.248 -2207.3023 - 200 0.02 0.029045634 -24.88579 66.573223 -12211.306 -2207.9613 - 210 0.021 0.027029857 -25.659817 63.791041 -12500.812 -2208.6115 - 220 0.022 0.025077742 -26.415541 61.079413 -12787.018 -2209.2498 - 230 0.023 0.023198048 -27.153652 58.469604 -13068.277 -2209.8722 - 240 0.024 0.02140599 -27.875313 55.992687 -13343.621 -2210.4756 - 250 0.025 0.019720922 -28.581973 53.678031 -13613.86 -2211.0588 - 260 0.026 0.018162738 -29.275283 51.552191 -13882.15 -2211.6232 - 270 0.027 0.016748514 -29.956802 49.638467 -14153.137 -2212.1718 - 280 0.028 0.01549075 -30.628043 47.957071 -14432.246 -2212.7087 - 290 0.029 0.014397611 -31.290177 46.525552 -14724.005 -2213.2371 - 300 0.03 0.013474315 -31.943984 45.359085 -15031.315 -2213.759 - 310 0.031 0.012723957 -32.589853 44.47023 -15355.595 -2214.275 - 320 0.032 0.012146358 -33.227585 43.868153 -15696.845 -2214.7851 - 330 0.033 0.011734827 -33.856656 43.557623 -16054.887 -2215.289 - 340 0.034 0.011472508 -34.476313 43.538346 -16429.77 -2215.7871 - 350 0.035 0.011330772 -35.085716 43.805034 -16821.627 -2216.2802 - 360 0.036 0.011271169 -35.684147 44.348312 -17230.21 -2216.7687 - 370 0.037 0.01125027 -36.271215 45.156046 -17654.485 -2217.2524 - 380 0.038 0.011225354 -36.847053 46.214576 -18092.623 -2217.7301 - 390 0.039 0.011159026 -37.412284 47.509345 -18542.156 -2218.2003 - 400 0.04 0.011022073 -37.967916 49.024843 -19000.554 -2218.6614 - 410 0.041 0.01079477 -38.515123 50.744046 -19465.713 -2219.1128 - 420 0.042 0.010467095 -39.054921 52.647653 -19935.873 -2219.5544 - 430 0.043 0.010038219 -39.588034 54.713405 -20409.666 -2219.9869 - 440 0.044 0.0095155267 -40.114703 56.915658 -20885.556 -2220.4109 - 450 0.045 0.0089134996 -40.634722 59.225397 -21361.621 -2220.8268 - 460 0.046 0.0082528918 -41.147681 61.610799 -21835.762 -2221.2347 - 470 0.047 0.0075606723 -41.653088 64.038349 -22305.687 -2221.6343 - 480 0.048 0.0068707613 -42.150486 66.474377 -22768.948 -2222.0253 - 490 0.049 0.0062249854 -42.639704 68.886721 -23223.418 -2222.4076 - 500 0.05 0.0056723593 -43.120772 71.24617 -23667.077 -2222.7814 - 510 0.051 0.00526312 -43.59404 73.527392 -24098.459 -2223.147 - 520 0.052 0.0050342241 -44.059917 75.709206 -24516.163 -2223.5051 - 530 0.053 0.0049906301 -44.518898 77.774314 -24919.192 -2223.8564 - 540 0.054 0.0050976586 -44.971364 79.708763 -25306.611 -2224.2014 - 550 0.055 0.0052941974 -45.417577 81.501347 -25677.67 -2224.5405 - 560 0.056 0.0055157717 -45.857628 83.143173 -26031.673 -2224.8736 - 570 0.057 0.0057113414 -46.291426 84.627457 -26367.904 -2225.2003 - 580 0.058 0.0058493207 -46.718709 85.949497 -26685.6 -2225.52 - 590 0.059 0.0059162201 -47.139052 87.10679 -26984.124 -2225.8316 - 600 0.06 0.0059118584 -47.551892 88.099176 -27263.145 -2226.1347 - 610 0.061 0.005843747 -47.956571 88.928929 -27522.773 -2226.4287 - 620 0.062 0.0057222223 -48.352422 89.600763 -27763.549 -2226.7139 - 630 0.063 0.0055570967 -48.738876 90.12173 -27986.321 -2226.9905 - 640 0.064 0.0053558993 -49.115723 90.501081 -28192.238 -2227.2593 - 650 0.065 0.0051233209 -49.483122 90.750056 -28382.3 -2227.5205 - 660 0.066 0.0048614512 -49.841791 90.881635 -28557.623 -2227.7746 - 670 0.067 0.0045706003 -50.192974 90.910245 -28719.422 -2228.0219 - 680 0.068 0.0042506564 -50.538196 90.851397 -28868.809 -2228.2627 - 690 0.069 0.0039028575 -50.879364 90.721317 -29007.619 -2228.4973 - 700 0.07 0.0035319814 -51.218193 90.536521 -29137.623 -2228.7265 - 710 0.071 0.0031491486 -51.556251 90.313501 -29261.193 -2228.9511 - 720 0.072 0.0027758205 -51.894643 90.068503 -29380.924 -2229.1724 - 730 0.073 0.002449449 -52.233987 89.817462 -29499.606 -2229.3917 - 740 0.074 0.0022276613 -52.574465 89.57612 -29620.196 -2229.6103 - 750 0.075 0.0021767124 -52.915641 89.360246 -29744.882 -2229.829 - 760 0.076 0.0023310362 -53.256843 89.185838 -29875.573 -2230.0485 - 770 0.077 0.0026637349 -53.597197 89.069228 -30013.477 -2230.2685 - 780 0.078 0.0031129938 -53.93565 89.026943 -30158.812 -2230.4882 - 790 0.079 0.0036204667 -54.271339 89.075322 -30311.602 -2230.7066 - 800 0.08 0.0041448552 -54.603455 89.229912 -30471.244 -2230.9226 - 810 0.081 0.0046613106 -54.931421 89.504766 -30636.938 -2231.1352 - 820 0.082 0.0051580947 -55.255056 89.911726 -30808.087 -2231.3434 - 830 0.083 0.0056329652 -55.574491 90.459766 -30984.153 -2231.5469 - 840 0.084 0.0060893356 -55.890024 91.154456 -31164.372 -2231.7452 - 850 0.085 0.0065324419 -56.202052 91.997528 -31347.792 -2231.9379 - 860 0.086 0.0069661977 -56.511206 92.986622 -31533.977 -2232.1249 - 870 0.087 0.0073913051 -56.817814 94.115192 -31721.92 -2232.306 - 880 0.088 0.0078048547 -57.122061 95.372548 -31910.795 -2232.4809 - 890 0.089 0.008201165 -57.423984 96.744135 -32100.108 -2232.65 - 900 0.09 0.0085732702 -57.723377 98.212046 -32289.532 -2232.8136 - 910 0.091 0.0089144724 -58.019938 99.755667 -32479.154 -2232.9728 - 920 0.092 0.0092194916 -58.313266 101.35254 -32669.227 -2233.1285 - 930 0.093 0.0094849872 -58.602956 102.97932 -32860.091 -2233.2822 - 940 0.094 0.0097093572 -58.888668 104.61271 -33051.981 -2233.4348 - 950 0.095 0.0098920175 -59.169925 106.23045 -33244.279 -2233.5871 - 960 0.096 0.01003244 -59.44662 107.81212 -33436.562 -2233.7396 - 970 0.097 0.010129313 -59.718668 109.33976 -33627.714 -2233.8925 - 980 0.098 0.010180127 -59.986126 110.79823 -33816.218 -2234.0455 - 990 0.099 0.010181304 -60.24929 112.17528 -34000.522 -2234.1984 - 1000 0.1 0.01012881 -60.508632 113.46137 -34179.052 -2234.3508 -Loop time of 2.93788 on 4 procs for 1000 steps with 500 atoms - -Performance: 2.941 ns/day, 8.161 hours/ns, 340.381 timesteps/s -100.0% CPU use with 4 MPI tasks x no OpenMP threads - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Pair | 0.72349 | 0.73783 | 0.7554 | 1.3 | 25.11 -Neigh | 0.00353 | 0.0036981 | 0.0038559 | 0.2 | 0.13 -Comm | 0.12285 | 0.14476 | 0.16041 | 3.6 | 4.93 -Output | 0.0046515 | 0.0047909 | 0.0050418 | 0.2 | 0.16 -Modify | 2.0407 | 2.0439 | 2.0482 | 0.2 | 69.57 -Other | | 0.00288 | | | 0.10 - -Nlocal: 125 ave 136 max 119 min -Histogram: 1 1 1 0 0 0 0 0 0 1 -Nghost: 1324 ave 1331 max 1310 min -Histogram: 1 0 0 0 0 0 0 0 2 1 -Neighs: 6897.25 ave 7552 max 6604 min -Histogram: 2 1 0 0 0 0 0 0 0 1 -FullNghs: 13794.5 ave 15117 max 13164 min -Histogram: 2 0 1 0 0 0 0 0 0 1 - -Total # of neighbors = 55178 -Ave neighs/atom = 110.356 -Neighbor list builds = 7 -Dangerous builds = 0 - -Please see the log.cite file for references relevant to this simulation - -Total wall time: 0:00:02 diff --git a/examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_cut.g++.1 b/examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_cut.g++.1 similarity index 75% rename from examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_cut.g++.1 rename to examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_cut.g++.1 index c2af23d16773e899bf63ec83bfdd5a1cd8c47de6..db58eec01d1061cb904f6d0da4ed462f6fa6ec15 100644 --- a/examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_cut.g++.1 +++ b/examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_cut.g++.1 @@ -1,7 +1,11 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bcc iron in a 3d periodic box clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task units metal atom_style spin @@ -19,7 +23,7 @@ Created orthogonal box = (0 0 0) to (34.398 34.398 34.398) 1 by 1 by 1 MPI processor grid create_atoms 1 box Created 3456 atoms - create_atoms CPU = 0.000741005 secs + create_atoms CPU = 0.00106382 secs # setting mass, mag. moments, and interactions for bcc iron @@ -88,23 +92,23 @@ Neighbor list info ... bin: none Per MPI rank memory allocation (min/avg/max) = 13.4 | 13.4 | 13.4 Mbytes Step Time v_magx v_magy v_magz v_magnorm v_tmag v_emag PotEng TotEng - 0 0 -1 0 0 1 1.0737264e-35 -768.37511 -15175.868 -15131.207 - 50 0.005 -1 -2.7722752e-10 -2.1828666e-10 1 6.8846921e-09 -768.35793 -15174.244 -15131.215 - 100 0.01 -1 -2.0983066e-09 -1.7330951e-09 1 1.0038885e-08 -768.30868 -15169.656 -15131.24 -Loop time of 7.86359 on 1 procs for 100 steps with 3456 atoms + 0 0 -1 0 0 1 1.0737264e-35 -384.18755 -15175.868 -15131.207 + 50 0.005 -1 -2.7725069e-10 -2.182903e-10 1 6.8851185e-09 -384.17896 -15174.244 -15131.207 + 100 0.01 -1 -2.0990209e-09 -1.7332235e-09 1 1.0040825e-08 -384.15433 -15169.655 -15131.207 +Loop time of 7.47017 on 1 procs for 100 steps with 3456 atoms -Performance: 0.110 ns/day, 218.433 hours/ns, 12.717 timesteps/s -100.0% CPU use with 1 MPI tasks x no OpenMP threads +Performance: 0.116 ns/day, 207.505 hours/ns, 13.387 timesteps/s +99.9% CPU use with 1 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 3.6134 | 3.6134 | 3.6134 | 0.0 | 45.95 +Pair | 3.1998 | 3.1998 | 3.1998 | 0.0 | 42.83 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.014062 | 0.014062 | 0.014062 | 0.0 | 0.18 -Output | 0.006057 | 0.006057 | 0.006057 | 0.0 | 0.08 -Modify | 4.226 | 4.226 | 4.226 | 0.0 | 53.74 -Other | | 0.004064 | | | 0.05 +Comm | 0.015094 | 0.015094 | 0.015094 | 0.0 | 0.20 +Output | 0.006531 | 0.006531 | 0.006531 | 0.0 | 0.09 +Modify | 4.2443 | 4.2443 | 4.2443 | 0.0 | 56.82 +Other | | 0.004467 | | | 0.06 Nlocal: 3456 ave 3456 max 3456 min Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_cut.g++.4 b/examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_cut.g++.4 similarity index 75% rename from examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_cut.g++.4 rename to examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_cut.g++.4 index 1f8157bb29a9f32f791e77c02e53928626895db8..a014aae3cd3b1c84ec7e8a5cd3cfbbd1602fe525 100644 --- a/examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_cut.g++.4 +++ b/examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_cut.g++.4 @@ -1,7 +1,11 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bcc iron in a 3d periodic box clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task units metal atom_style spin @@ -19,7 +23,7 @@ Created orthogonal box = (0 0 0) to (34.398 34.398 34.398) 1 by 2 by 2 MPI processor grid create_atoms 1 box Created 3456 atoms - create_atoms CPU = 0.00090003 secs + create_atoms CPU = 0.00132084 secs # setting mass, mag. moments, and interactions for bcc iron @@ -88,23 +92,23 @@ Neighbor list info ... bin: none Per MPI rank memory allocation (min/avg/max) = 9.217 | 9.217 | 9.217 Mbytes Step Time v_magx v_magy v_magz v_magnorm v_tmag v_emag PotEng TotEng - 0 0 -1 0 0 1 1.0737264e-35 -768.37511 -15560.055 -15515.394 - 50 0.005 -1 9.6204015e-11 -3.3767807e-10 1 6.6905249e-09 -768.35767 -15558.438 -15515.394 - 100 0.01 -1 7.8881609e-10 -2.7017321e-09 1 9.8111281e-09 -768.30769 -15553.868 -15515.394 -Loop time of 2.29116 on 4 procs for 100 steps with 3456 atoms + 0 0 -1 0 0 1 1.0737264e-35 -384.18755 -15175.868 -15131.207 + 50 0.005 -1 9.6205501e-11 -3.3769045e-10 1 6.6909444e-09 -384.17884 -15174.259 -15131.207 + 100 0.01 -1 7.8887025e-10 -2.7021386e-09 1 9.8130686e-09 -384.15383 -15169.712 -15131.207 +Loop time of 2.27865 on 4 procs for 100 steps with 3456 atoms -Performance: 0.377 ns/day, 63.643 hours/ns, 43.646 timesteps/s -99.9% CPU use with 4 MPI tasks x no OpenMP threads +Performance: 0.379 ns/day, 63.296 hours/ns, 43.886 timesteps/s +99.5% CPU use with 4 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 0.92259 | 0.92963 | 0.93393 | 0.4 | 40.57 +Pair | 0.80959 | 0.8184 | 0.82996 | 0.9 | 35.92 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.02284 | 0.027597 | 0.035185 | 2.8 | 1.20 -Output | 0.0018489 | 0.0018544 | 0.0018642 | 0.0 | 0.08 -Modify | 1.3296 | 1.3303 | 1.3308 | 0.0 | 58.06 -Other | | 0.001818 | | | 0.08 +Comm | 0.035061 | 0.046857 | 0.055443 | 3.9 | 2.06 +Output | 0.0018592 | 0.0018642 | 0.0018783 | 0.0 | 0.08 +Modify | 1.4085 | 1.4095 | 1.41 | 0.1 | 61.86 +Other | | 0.002023 | | | 0.09 Nlocal: 864 ave 864 max 864 min Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_ewald.g++.1 b/examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_ewald.g++.1 similarity index 72% rename from examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_ewald.g++.1 rename to examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_ewald.g++.1 index 9a38445af5c112676ec020710edf5516e1b82769..f6c015bb20defef8915767cfe263ab3ac0c12b06 100644 --- a/examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_ewald.g++.1 +++ b/examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_ewald.g++.1 @@ -1,7 +1,11 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bcc iron in a 3d periodic box clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task units metal atom_style spin @@ -19,7 +23,7 @@ Created orthogonal box = (0 0 0) to (34.398 34.398 34.398) 1 by 1 by 1 MPI processor grid create_atoms 1 box Created 3456 atoms - create_atoms CPU = 0.00166988 secs + create_atoms CPU = 0.00187302 secs # setting mass, mag. moments, and interactions for bcc iron @@ -67,10 +71,10 @@ dump 1 all custom 100 dump_iron.lammpstrj type x y z c_outsp[1] c_outsp[2] c_o run 100 EwaldDipoleSpin initialization ... - using 12-bit tables for long-range coulomb (../kspace.cpp:323) + using 12-bit tables for long-range coulomb (../kspace.cpp:332) G vector (1/distance) = 0.324623 - estimated absolute RMS force accuracy = 9.55526e-84 - estimated relative force accuracy = 6.63576e-85 + estimated absolute RMS force accuracy = 1.69788e-59 + estimated relative force accuracy = 1.17911e-60 KSpace vectors: actual max1d max3d = 2084 10 4630 kxmax kymax kzmax = 10 10 10 Neighbor list info ... @@ -97,24 +101,24 @@ Neighbor list info ... bin: none Per MPI rank memory allocation (min/avg/max) = 30.07 | 30.07 | 30.07 Mbytes Step Time v_magx v_magy v_magz v_magnorm v_tmag v_emag PotEng TotEng - 0 0 -1 0 0 1 2.5872886e-37 -767.88567 -15175.39 -15130.729 - 50 0.005 -1 4.3660916e-09 -2.1918692e-09 1 5.3480999e-10 -767.86847 -15173.766 -15130.738 - 100 0.01 -1 9.9854966e-09 -4.2823677e-09 1 2.3267629e-09 -767.81917 -15169.178 -15130.762 -Loop time of 24.9345 on 1 procs for 100 steps with 3456 atoms + 0 0 -1 0 0 1 2.5872886e-37 -383.94283 -15175.635 -15130.974 + 50 0.005 -1 4.3660908e-09 -2.1918693e-09 1 5.3484784e-10 -383.93423 -15174.011 -15130.974 + 100 0.01 -1 9.9854743e-09 -4.282369e-09 1 2.3273467e-09 -383.90957 -15169.421 -15130.974 +Loop time of 24.8682 on 1 procs for 100 steps with 3456 atoms -Performance: 0.035 ns/day, 692.624 hours/ns, 4.011 timesteps/s -100.0% CPU use with 1 MPI tasks x no OpenMP threads +Performance: 0.035 ns/day, 690.783 hours/ns, 4.021 timesteps/s +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 4.8022 | 4.8022 | 4.8022 | 0.0 | 19.26 -Kspace | 10.337 | 10.337 | 10.337 | 0.0 | 41.46 +Pair | 4.6097 | 4.6097 | 4.6097 | 0.0 | 18.54 +Kspace | 10.303 | 10.303 | 10.303 | 0.0 | 41.43 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.013856 | 0.013856 | 0.013856 | 0.0 | 0.06 -Output | 0.007138 | 0.007138 | 0.007138 | 0.0 | 0.03 -Modify | 9.7705 | 9.7705 | 9.7705 | 0.0 | 39.18 -Other | | 0.004077 | | | 0.02 +Comm | 0.015258 | 0.015258 | 0.015258 | 0.0 | 0.06 +Output | 0.006583 | 0.006583 | 0.006583 | 0.0 | 0.03 +Modify | 9.9285 | 9.9285 | 9.9285 | 0.0 | 39.92 +Other | | 0.004751 | | | 0.02 Nlocal: 3456 ave 3456 max 3456 min Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_ewald.g++.4 b/examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_ewald.g++.4 similarity index 72% rename from examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_ewald.g++.4 rename to examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_ewald.g++.4 index 306799f57651676e6de0729655d11ddc12a25b69..b70221f6031195133a27159fe75135a5f41ee716 100644 --- a/examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_ewald.g++.4 +++ b/examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_ewald.g++.4 @@ -1,7 +1,11 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bcc iron in a 3d periodic box clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task units metal atom_style spin @@ -19,7 +23,7 @@ Created orthogonal box = (0 0 0) to (34.398 34.398 34.398) 1 by 2 by 2 MPI processor grid create_atoms 1 box Created 3456 atoms - create_atoms CPU = 0.00088191 secs + create_atoms CPU = 0.000773907 secs # setting mass, mag. moments, and interactions for bcc iron @@ -67,10 +71,10 @@ dump 1 all custom 100 dump_iron.lammpstrj type x y z c_outsp[1] c_outsp[2] c_o run 100 EwaldDipoleSpin initialization ... - using 12-bit tables for long-range coulomb (../kspace.cpp:323) + using 12-bit tables for long-range coulomb (../kspace.cpp:332) G vector (1/distance) = 0.324623 - estimated absolute RMS force accuracy = 9.29828e-84 - estimated relative force accuracy = 6.4573e-85 + estimated absolute RMS force accuracy = 2.94041e-64 + estimated relative force accuracy = 2.042e-65 KSpace vectors: actual max1d max3d = 2084 10 4630 kxmax kymax kzmax = 10 10 10 Neighbor list info ... @@ -97,24 +101,24 @@ Neighbor list info ... bin: none Per MPI rank memory allocation (min/avg/max) = 25.89 | 25.89 | 25.89 Mbytes Step Time v_magx v_magy v_magz v_magnorm v_tmag v_emag PotEng TotEng - 0 0 -1 0 0 1 3.5107565e-37 -767.88567 -15559.577 -15514.916 - 50 0.005 -1 4.3196063e-09 -2.1966927e-09 1 5.1719577e-10 -767.86822 -15557.96 -15514.916 - 100 0.01 -1 9.7636593e-09 -4.3236953e-09 1 2.2443181e-09 -767.81819 -15553.39 -15514.916 -Loop time of 6.80139 on 4 procs for 100 steps with 3456 atoms + 0 0 -1 0 0 1 3.5107565e-37 -383.94283 -15175.635 -15130.974 + 50 0.005 -1 4.3196054e-09 -2.1966927e-09 1 5.1723249e-10 -383.93411 -15174.026 -15130.974 + 100 0.01 -1 9.7636345e-09 -4.3236965e-09 1 2.2448849e-09 -383.90908 -15169.479 -15130.974 +Loop time of 7.03264 on 4 procs for 100 steps with 3456 atoms -Performance: 0.127 ns/day, 188.927 hours/ns, 14.703 timesteps/s -100.0% CPU use with 4 MPI tasks x no OpenMP threads +Performance: 0.123 ns/day, 195.351 hours/ns, 14.219 timesteps/s +99.9% CPU use with 4 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 1.248 | 1.2649 | 1.2816 | 1.1 | 18.60 -Kspace | 2.523 | 2.5743 | 2.6505 | 3.0 | 37.85 +Pair | 1.1743 | 1.207 | 1.2416 | 2.2 | 17.16 +Kspace | 2.6173 | 2.6542 | 2.7273 | 2.7 | 37.74 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.029461 | 0.087268 | 0.13754 | 13.0 | 1.28 -Output | 0.0018618 | 0.001869 | 0.0018811 | 0.0 | 0.03 -Modify | 2.8692 | 2.8709 | 2.8741 | 0.1 | 42.21 -Other | | 0.002119 | | | 0.03 +Comm | 0.042837 | 0.11362 | 0.1882 | 16.9 | 1.62 +Output | 0.0018778 | 0.0018882 | 0.0019088 | 0.0 | 0.03 +Modify | 3.0484 | 3.0535 | 3.0606 | 0.3 | 43.42 +Other | | 0.002387 | | | 0.03 Nlocal: 864 ave 864 max 864 min Histogram: 4 0 0 0 0 0 0 0 0 0 @@ -132,4 +136,4 @@ Dangerous builds = 0 Please see the log.cite file for references relevant to this simulation -Total wall time: 0:00:06 +Total wall time: 0:00:07 diff --git a/examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_pppm.g++.1 b/examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_pppm.g++.1 similarity index 77% rename from examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_pppm.g++.1 rename to examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_pppm.g++.1 index b0e87d786d70a634276f9b92d1b70f8eecfad26a..3b2c36b40cf9774fc813d4a7a18edd13bfff00f6 100644 --- a/examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_pppm.g++.1 +++ b/examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_pppm.g++.1 @@ -1,7 +1,11 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bcc iron in a 3d periodic box clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task units metal atom_style spin @@ -19,7 +23,7 @@ Created orthogonal box = (0 0 0) to (34.398 34.398 34.398) 1 by 1 by 1 MPI processor grid create_atoms 1 box Created 3456 atoms - create_atoms CPU = 0.00166583 secs + create_atoms CPU = 0.00192595 secs # setting mass, mag. moments, and interactions for bcc iron @@ -99,24 +103,24 @@ Neighbor list info ... bin: none Per MPI rank memory allocation (min/avg/max) = 16.27 | 16.27 | 16.27 Mbytes Step Time v_magx v_magy v_magz v_magnorm v_tmag v_emag PotEng TotEng - 0 0 -1 0 0 1 3.7996771e-37 -767.89759 -15175.402 -15130.741 - 50 0.005 -1 3.6585337e-09 -1.9445403e-09 1 5.1405121e-10 -767.88039 -15173.779 -15130.75 - 100 0.01 -1 7.3585728e-09 -3.8640878e-09 1 2.0194927e-09 -767.83109 -15169.191 -15130.774 -Loop time of 15.3615 on 1 procs for 100 steps with 3456 atoms + 0 0 -1 0 0 1 3.7996771e-37 -383.94879 -15175.641 -15130.98 + 50 0.005 -1 3.6585337e-09 -1.9445403e-09 1 5.1408909e-10 -383.94019 -15174.017 -15130.98 + 100 0.01 -1 7.3585736e-09 -3.8640869e-09 1 2.0200831e-09 -383.91553 -15169.428 -15130.98 +Loop time of 15.1465 on 1 procs for 100 steps with 3456 atoms -Performance: 0.056 ns/day, 426.709 hours/ns, 6.510 timesteps/s -99.9% CPU use with 1 MPI tasks x no OpenMP threads +Performance: 0.057 ns/day, 420.737 hours/ns, 6.602 timesteps/s +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 4.8418 | 4.8418 | 4.8418 | 0.0 | 31.52 -Kspace | 0.66626 | 0.66626 | 0.66626 | 0.0 | 4.34 +Pair | 4.6008 | 4.6008 | 4.6008 | 0.0 | 30.38 +Kspace | 0.65995 | 0.65995 | 0.65995 | 0.0 | 4.36 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.014248 | 0.014248 | 0.014248 | 0.0 | 0.09 -Output | 0.0064788 | 0.0064788 | 0.0064788 | 0.0 | 0.04 -Modify | 9.8279 | 9.8279 | 9.8279 | 0.0 | 63.98 -Other | | 0.00478 | | | 0.03 +Comm | 0.01495 | 0.01495 | 0.01495 | 0.0 | 0.10 +Output | 0.0065951 | 0.0065951 | 0.0065951 | 0.0 | 0.04 +Modify | 9.8589 | 9.8589 | 9.8589 | 0.0 | 65.09 +Other | | 0.005332 | | | 0.04 Nlocal: 3456 ave 3456 max 3456 min Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_pppm.g++.4 b/examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_pppm.g++.4 similarity index 76% rename from examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_pppm.g++.4 rename to examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_pppm.g++.4 index 4fbd0eb52b77b526dd571705cd2638775603f963..1c3394dfbf0c923899cc0d3ef618ceb9e5747878 100644 --- a/examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_pppm.g++.4 +++ b/examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_pppm.g++.4 @@ -1,7 +1,11 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bcc iron in a 3d periodic box clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task units metal atom_style spin @@ -19,7 +23,7 @@ Created orthogonal box = (0 0 0) to (34.398 34.398 34.398) 1 by 2 by 2 MPI processor grid create_atoms 1 box Created 3456 atoms - create_atoms CPU = 0.00123286 secs + create_atoms CPU = 0.0007658 secs # setting mass, mag. moments, and interactions for bcc iron @@ -99,24 +103,24 @@ Neighbor list info ... bin: none Per MPI rank memory allocation (min/avg/max) = 10.42 | 10.42 | 10.42 Mbytes Step Time v_magx v_magy v_magz v_magnorm v_tmag v_emag PotEng TotEng - 0 0 -1 0 0 1 2.3173191e-37 -767.89759 -15559.59 -15514.929 - 50 0.005 -1 3.6593054e-09 -1.9379563e-09 1 4.9747018e-10 -767.88014 -15557.972 -15514.929 - 100 0.01 -1 7.3731919e-09 -3.8151563e-09 1 1.9544299e-09 -767.8301 -15553.402 -15514.929 -Loop time of 4.4084 on 4 procs for 100 steps with 3456 atoms + 0 0 -1 0 0 1 2.3173191e-37 -383.94879 -15175.641 -15130.98 + 50 0.005 -1 3.6593053e-09 -1.9379563e-09 1 4.9750695e-10 -383.94007 -15174.032 -15130.98 + 100 0.01 -1 7.3731899e-09 -3.8151552e-09 1 1.9550037e-09 -383.91504 -15169.485 -15130.98 +Loop time of 4.3717 on 4 procs for 100 steps with 3456 atoms -Performance: 0.196 ns/day, 122.455 hours/ns, 22.684 timesteps/s -100.0% CPU use with 4 MPI tasks x no OpenMP threads +Performance: 0.198 ns/day, 121.436 hours/ns, 22.874 timesteps/s +100.0% CPU use with 4 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 1.2326 | 1.2513 | 1.2693 | 1.3 | 28.38 -Kspace | 0.22823 | 0.24585 | 0.26385 | 2.8 | 5.58 +Pair | 1.1624 | 1.1869 | 1.2125 | 1.8 | 27.15 +Kspace | 0.24468 | 0.26758 | 0.29157 | 3.6 | 6.12 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.025352 | 0.028409 | 0.032299 | 1.6 | 0.64 -Output | 0.001868 | 0.0018761 | 0.0018861 | 0.0 | 0.04 -Modify | 2.8753 | 2.8788 | 2.8818 | 0.1 | 65.30 -Other | | 0.002175 | | | 0.05 +Comm | 0.027149 | 0.030758 | 0.033902 | 1.7 | 0.70 +Output | 0.0030079 | 0.0030248 | 0.0030622 | 0.0 | 0.07 +Modify | 2.8782 | 2.8806 | 2.8837 | 0.1 | 65.89 +Other | | 0.002793 | | | 0.06 Nlocal: 864 ave 864 max 864 min Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/iron/in.spin.iron b/examples/SPIN/iron/in.spin.iron index d60e6b86f5fb0626fcd5b473cc7ee435dc133548..58c0537af786a8004dc72258e62a50fcfefa24f0 100644 --- a/examples/SPIN/iron/in.spin.iron +++ b/examples/SPIN/iron/in.spin.iron @@ -31,6 +31,7 @@ neighbor 0.1 bin neigh_modify every 10 check yes delay 20 fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 +fix_modify 1 energy yes fix 2 all langevin/spin 0.0 0.0 21 fix 3 all nve/spin lattice moving diff --git a/examples/SPIN/iron/log.14Apr20.spin.iron.g++.1 b/examples/SPIN/iron/log.14Apr20.spin.iron.g++.1 new file mode 100644 index 0000000000000000000000000000000000000000..0f3257bc0546a990ddd9d90821310ca2a9f1f388 --- /dev/null +++ b/examples/SPIN/iron/log.14Apr20.spin.iron.g++.1 @@ -0,0 +1,141 @@ +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# bcc iron in a 3d periodic box + +clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +units metal +atom_style spin + +dimension 3 +boundary p p p + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice bcc 2.8665 +Lattice spacing in x,y,z = 2.8665 2.8665 2.8665 +region box block 0.0 5.0 0.0 5.0 0.0 5.0 +create_box 1 box +Created orthogonal box = (0 0 0) to (14.3325 14.3325 14.3325) + 1 by 1 by 1 MPI processor grid +create_atoms 1 box +Created 250 atoms + create_atoms CPU = 0.00103498 secs + +# setting mass, mag. moments, and interactions for bcc iron + +mass 1 55.845 + +set group all spin/random 31 2.2 + 250 settings made for spin/random +# set group all spin 2.2 0.0 0.0 1.0 +velocity all create 100 4928459 rot yes dist gaussian + +pair_style hybrid/overlay eam/alloy spin/exchange 3.5 +pair_coeff * * eam/alloy Fe_Mishin2006.eam.alloy Fe +pair_coeff * * spin/exchange exchange 3.4 0.02726 0.2171 1.841 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 +fix_modify 1 energy yes +fix 2 all langevin/spin 0.0 0.0 21 + +fix 3 all nve/spin lattice moving +timestep 0.0001 + +# compute and output options + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable magz equal c_out_mag[3] +variable magnorm equal c_out_mag[4] +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_magnorm v_tmag temp v_emag ke pe press etotal +thermo 50 + +compute outsp all property/atom spx spy spz sp fmx fmy fmz +dump 1 all custom 100 dump_iron.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] + +run 1000 +Neighbor list info ... + update every 10 steps, delay 20 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 5.77337 + ghost atom cutoff = 5.77337 + binsize = 2.88668, bins = 5 5 5 + 2 neighbor lists, perpetual/occasional/extra = 2 0 0 + (1) pair eam/alloy, perpetual, half/full from (2) + attributes: half, newton on + pair build: halffull/newton + stencil: none + bin: none + (2) pair spin/exchange, perpetual + attributes: full, newton on + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 7.82 | 7.82 | 7.82 Mbytes +Step Time v_magnorm v_tmag Temp v_emag KinEng PotEng Press TotEng + 0 0 0.076456975 4554.5462 100.00358 -0.42895634 3.2186929 -1070.429 394.43342 -1067.2103 + 50 0.005 0.076456974 4671.3033 96.520068 -0.43128393 3.1065733 -1070.3169 707.2166 -1067.2103 + 100 0.01 0.076456983 4793.2674 86.525198 -0.43550567 2.7848806 -1069.9952 1456.8628 -1067.2103 + 150 0.015 0.076456973 4894.1924 71.664875 -0.44035101 2.3065896 -1069.5169 2510.4447 -1067.2103 + 200 0.02 0.076456944 4859.9379 54.610496 -0.44591875 1.7576812 -1068.968 3686.7124 -1067.2103 + 250 0.025 0.076456953 4636.3427 38.560198 -0.45258831 1.2410899 -1068.4514 4757.3215 -1067.2103 + 300 0.03 0.076457027 4333.5734 26.459387 -0.45961266 0.85161592 -1068.0619 5505.1963 -1067.2103 + 350 0.035 0.076457102 4124.1453 20.205123 -0.46569388 0.65031758 -1067.8606 5768.9329 -1067.2103 + 400 0.04 0.076457116 4064.6843 20.142986 -0.47038245 0.64831763 -1067.8586 5521.4162 -1067.2103 + 450 0.045 0.076457072 4076.2576 25.084719 -0.4741092 0.80737114 -1068.0176 4890.8714 -1067.2103 + 500 0.05 0.076457001 4105.5788 32.863873 -0.47765283 1.0577493 -1068.268 4093.2603 -1067.2103 + 550 0.055 0.076456962 4109.0613 41.126745 -0.48188168 1.3236962 -1068.534 3337.5111 -1067.2103 + 600 0.06 0.076456996 3993.3891 47.996188 -0.48790867 1.5447946 -1068.7551 2767.4353 -1067.2103 + 650 0.065 0.076457077 3776.2342 52.386407 -0.49621451 1.6860972 -1068.8964 2458.9058 -1067.2103 + 700 0.07 0.076457137 3586.7963 54.031337 -0.50582777 1.7390405 -1068.9493 2397.5333 -1067.2103 + 750 0.075 0.076457135 3535.1994 53.389736 -0.51542639 1.7183901 -1068.9287 2514.1889 -1067.2103 + 800 0.08 0.076457118 3585.6546 51.428399 -0.52423597 1.6552629 -1068.8655 2732.6669 -1067.2103 + 850 0.085 0.076457118 3634.4891 49.293001 -0.53165471 1.5865335 -1068.7968 2977.6259 -1067.2103 + 900 0.09 0.076457126 3660.3333 47.999274 -0.53780479 1.5448939 -1068.7552 3176.4112 -1067.2103 + 950 0.095 0.076457129 3671.3259 48.180409 -0.54376787 1.5507239 -1068.761 3283.2242 -1067.2103 + 1000 0.1 0.076457139 3628.6038 49.917341 -0.55029097 1.6066284 -1068.8169 3283.2361 -1067.2103 +Loop time of 1.66624 on 1 procs for 1000 steps with 250 atoms + +Performance: 5.185 ns/day, 4.628 hours/ns, 600.152 timesteps/s +99.7% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0.73884 | 0.73884 | 0.73884 | 0.0 | 44.34 +Neigh | 0.0045731 | 0.0045731 | 0.0045731 | 0.0 | 0.27 +Comm | 0.023681 | 0.023681 | 0.023681 | 0.0 | 1.42 +Output | 0.0042441 | 0.0042441 | 0.0042441 | 0.0 | 0.25 +Modify | 0.89131 | 0.89131 | 0.89131 | 0.0 | 53.49 +Other | | 0.003589 | | | 0.22 + +Nlocal: 250 ave 250 max 250 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 1407 ave 1407 max 1407 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 7868 ave 7868 max 7868 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +FullNghs: 15736 ave 15736 max 15736 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 15736 +Ave neighs/atom = 62.944 +Neighbor list builds = 6 +Dangerous builds = 0 + +Please see the log.cite file for references relevant to this simulation + +Total wall time: 0:00:01 diff --git a/examples/SPIN/iron/log.14Apr20.spin.iron.g++.4 b/examples/SPIN/iron/log.14Apr20.spin.iron.g++.4 new file mode 100644 index 0000000000000000000000000000000000000000..1cfa74559ba6f15a3491bd9d8108108645ec42ad --- /dev/null +++ b/examples/SPIN/iron/log.14Apr20.spin.iron.g++.4 @@ -0,0 +1,141 @@ +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# bcc iron in a 3d periodic box + +clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +units metal +atom_style spin + +dimension 3 +boundary p p p + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice bcc 2.8665 +Lattice spacing in x,y,z = 2.8665 2.8665 2.8665 +region box block 0.0 5.0 0.0 5.0 0.0 5.0 +create_box 1 box +Created orthogonal box = (0 0 0) to (14.3325 14.3325 14.3325) + 1 by 2 by 2 MPI processor grid +create_atoms 1 box +Created 250 atoms + create_atoms CPU = 0.000688791 secs + +# setting mass, mag. moments, and interactions for bcc iron + +mass 1 55.845 + +set group all spin/random 31 2.2 + 250 settings made for spin/random +# set group all spin 2.2 0.0 0.0 1.0 +velocity all create 100 4928459 rot yes dist gaussian + +pair_style hybrid/overlay eam/alloy spin/exchange 3.5 +pair_coeff * * eam/alloy Fe_Mishin2006.eam.alloy Fe +pair_coeff * * spin/exchange exchange 3.4 0.02726 0.2171 1.841 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 +fix_modify 1 energy yes +fix 2 all langevin/spin 0.0 0.0 21 + +fix 3 all nve/spin lattice moving +timestep 0.0001 + +# compute and output options + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable magz equal c_out_mag[3] +variable magnorm equal c_out_mag[4] +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_magnorm v_tmag temp v_emag ke pe press etotal +thermo 50 + +compute outsp all property/atom spx spy spz sp fmx fmy fmz +dump 1 all custom 100 dump_iron.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] + +run 1000 +Neighbor list info ... + update every 10 steps, delay 20 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 5.77337 + ghost atom cutoff = 5.77337 + binsize = 2.88668, bins = 5 5 5 + 2 neighbor lists, perpetual/occasional/extra = 2 0 0 + (1) pair eam/alloy, perpetual, half/full from (2) + attributes: half, newton on + pair build: halffull/newton + stencil: none + bin: none + (2) pair spin/exchange, perpetual + attributes: full, newton on + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 7.766 | 7.766 | 7.766 Mbytes +Step Time v_magnorm v_tmag Temp v_emag KinEng PotEng Press TotEng + 0 0 0.076456975 4554.5462 100.00358 -0.42895634 3.2186929 -1070.429 394.43342 -1067.2103 + 50 0.005 0.076456995 4714.366 96.279315 -0.42705836 3.0988245 -1070.3091 714.16563 -1067.2103 + 100 0.01 0.076457028 4844.7708 86.007787 -0.43034795 2.7682274 -1069.9785 1479.8537 -1067.2103 + 150 0.015 0.076457073 4938.5943 70.888778 -0.43554708 2.2816103 -1069.4919 2538.6386 -1067.2103 + 200 0.02 0.076457107 4910.2627 53.612031 -0.44069391 1.7255448 -1068.9358 3702.0713 -1067.2103 + 250 0.025 0.07645713 4705.3075 37.374184 -0.44525189 1.2029171 -1068.4132 4749.8323 -1067.2103 + 300 0.03 0.076457162 4418.4389 25.117093 -0.44873668 0.80841314 -1068.0187 5473.2266 -1067.2103 + 350 0.035 0.07645722 4233.0963 18.792985 -0.45135838 0.60486682 -1067.8151 5710.4039 -1067.2103 + 400 0.04 0.07645726 4204.002 18.876267 -0.4546575 0.60754729 -1067.8178 5437.712 -1067.2103 + 450 0.045 0.076457242 4221.8277 24.290539 -0.45989731 0.78180986 -1067.9921 4788.1384 -1067.2103 + 500 0.05 0.076457208 4226.337 32.849365 -0.46668115 1.0572823 -1068.2676 3961.2587 -1067.2103 + 550 0.055 0.076457211 4195.1414 42.028524 -0.47379263 1.3527207 -1068.563 3159.4718 -1067.2103 + 600 0.06 0.076457222 4043.6401 49.722783 -0.48155951 1.6003664 -1068.8106 2554.4057 -1067.2103 + 650 0.065 0.07645721 3784.8692 54.624328 -0.49066003 1.7581264 -1068.9684 2266.0816 -1067.2103 + 700 0.07 0.076457187 3576.7472 56.274292 -0.49932577 1.8112318 -1069.0215 2298.3908 -1067.2103 + 750 0.075 0.07645717 3531.6724 55.083486 -0.50591093 1.7729047 -1068.9832 2557.6667 -1067.2103 + 800 0.08 0.076457176 3593.0894 52.172747 -0.51103604 1.6792204 -1068.8895 2933.0411 -1067.2103 + 850 0.085 0.076457206 3688.4988 48.957423 -0.51573858 1.5757327 -1068.786 3313.8291 -1067.2103 + 900 0.09 0.076457231 3788.943 46.719714 -0.52043742 1.5037103 -1068.714 3600.8734 -1067.2103 + 950 0.095 0.076457251 3854.0552 46.272425 -0.52460019 1.4893139 -1068.6996 3718.2987 -1067.2103 + 1000 0.1 0.076457302 3859.0984 47.806309 -0.52719778 1.5386831 -1068.749 3641.2287 -1067.2103 +Loop time of 1.55258 on 4 procs for 1000 steps with 250 atoms + +Performance: 5.565 ns/day, 4.313 hours/ns, 644.089 timesteps/s +99.8% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0.2109 | 0.21798 | 0.22568 | 1.1 | 14.04 +Neigh | 0.0011308 | 0.0011812 | 0.0012279 | 0.1 | 0.08 +Comm | 0.074407 | 0.082247 | 0.090297 | 2.0 | 5.30 +Output | 0.0019011 | 0.0019355 | 0.0020187 | 0.1 | 0.12 +Modify | 1.2468 | 1.2477 | 1.249 | 0.1 | 80.36 +Other | | 0.001532 | | | 0.10 + +Nlocal: 62.5 ave 66 max 60 min +Histogram: 1 1 0 0 0 1 0 0 0 1 +Nghost: 846.25 ave 861 max 831 min +Histogram: 1 0 1 0 0 0 0 1 0 1 +Neighs: 1962.25 ave 2091 max 1866 min +Histogram: 1 0 0 2 0 0 0 0 0 1 +FullNghs: 3924.5 ave 4138 max 3771 min +Histogram: 1 1 0 0 0 1 0 0 0 1 + +Total # of neighbors = 15698 +Ave neighs/atom = 62.792 +Neighbor list builds = 6 +Dangerous builds = 0 + +Please see the log.cite file for references relevant to this simulation + +Total wall time: 0:00:01 diff --git a/examples/SPIN/iron/log.19Nov19.spin.iron_cubic.g++.1 b/examples/SPIN/iron/log.14Apr20.spin.iron_cubic.g++.1 similarity index 65% rename from examples/SPIN/iron/log.19Nov19.spin.iron_cubic.g++.1 rename to examples/SPIN/iron/log.14Apr20.spin.iron_cubic.g++.1 index 22957c84984cf14d31cff1626ee7a5bca9eca688..5d07f1b71a5dbeebadd3433a3cd0f78d6c7d7a46 100644 --- a/examples/SPIN/iron/log.19Nov19.spin.iron_cubic.g++.1 +++ b/examples/SPIN/iron/log.14Apr20.spin.iron_cubic.g++.1 @@ -1,7 +1,11 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bcc iron in a 3d periodic box clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task units metal atom_style spin @@ -19,7 +23,7 @@ Created orthogonal box = (0 0 0) to (14.3325 14.3325 14.3325) 1 by 1 by 1 MPI processor grid create_atoms 1 box Created 250 atoms - create_atoms CPU = 0.00101709 secs + create_atoms CPU = 0.000530005 secs # setting mass, mag. moments, and interactions for bcc iron @@ -81,53 +85,53 @@ Neighbor list info ... bin: standard Per MPI rank memory allocation (min/avg/max) = 7.82 | 7.82 | 7.82 Mbytes Step Time v_magx v_magy v_magz v_magnorm v_tmag v_emag PotEng TotEng - 0 0 -1 0 0 1 0 -55.58269 -1097.7914 -1094.5727 - 50 0.005 -1 0 0 1 0 -55.581417 -1097.6764 -1094.5733 - 100 0.01 -1 0 0 1 0 -55.577759 -1097.35 -1094.5751 - 150 0.015 -1 0 0 1 0 -55.57219 -1096.8677 -1094.5779 - 200 0.02 -1 0 0 1 0 -55.565438 -1096.3163 -1094.5813 - 250 0.025 -1 0 0 1 0 -55.558379 -1095.7987 -1094.5848 - 300 0.03 -1 0 0 1 0 -55.551886 -1095.4103 -1094.5881 - 350 0.035 -1 0 0 1 0 -55.546675 -1095.2124 -1094.5907 - 400 0.04 -1 0 0 1 0 -55.543187 -1095.2153 -1094.5924 - 450 0.045 -1 0 0 1 0 -55.54154 -1095.379 -1094.5932 - 500 0.05 -1 0 0 1 0 -55.541574 -1095.633 -1094.5932 - 550 0.055 -1 0 0 1 0 -55.542941 -1095.9006 -1094.5925 - 600 0.06 -1 0 0 1 0 -55.545209 -1096.1205 -1094.5914 - 650 0.065 -1 0 0 1 0 -55.547951 -1096.2575 -1094.59 - 700 0.07 -1 0 0 1 0 -55.550801 -1096.3044 -1094.5886 - 750 0.075 -1 0 0 1 0 -55.553483 -1096.2778 -1094.5873 - 800 0.08 -1 0 0 1 0 -55.555816 -1096.2098 -1094.5861 - 850 0.085 -1 0 0 1 0 -55.557706 -1096.1372 -1094.5852 - 900 0.09 -1 0 0 1 0 -55.55913 -1096.0919 -1094.5844 - 950 0.095 -1 0 0 1 0 -55.560111 -1096.0925 -1094.584 - 1000 0.1 -1 0 0 1 0 -55.560705 -1096.1411 -1094.5837 -Loop time of 1.74825 on 1 procs for 1000 steps with 250 atoms - -Performance: 4.942 ns/day, 4.856 hours/ns, 571.999 timesteps/s -100.0% CPU use with 1 MPI tasks x no OpenMP threads + 0 0 -1 0 0 1 0 -27.791345 -1097.7914 -1094.5727 + 50 0.005 -1 0 0 1 0 -27.790708 -1097.6764 -1094.5727 + 100 0.01 -1 0 0 1 0 -27.788879 -1097.3499 -1094.5727 + 150 0.015 -1 0 0 1 0 -27.78609 -1096.8672 -1094.5727 + 200 0.02 -1 0 0 1 0 -27.782705 -1096.3147 -1094.5727 + 250 0.025 -1 0 0 1 0 -27.779157 -1095.7952 -1094.5727 + 300 0.03 -1 0 0 1 0 -27.775883 -1095.4038 -1094.5727 + 350 0.035 -1 0 0 1 0 -27.773241 -1095.2023 -1094.5727 + 400 0.04 -1 0 0 1 0 -27.771451 -1095.201 -1094.5727 + 450 0.045 -1 0 0 1 0 -27.770578 -1095.3608 -1094.5727 + 500 0.05 -1 0 0 1 0 -27.770546 -1095.6113 -1094.5727 + 550 0.055 -1 0 0 1 0 -27.771185 -1095.8764 -1094.5727 + 600 0.06 -1 0 0 1 0 -27.772282 -1096.0948 -1094.5727 + 650 0.065 -1 0 0 1 0 -27.773629 -1096.2313 -1094.5727 + 700 0.07 -1 0 0 1 0 -27.775042 -1096.2787 -1094.5727 + 750 0.075 -1 0 0 1 0 -27.776384 -1096.2534 -1094.5727 + 800 0.08 -1 0 0 1 0 -27.777564 -1096.1872 -1094.5727 + 850 0.085 -1 0 0 1 0 -27.778533 -1096.117 -1094.5727 + 900 0.09 -1 0 0 1 0 -27.779276 -1096.0741 -1094.5727 + 950 0.095 -1 0 0 1 0 -27.779802 -1096.0771 -1094.5727 + 1000 0.1 -1 0 0 1 0 -27.780134 -1096.1278 -1094.5727 +Loop time of 1.70062 on 1 procs for 1000 steps with 250 atoms + +Performance: 5.080 ns/day, 4.724 hours/ns, 588.019 timesteps/s +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 0.80384 | 0.80384 | 0.80384 | 0.0 | 45.98 -Neigh | 0.004528 | 0.004528 | 0.004528 | 0.0 | 0.26 -Comm | 0.022954 | 0.022954 | 0.022954 | 0.0 | 1.31 -Output | 0.0034568 | 0.0034568 | 0.0034568 | 0.0 | 0.20 -Modify | 0.91007 | 0.91007 | 0.91007 | 0.0 | 52.06 -Other | | 0.003404 | | | 0.19 +Pair | 0.72617 | 0.72617 | 0.72617 | 0.0 | 42.70 +Neigh | 0.0045958 | 0.0045958 | 0.0045958 | 0.0 | 0.27 +Comm | 0.023132 | 0.023132 | 0.023132 | 0.0 | 1.36 +Output | 0.0035856 | 0.0035856 | 0.0035856 | 0.0 | 0.21 +Modify | 0.93966 | 0.93966 | 0.93966 | 0.0 | 55.25 +Other | | 0.003483 | | | 0.20 Nlocal: 250 ave 250 max 250 min Histogram: 1 0 0 0 0 0 0 0 0 0 Nghost: 1415 ave 1415 max 1415 min Histogram: 1 0 0 0 0 0 0 0 0 0 -Neighs: 7873 ave 7873 max 7873 min +Neighs: 7872 ave 7872 max 7872 min Histogram: 1 0 0 0 0 0 0 0 0 0 -FullNghs: 15746 ave 15746 max 15746 min +FullNghs: 15744 ave 15744 max 15744 min Histogram: 1 0 0 0 0 0 0 0 0 0 -Total # of neighbors = 15746 -Ave neighs/atom = 62.984 +Total # of neighbors = 15744 +Ave neighs/atom = 62.976 Neighbor list builds = 6 Dangerous builds = 0 # min_style spin diff --git a/examples/SPIN/iron/log.19Nov19.spin.iron_cubic.g++.4 b/examples/SPIN/iron/log.14Apr20.spin.iron_cubic.g++.4 similarity index 65% rename from examples/SPIN/iron/log.19Nov19.spin.iron_cubic.g++.4 rename to examples/SPIN/iron/log.14Apr20.spin.iron_cubic.g++.4 index 60db1064e0d41d7975c4ddcd370f5402291d3185..d1d0868e2fc8522ecd8890169d6e9c61d755f823 100644 --- a/examples/SPIN/iron/log.19Nov19.spin.iron_cubic.g++.4 +++ b/examples/SPIN/iron/log.14Apr20.spin.iron_cubic.g++.4 @@ -1,7 +1,11 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bcc iron in a 3d periodic box clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task units metal atom_style spin @@ -19,7 +23,7 @@ Created orthogonal box = (0 0 0) to (14.3325 14.3325 14.3325) 1 by 2 by 2 MPI processor grid create_atoms 1 box Created 250 atoms - create_atoms CPU = 0.000651121 secs + create_atoms CPU = 0.00071311 secs # setting mass, mag. moments, and interactions for bcc iron @@ -81,53 +85,53 @@ Neighbor list info ... bin: standard Per MPI rank memory allocation (min/avg/max) = 7.766 | 7.766 | 7.766 Mbytes Step Time v_magx v_magy v_magz v_magnorm v_tmag v_emag PotEng TotEng - 0 0 -1 0 0 1 0 -55.58269 -1125.5827 -1122.364 - 50 0.005 -1 0 0 1 0 -55.581457 -1125.4635 -1122.364 - 100 0.01 -1 0 0 1 0 -55.577922 -1125.1262 -1122.364 - 150 0.015 -1 0 0 1 0 -55.572562 -1124.6305 -1122.364 - 200 0.02 -1 0 0 1 0 -55.566098 -1124.067 -1122.364 - 250 0.025 -1 0 0 1 0 -55.559384 -1123.5412 -1122.364 - 300 0.03 -1 0 0 1 0 -55.553261 -1123.1491 -1122.364 - 350 0.035 -1 0 0 1 0 -55.548413 -1122.9526 -1122.364 - 400 0.04 -1 0 0 1 0 -55.545248 -1122.9623 -1122.364 - 450 0.045 -1 0 0 1 0 -55.54387 -1123.1395 -1122.364 - 500 0.05 -1 0 0 1 0 -55.544101 -1123.4126 -1122.364 - 550 0.055 -1 0 0 1 0 -55.54558 -1123.7021 -1122.364 - 600 0.06 -1 0 0 1 0 -55.547857 -1123.9414 -1122.364 - 650 0.065 -1 0 0 1 0 -55.550495 -1124.0897 -1122.364 - 700 0.07 -1 0 0 1 0 -55.553127 -1124.136 -1122.364 - 750 0.075 -1 0 0 1 0 -55.555497 -1124.0961 -1122.364 - 800 0.08 -1 0 0 1 0 -55.557466 -1124.0053 -1122.364 - 850 0.085 -1 0 0 1 0 -55.559001 -1123.9069 -1122.364 - 900 0.09 -1 0 0 1 0 -55.560147 -1123.8404 -1122.364 - 950 0.095 -1 0 0 1 0 -55.560992 -1123.8312 -1122.364 - 1000 0.1 -1 0 0 1 0 -55.561635 -1123.8853 -1122.364 -Loop time of 1.5074 on 4 procs for 1000 steps with 250 atoms - -Performance: 5.732 ns/day, 4.187 hours/ns, 663.393 timesteps/s -99.9% CPU use with 4 MPI tasks x no OpenMP threads + 0 0 -1 0 0 1 0 -27.791345 -1097.7914 -1094.5727 + 50 0.005 -1 0 0 1 0 -27.790728 -1097.6727 -1094.5727 + 100 0.01 -1 0 0 1 0 -27.78896 -1097.3371 -1094.5727 + 150 0.015 -1 0 0 1 0 -27.786276 -1096.8437 -1094.5727 + 200 0.02 -1 0 0 1 0 -27.783035 -1096.2824 -1094.5727 + 250 0.025 -1 0 0 1 0 -27.779661 -1095.758 -1094.5727 + 300 0.03 -1 0 0 1 0 -27.776574 -1095.3661 -1094.5727 + 350 0.035 -1 0 0 1 0 -27.774114 -1095.1684 -1094.5727 + 400 0.04 -1 0 0 1 0 -27.772489 -1095.1758 -1094.5727 + 450 0.045 -1 0 0 1 0 -27.771753 -1095.3498 -1094.5727 + 500 0.05 -1 0 0 1 0 -27.771823 -1095.6196 -1094.5727 + 550 0.055 -1 0 0 1 0 -27.772521 -1095.9061 -1094.5727 + 600 0.06 -1 0 0 1 0 -27.773627 -1096.1431 -1094.5727 + 650 0.065 -1 0 0 1 0 -27.774925 -1096.2899 -1094.5727 + 700 0.07 -1 0 0 1 0 -27.776234 -1096.3356 -1094.5727 + 750 0.075 -1 0 0 1 0 -27.777423 -1096.2961 -1094.5727 + 800 0.08 -1 0 0 1 0 -27.778424 -1096.2063 -1094.5727 + 850 0.085 -1 0 0 1 0 -27.779215 -1096.1093 -1094.5727 + 900 0.09 -1 0 0 1 0 -27.779817 -1096.0443 -1094.5727 + 950 0.095 -1 0 0 1 0 -27.780271 -1096.0363 -1094.5727 + 1000 0.1 -1 0 0 1 0 -27.780622 -1096.0914 -1094.5727 +Loop time of 1.54663 on 4 procs for 1000 steps with 250 atoms + +Performance: 5.586 ns/day, 4.296 hours/ns, 646.566 timesteps/s +99.9% CPU use with 4 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 0.22156 | 0.23223 | 0.24219 | 1.5 | 15.41 -Neigh | 0.0011292 | 0.0011852 | 0.0012362 | 0.1 | 0.08 -Comm | 0.067507 | 0.076341 | 0.087237 | 2.6 | 5.06 -Output | 0.0015073 | 0.0015442 | 0.0015914 | 0.1 | 0.10 -Modify | 1.1934 | 1.1947 | 1.1955 | 0.1 | 79.25 -Other | | 0.001434 | | | 0.10 +Pair | 0.2086 | 0.21716 | 0.22473 | 1.2 | 14.04 +Neigh | 0.0011575 | 0.0011975 | 0.001235 | 0.1 | 0.08 +Comm | 0.070766 | 0.080324 | 0.088558 | 2.3 | 5.19 +Output | 0.0016837 | 0.0017157 | 0.0017838 | 0.1 | 0.11 +Modify | 1.2424 | 1.2446 | 1.2467 | 0.1 | 80.47 +Other | | 0.001639 | | | 0.11 Nlocal: 62.5 ave 66 max 60 min Histogram: 1 1 0 0 0 1 0 0 0 1 Nghost: 848.25 ave 861 max 834 min Histogram: 1 0 0 0 1 0 1 0 0 1 -Neighs: 1962.75 ave 2087 max 1870 min +Neighs: 1962.25 ave 2085 max 1870 min Histogram: 1 1 0 0 0 0 1 0 0 1 -FullNghs: 3925.5 ave 4138 max 3776 min +FullNghs: 3924.5 ave 4136 max 3776 min Histogram: 1 1 0 0 0 1 0 0 0 1 -Total # of neighbors = 15702 -Ave neighs/atom = 62.808 +Total # of neighbors = 15698 +Ave neighs/atom = 62.792 Neighbor list builds = 6 Dangerous builds = 0 # min_style spin diff --git a/examples/SPIN/iron/log.19Nov19.spin.iron.g++.1 b/examples/SPIN/iron/log.19Nov19.spin.iron.g++.1 deleted file mode 100644 index cbc749a1f7b61a36cd96cd6862baf5ce45222b9e..0000000000000000000000000000000000000000 --- a/examples/SPIN/iron/log.19Nov19.spin.iron.g++.1 +++ /dev/null @@ -1,136 +0,0 @@ -LAMMPS (30 Oct 2019) -# bcc iron in a 3d periodic box - -clear -units metal -atom_style spin - -dimension 3 -boundary p p p - -# necessary for the serial algorithm (sametag) -atom_modify map array - -lattice bcc 2.8665 -Lattice spacing in x,y,z = 2.8665 2.8665 2.8665 -region box block 0.0 5.0 0.0 5.0 0.0 5.0 -create_box 1 box -Created orthogonal box = (0 0 0) to (14.3325 14.3325 14.3325) - 1 by 1 by 1 MPI processor grid -create_atoms 1 box -Created 250 atoms - create_atoms CPU = 0.000422955 secs - -# setting mass, mag. moments, and interactions for bcc iron - -mass 1 55.845 - -set group all spin/random 31 2.2 - 250 settings made for spin/random -# set group all spin 2.2 0.0 0.0 1.0 -velocity all create 100 4928459 rot yes dist gaussian - -pair_style hybrid/overlay eam/alloy spin/exchange 3.5 -pair_coeff * * eam/alloy Fe_Mishin2006.eam.alloy Fe -pair_coeff * * spin/exchange exchange 3.4 0.02726 0.2171 1.841 - -neighbor 0.1 bin -neigh_modify every 10 check yes delay 20 - -fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 -fix 2 all langevin/spin 0.0 0.0 21 - -fix 3 all nve/spin lattice moving -timestep 0.0001 - -# compute and output options - -compute out_mag all spin -compute out_pe all pe -compute out_ke all ke -compute out_temp all temp - -variable magz equal c_out_mag[3] -variable magnorm equal c_out_mag[4] -variable emag equal c_out_mag[5] -variable tmag equal c_out_mag[6] - -thermo_style custom step time v_magnorm v_tmag temp v_emag ke pe press etotal -thermo 50 - -compute outsp all property/atom spx spy spz sp fmx fmy fmz -dump 1 all custom 100 dump_iron.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] - -run 1000 -Neighbor list info ... - update every 10 steps, delay 20 steps, check yes - max neighbors/atom: 2000, page size: 100000 - master list distance cutoff = 5.77337 - ghost atom cutoff = 5.77337 - binsize = 2.88668, bins = 5 5 5 - 2 neighbor lists, perpetual/occasional/extra = 2 0 0 - (1) pair eam/alloy, perpetual, half/full from (2) - attributes: half, newton on - pair build: halffull/newton - stencil: none - bin: none - (2) pair spin/exchange, perpetual - attributes: full, newton on - pair build: full/bin/atomonly - stencil: full/bin/3d - bin: standard -Per MPI rank memory allocation (min/avg/max) = 7.82 | 7.82 | 7.82 Mbytes -Step Time v_magnorm v_tmag Temp v_emag KinEng PotEng Press TotEng - 0 0 0.076456975 4554.5462 100.00358 -0.85791269 3.2186929 -1070.429 394.43342 -1067.2103 - 50 0.005 0.076456974 4658.383 96.663685 -0.86504718 3.1111957 -1070.3179 709.50826 -1067.2067 - 100 0.01 0.076456983 4744.1872 86.965803 -0.88035771 2.7990619 -1069.9981 1466.6938 -1067.1991 - 150 0.015 0.076456973 4794.5283 72.421197 -0.8996913 2.3309324 -1069.5203 2534.3867 -1067.1894 - 200 0.02 0.076456944 4707.6548 55.633188 -0.921682 1.7905973 -1068.969 3732.183 -1067.1784 - 250 0.025 0.076456953 4439.4697 39.802206 -0.94649004 1.2810649 -1068.447 4831.5559 -1067.166 - 300 0.03 0.076457027 4101.6694 27.882295 -0.97253854 0.8974133 -1068.0504 5612.0928 -1067.153 - 350 0.035 0.076457103 3860.1545 21.776538 -0.99708692 0.70089477 -1067.8416 5906.3057 -1067.1407 - 400 0.04 0.076457117 3765.5341 21.857102 -1.0190244 0.70348778 -1067.8332 5682.0053 -1067.1297 - 450 0.045 0.076457072 3739.9037 26.959407 -1.0389343 0.86770942 -1067.9875 5066.5077 -1067.1198 - 500 0.05 0.076457001 3730.8342 34.92521 -1.0582008 1.124095 -1068.2342 4279.2424 -1067.1101 - 550 0.055 0.076456962 3698.0556 43.405912 -1.0785156 1.397053 -1068.497 3533.4153 -1067.1 - 600 0.06 0.076456997 3560.947 50.544844 -1.102048 1.626825 -1068.715 2975.8479 -1067.0882 - 650 0.065 0.076457079 3341.7402 55.261218 -1.1296588 1.7786252 -1068.853 2683.3023 -1067.0744 - 700 0.07 0.076457136 3156.8448 57.25083 -1.1595102 1.8426624 -1068.9021 2640.5967 -1067.0595 - 750 0.075 0.076457132 3099.5181 56.934336 -1.1893875 1.8324758 -1068.877 2778.3261 -1067.0445 - 800 0.08 0.076457116 3132.9985 55.266343 -1.2181223 1.7787901 -1068.809 3020.1175 -1067.0302 - 850 0.085 0.076457116 3163.2943 53.376453 -1.2443326 1.7179626 -1068.735 3287.9042 -1067.0171 - 900 0.09 0.076457121 3168.063 52.279557 -1.2676425 1.6826581 -1068.6881 3504.7334 -1067.0054 - 950 0.095 0.076457122 3144.2102 52.667743 -1.2902335 1.6951522 -1068.6893 3622.1382 -1066.9941 - 1000 0.1 0.076457135 3061.0811 54.684094 -1.314147 1.76005 -1068.7422 3625.2935 -1066.9822 -Loop time of 1.6779 on 1 procs for 1000 steps with 250 atoms - -Performance: 5.149 ns/day, 4.661 hours/ns, 595.982 timesteps/s -100.0% CPU use with 1 MPI tasks x no OpenMP threads - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Pair | 0.78285 | 0.78285 | 0.78285 | 0.0 | 46.66 -Neigh | 0.004487 | 0.004487 | 0.004487 | 0.0 | 0.27 -Comm | 0.022926 | 0.022926 | 0.022926 | 0.0 | 1.37 -Output | 0.003927 | 0.003927 | 0.003927 | 0.0 | 0.23 -Modify | 0.86033 | 0.86033 | 0.86033 | 0.0 | 51.27 -Other | | 0.003381 | | | 0.20 - -Nlocal: 250 ave 250 max 250 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Nghost: 1399 ave 1399 max 1399 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Neighs: 7855 ave 7855 max 7855 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -FullNghs: 15710 ave 15710 max 15710 min -Histogram: 1 0 0 0 0 0 0 0 0 0 - -Total # of neighbors = 15710 -Ave neighs/atom = 62.84 -Neighbor list builds = 6 -Dangerous builds = 0 - -Please see the log.cite file for references relevant to this simulation - -Total wall time: 0:00:01 diff --git a/examples/SPIN/iron/log.19Nov19.spin.iron.g++.4 b/examples/SPIN/iron/log.19Nov19.spin.iron.g++.4 deleted file mode 100644 index a5cc9d7f0b2cde1e6a59bbc277f7fff63af451aa..0000000000000000000000000000000000000000 --- a/examples/SPIN/iron/log.19Nov19.spin.iron.g++.4 +++ /dev/null @@ -1,136 +0,0 @@ -LAMMPS (30 Oct 2019) -# bcc iron in a 3d periodic box - -clear -units metal -atom_style spin - -dimension 3 -boundary p p p - -# necessary for the serial algorithm (sametag) -atom_modify map array - -lattice bcc 2.8665 -Lattice spacing in x,y,z = 2.8665 2.8665 2.8665 -region box block 0.0 5.0 0.0 5.0 0.0 5.0 -create_box 1 box -Created orthogonal box = (0 0 0) to (14.3325 14.3325 14.3325) - 1 by 2 by 2 MPI processor grid -create_atoms 1 box -Created 250 atoms - create_atoms CPU = 0.000705957 secs - -# setting mass, mag. moments, and interactions for bcc iron - -mass 1 55.845 - -set group all spin/random 31 2.2 - 250 settings made for spin/random -# set group all spin 2.2 0.0 0.0 1.0 -velocity all create 100 4928459 rot yes dist gaussian - -pair_style hybrid/overlay eam/alloy spin/exchange 3.5 -pair_coeff * * eam/alloy Fe_Mishin2006.eam.alloy Fe -pair_coeff * * spin/exchange exchange 3.4 0.02726 0.2171 1.841 - -neighbor 0.1 bin -neigh_modify every 10 check yes delay 20 - -fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 -fix 2 all langevin/spin 0.0 0.0 21 - -fix 3 all nve/spin lattice moving -timestep 0.0001 - -# compute and output options - -compute out_mag all spin -compute out_pe all pe -compute out_ke all ke -compute out_temp all temp - -variable magz equal c_out_mag[3] -variable magnorm equal c_out_mag[4] -variable emag equal c_out_mag[5] -variable tmag equal c_out_mag[6] - -thermo_style custom step time v_magnorm v_tmag temp v_emag ke pe press etotal -thermo 50 - -compute outsp all property/atom spx spy spz sp fmx fmy fmz -dump 1 all custom 100 dump_iron.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] - -run 1000 -Neighbor list info ... - update every 10 steps, delay 20 steps, check yes - max neighbors/atom: 2000, page size: 100000 - master list distance cutoff = 5.77337 - ghost atom cutoff = 5.77337 - binsize = 2.88668, bins = 5 5 5 - 2 neighbor lists, perpetual/occasional/extra = 2 0 0 - (1) pair eam/alloy, perpetual, half/full from (2) - attributes: half, newton on - pair build: halffull/newton - stencil: none - bin: none - (2) pair spin/exchange, perpetual - attributes: full, newton on - pair build: full/bin/atomonly - stencil: full/bin/3d - bin: standard -Per MPI rank memory allocation (min/avg/max) = 7.766 | 7.766 | 7.766 Mbytes -Step Time v_magnorm v_tmag Temp v_emag KinEng PotEng Press TotEng - 0 0 0.076456975 4554.5462 100.00358 -0.85791269 3.2186929 -1070.8579 394.43342 -1067.6392 - 50 0.005 0.076456995 4701.2004 96.298333 -0.85659448 3.0994366 -1070.7387 714.37866 -1067.6392 - 100 0.01 0.076457028 4794.5923 86.330828 -0.87003341 2.7786247 -1070.4179 1484.2951 -1067.6392 - 150 0.015 0.076457074 4836.9634 71.603402 -0.89006992 2.3046111 -1069.9438 2551.9258 -1067.6392 - 200 0.02 0.076457106 4754.5574 54.648817 -0.91124541 1.7589146 -1069.3981 3731.1494 -1067.6392 - 250 0.025 0.076457128 4502.135 38.599515 -0.93187522 1.2423553 -1068.8816 4804.619 -1067.6392 - 300 0.03 0.076457157 4176.7186 26.383018 -0.95082226 0.8491579 -1068.4884 5563.3287 -1067.6392 - 350 0.035 0.076457207 3955.5658 20.01039 -0.96826468 0.64404992 -1068.2833 5839.6479 -1067.6392 - 400 0.04 0.076457243 3887.9746 20.097682 -0.98706373 0.64685949 -1068.2861 5601.1255 -1067.6392 - 450 0.045 0.076457231 3868.5613 25.687511 -1.0095684 0.82677249 -1068.466 4974.0031 -1067.6392 - 500 0.05 0.076457204 3838.4905 34.604697 -1.0349855 1.113779 -1068.753 4157.1837 -1067.6392 - 550 0.055 0.076457196 3775.1404 44.251809 -1.0609123 1.4242788 -1069.0635 3357.1 -1067.6392 - 600 0.06 0.076457188 3604.8828 52.475202 -1.0880854 1.6889551 -1069.3282 2752.0424 -1067.6392 - 650 0.065 0.07645718 3345.5894 57.926479 -1.1179657 1.8644087 -1069.5036 2467.7403 -1067.6392 - 700 0.07 0.076457185 3138.2001 60.030548 -1.1469999 1.9321298 -1069.5714 2510.1752 -1067.6392 - 750 0.075 0.07645719 3074.9626 59.122504 -1.1721939 1.9029037 -1069.5421 2788.7489 -1067.6392 - 800 0.08 0.076457195 3103.5294 56.349146 -1.1949365 1.813641 -1069.4529 3192.5158 -1067.6392 - 850 0.085 0.076457199 3164.2317 53.154464 -1.2164642 1.7108177 -1069.35 3602.931 -1067.6392 - 900 0.09 0.076457199 3228.1358 50.837416 -1.2366018 1.6362417 -1069.2755 3917.0758 -1067.6392 - 950 0.095 0.076457222 3247.5532 50.234549 -1.2539657 1.6168379 -1069.2561 4059.9275 -1067.6392 - 1000 0.1 0.076457266 3208.3875 51.592727 -1.2671834 1.6605519 -1069.2998 4001.4995 -1067.6392 -Loop time of 1.47769 on 4 procs for 1000 steps with 250 atoms - -Performance: 5.847 ns/day, 4.105 hours/ns, 676.731 timesteps/s -100.0% CPU use with 4 MPI tasks x no OpenMP threads - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Pair | 0.21791 | 0.22724 | 0.23568 | 1.4 | 15.38 -Neigh | 0.001137 | 0.0011771 | 0.0012221 | 0.1 | 0.08 -Comm | 0.066727 | 0.074288 | 0.083826 | 2.3 | 5.03 -Output | 0.0017431 | 0.0017657 | 0.0018256 | 0.1 | 0.12 -Modify | 1.1707 | 1.1718 | 1.1725 | 0.1 | 79.30 -Other | | 0.001427 | | | 0.10 - -Nlocal: 62.5 ave 66 max 60 min -Histogram: 1 0 0 2 0 0 0 0 0 1 -Nghost: 844 ave 857 max 829 min -Histogram: 1 0 0 1 0 0 0 0 1 1 -Neighs: 1962.5 ave 2096 max 1855 min -Histogram: 1 0 1 0 0 1 0 0 0 1 -FullNghs: 3925 ave 4139 max 3766 min -Histogram: 1 0 0 2 0 0 0 0 0 1 - -Total # of neighbors = 15700 -Ave neighs/atom = 62.8 -Neighbor list builds = 6 -Dangerous builds = 0 - -Please see the log.cite file for references relevant to this simulation - -Total wall time: 0:00:01 diff --git a/examples/SPIN/nickel/in.spin.nickel b/examples/SPIN/nickel/in.spin.nickel index 1d62188d8f8496f4b75ad8df7e68516e1b20de99..0fd2e5f345d32210b96cc1400476cd1765b8b013 100644 --- a/examples/SPIN/nickel/in.spin.nickel +++ b/examples/SPIN/nickel/in.spin.nickel @@ -31,6 +31,7 @@ neighbor 0.1 bin neigh_modify every 10 check yes delay 20 fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 +fix_modify 1 energy yes fix 2 all langevin/spin 0.0 0.0 21 fix 3 all nve/spin lattice moving @@ -48,7 +49,7 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -thermo_style custom step time v_magnorm v_emag temp v_tmag etotal +thermo_style custom step time v_magnorm pe v_emag temp v_tmag etotal thermo 50 compute outsp all property/atom spx spy spz sp fmx fmy fmz diff --git a/examples/SPIN/nickel/in.spin.nickel_cubic b/examples/SPIN/nickel/in.spin.nickel_cubic index 1ae069a64f26b429fb136eff0d734dc66b7c1c7c..88c477132ea8c930b272da8cf7a67a43429e8e01 100644 --- a/examples/SPIN/nickel/in.spin.nickel_cubic +++ b/examples/SPIN/nickel/in.spin.nickel_cubic @@ -50,7 +50,7 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -thermo_style custom step time v_magnorm v_emag temp v_tmag etotal +thermo_style custom step time v_magnorm pe v_emag temp v_tmag etotal thermo 50 compute outsp all property/atom spx spy spz sp fmx fmy fmz diff --git a/examples/SPIN/nickel/log.14Apr20.spin.nickel.g++.1 b/examples/SPIN/nickel/log.14Apr20.spin.nickel.g++.1 new file mode 100644 index 0000000000000000000000000000000000000000..e4bcd538c8e8ab5449f025f5285a4c18f464f22c --- /dev/null +++ b/examples/SPIN/nickel/log.14Apr20.spin.nickel.g++.1 @@ -0,0 +1,141 @@ +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# fcc nickel in a 3d periodic box + +clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +units metal +atom_style spin + +dimension 3 +boundary p p p + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice fcc 3.524 +Lattice spacing in x,y,z = 3.524 3.524 3.524 +region box block 0.0 5.0 0.0 5.0 0.0 5.0 +create_box 1 box +Created orthogonal box = (0 0 0) to (17.62 17.62 17.62) + 1 by 1 by 1 MPI processor grid +create_atoms 1 box +Created 500 atoms + create_atoms CPU = 0.00111699 secs + +# setting mass, mag. moments, and interactions for cobalt + +mass 1 58.69 + +set group all spin/random 31 0.63 + 500 settings made for spin/random +#set group all spin 0.63 0.0 0.0 1.0 +velocity all create 100 4928459 rot yes dist gaussian + +pair_style hybrid/overlay eam/alloy spin/exchange 4.0 +pair_coeff * * eam/alloy Ni99.eam.alloy Ni +pair_coeff * * spin/exchange exchange 4.0 0.50 0.2280246862 1.229983475 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 +fix_modify 1 energy yes +fix 2 all langevin/spin 0.0 0.0 21 + +fix 3 all nve/spin lattice moving +timestep 0.0001 + +# compute and output options + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable magz equal c_out_mag[3] +variable magnorm equal c_out_mag[4] +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_magnorm pe v_emag temp v_tmag etotal +thermo 50 + +compute outsp all property/atom spx spy spz sp fmx fmy fmz +dump 1 all custom 100 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] c_outsp[4] c_outsp[5] c_outsp[6] c_outsp[7] + +run 1000 +Neighbor list info ... + update every 10 steps, delay 20 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 5.90375 + ghost atom cutoff = 5.90375 + binsize = 2.95187, bins = 6 6 6 + 2 neighbor lists, perpetual/occasional/extra = 2 0 0 + (1) pair eam/alloy, perpetual, half/full from (2) + attributes: half, newton on + pair build: halffull/newton + stencil: none + bin: none + (2) pair spin/exchange, perpetual + attributes: full, newton on + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 7.885 | 7.885 | 7.885 Mbytes +Step Time v_magnorm PotEng v_emag Temp v_tmag TotEng + 0 0 0.028733803 -2224.7591 0.20498788 100.03408 -4775.0671 -2218.3068 + 50 0.005 0.028733807 -2224.6271 0.093798492 97.987469 -10540.61 -2218.3068 + 100 0.01 0.028733815 -2224.1217 -0.12620081 90.153107 8107.9288 -2218.3068 + 150 0.015 0.028733823 -2223.2984 -0.433762 77.388213 2482.6323 -2218.3068 + 200 0.02 0.028733827 -2222.3037 -0.80329408 61.9663 1426.722 -2218.3068 + 250 0.025 0.028733825 -2221.3495 -1.21118 47.173828 1010.1027 -2218.3068 + 300 0.03 0.028733813 -2220.6532 -1.6389575 36.378126 789.93596 -2218.3068 + 350 0.035 0.028733795 -2220.3656 -2.0724228 31.919862 648.39236 -2218.3068 + 400 0.04 0.028733778 -2220.5225 -2.498882 34.352305 545.8856 -2218.3068 + 450 0.045 0.02873377 -2221.0414 -2.9067963 42.396008 468.97305 -2218.3068 + 500 0.05 0.02873377 -2221.7614 -3.2881407 53.559839 411.03247 -2218.3068 + 550 0.055 0.028733775 -2222.5025 -3.6391769 65.048917 366.0991 -2218.3068 + 600 0.06 0.028733784 -2223.1215 -3.9612604 74.645803 330.37583 -2218.3068 + 650 0.065 0.028733794 -2223.5495 -4.2623843 81.281536 303.0489 -2218.3068 + 700 0.07 0.028733802 -2223.795 -4.5550161 85.087649 283.0897 -2218.3068 + 750 0.075 0.028733805 -2223.9222 -4.8523832 87.059996 267.48601 -2218.3068 + 800 0.08 0.028733804 -2224.0176 -5.1656984 88.538724 253.94215 -2218.3068 + 850 0.085 0.028733799 -2224.1506 -5.4981915 90.600346 242.22988 -2218.3068 + 900 0.09 0.028733794 -2224.3425 -5.8400709 93.576582 232.32115 -2218.3068 + 950 0.095 0.028733796 -2224.5674 -6.1746129 97.063173 222.64192 -2218.3068 + 1000 0.1 0.028733805 -2224.7832 -6.4920444 100.40752 212.7708 -2218.3068 +Loop time of 4.1438 on 1 procs for 1000 steps with 500 atoms + +Performance: 2.085 ns/day, 11.511 hours/ns, 241.324 timesteps/s +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 1.8846 | 1.8846 | 1.8846 | 0.0 | 45.48 +Neigh | 0.012244 | 0.012244 | 0.012244 | 0.0 | 0.30 +Comm | 0.033385 | 0.033385 | 0.033385 | 0.0 | 0.81 +Output | 0.014202 | 0.014202 | 0.014202 | 0.0 | 0.34 +Modify | 2.1938 | 2.1938 | 2.1938 | 0.0 | 52.94 +Other | | 0.005526 | | | 0.13 + +Nlocal: 500 ave 500 max 500 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 1956 ave 1956 max 1956 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 19500 ave 19500 max 19500 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +FullNghs: 39000 ave 39000 max 39000 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 39000 +Ave neighs/atom = 78 +Neighbor list builds = 7 +Dangerous builds = 0 + +Please see the log.cite file for references relevant to this simulation + +Total wall time: 0:00:04 diff --git a/examples/SPIN/nickel/log.14Apr20.spin.nickel.g++.4 b/examples/SPIN/nickel/log.14Apr20.spin.nickel.g++.4 new file mode 100644 index 0000000000000000000000000000000000000000..2ef20e47899234b2c8f5b3ca0e49f14892989236 --- /dev/null +++ b/examples/SPIN/nickel/log.14Apr20.spin.nickel.g++.4 @@ -0,0 +1,141 @@ +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# fcc nickel in a 3d periodic box + +clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +units metal +atom_style spin + +dimension 3 +boundary p p p + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice fcc 3.524 +Lattice spacing in x,y,z = 3.524 3.524 3.524 +region box block 0.0 5.0 0.0 5.0 0.0 5.0 +create_box 1 box +Created orthogonal box = (0 0 0) to (17.62 17.62 17.62) + 1 by 2 by 2 MPI processor grid +create_atoms 1 box +Created 500 atoms + create_atoms CPU = 0.000680923 secs + +# setting mass, mag. moments, and interactions for cobalt + +mass 1 58.69 + +set group all spin/random 31 0.63 + 500 settings made for spin/random +#set group all spin 0.63 0.0 0.0 1.0 +velocity all create 100 4928459 rot yes dist gaussian + +pair_style hybrid/overlay eam/alloy spin/exchange 4.0 +pair_coeff * * eam/alloy Ni99.eam.alloy Ni +pair_coeff * * spin/exchange exchange 4.0 0.50 0.2280246862 1.229983475 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 +fix_modify 1 energy yes +fix 2 all langevin/spin 0.0 0.0 21 + +fix 3 all nve/spin lattice moving +timestep 0.0001 + +# compute and output options + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable magz equal c_out_mag[3] +variable magnorm equal c_out_mag[4] +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_magnorm pe v_emag temp v_tmag etotal +thermo 50 + +compute outsp all property/atom spx spy spz sp fmx fmy fmz +dump 1 all custom 100 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] c_outsp[4] c_outsp[5] c_outsp[6] c_outsp[7] + +run 1000 +Neighbor list info ... + update every 10 steps, delay 20 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 5.90375 + ghost atom cutoff = 5.90375 + binsize = 2.95187, bins = 6 6 6 + 2 neighbor lists, perpetual/occasional/extra = 2 0 0 + (1) pair eam/alloy, perpetual, half/full from (2) + attributes: half, newton on + pair build: halffull/newton + stencil: none + bin: none + (2) pair spin/exchange, perpetual + attributes: full, newton on + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 7.799 | 7.799 | 7.799 Mbytes +Step Time v_magnorm PotEng v_emag Temp v_tmag TotEng + 0 0 0.028733803 -2224.7591 0.20498788 100.03408 -4775.0671 -2218.3068 + 50 0.005 0.028733805 -2224.5402 0.18523081 96.640497 -5356.0839 -2218.3068 + 100 0.01 0.028733812 -2223.9764 0.042321266 87.900506 -24443.229 -2218.3068 + 150 0.015 0.028733818 -2223.1275 -0.20362758 74.73931 5409.1535 -2218.3068 + 200 0.02 0.028733826 -2222.1368 -0.52607498 59.379029 2258.3662 -2218.3068 + 250 0.025 0.028733831 -2221.2102 -0.90147712 45.013713 1422.5963 -2218.3068 + 300 0.03 0.028733827 -2220.558 -1.3121131 34.902868 1037.9833 -2218.3068 + 350 0.035 0.028733817 -2220.3198 -1.7411113 31.20948 805.99534 -2218.3068 + 400 0.04 0.028733807 -2220.5162 -2.1695831 34.253525 646.07984 -2218.3068 + 450 0.045 0.028733802 -2221.0547 -2.581156 42.602248 532.25918 -2218.3068 + 500 0.05 0.028733805 -2221.7784 -2.9684862 53.822318 451.09419 -2218.3068 + 550 0.055 0.028733814 -2222.5255 -3.3338614 65.405638 392.41628 -2218.3068 + 600 0.06 0.028733822 -2223.1693 -3.6823124 75.38674 349.28401 -2218.3068 + 650 0.065 0.028733828 -2223.6339 -4.0182131 82.590109 317.40382 -2218.3068 + 700 0.07 0.028733827 -2223.907 -4.3515619 86.82355 294.41794 -2218.3068 + 750 0.075 0.028733822 -2224.0293 -4.6966825 88.720045 278.64174 -2218.3068 + 800 0.08 0.02873382 -2224.064 -5.0589353 89.257884 267.43696 -2218.3068 + 850 0.085 0.028733824 -2224.0702 -5.4291065 89.354006 257.51219 -2218.3068 + 900 0.09 0.028733833 -2224.0977 -5.7927329 89.780835 246.96548 -2218.3068 + 950 0.095 0.028733835 -2224.1852 -6.1414644 91.136895 236.30442 -2218.3068 + 1000 0.1 0.028733828 -2224.3442 -6.472877 93.602509 227.18362 -2218.3068 +Loop time of 2.57923 on 4 procs for 1000 steps with 500 atoms + +Performance: 3.350 ns/day, 7.165 hours/ns, 387.713 timesteps/s +99.4% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0.52633 | 0.53745 | 0.55615 | 1.6 | 20.84 +Neigh | 0.0030599 | 0.0031699 | 0.0032969 | 0.2 | 0.12 +Comm | 0.10599 | 0.1261 | 0.13713 | 3.4 | 4.89 +Output | 0.0051079 | 0.0051572 | 0.0052865 | 0.1 | 0.20 +Modify | 1.9032 | 1.9052 | 1.9065 | 0.1 | 73.87 +Other | | 0.002202 | | | 0.09 + +Nlocal: 125 ave 138 max 115 min +Histogram: 1 0 0 2 0 0 0 0 0 1 +Nghost: 1099 ave 1109 max 1086 min +Histogram: 1 0 0 0 0 0 2 0 0 1 +Neighs: 4875 ave 5336 max 4525 min +Histogram: 1 0 0 2 0 0 0 0 0 1 +FullNghs: 9750 ave 10764 max 8970 min +Histogram: 1 0 0 2 0 0 0 0 0 1 + +Total # of neighbors = 39000 +Ave neighs/atom = 78 +Neighbor list builds = 7 +Dangerous builds = 0 + +Please see the log.cite file for references relevant to this simulation + +Total wall time: 0:00:02 diff --git a/examples/SPIN/nickel/log.14Apr20.spin.nickel_cubic.g++.1 b/examples/SPIN/nickel/log.14Apr20.spin.nickel_cubic.g++.1 new file mode 100644 index 0000000000000000000000000000000000000000..a9c6dc1ea40f5b9348b7d557cc86fcd13c84f2c6 --- /dev/null +++ b/examples/SPIN/nickel/log.14Apr20.spin.nickel_cubic.g++.1 @@ -0,0 +1,142 @@ +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# fcc nickel in a 3d periodic box + +clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +units metal +atom_style spin + +dimension 3 +boundary p p p + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice fcc 3.524 +Lattice spacing in x,y,z = 3.524 3.524 3.524 +region box block 0.0 5.0 0.0 5.0 0.0 5.0 +create_box 1 box +Created orthogonal box = (0 0 0) to (17.62 17.62 17.62) + 1 by 1 by 1 MPI processor grid +create_atoms 1 box +Created 500 atoms + create_atoms CPU = 0.000916958 secs + +# setting mass, mag. moments, and interactions for cobalt + +mass 1 58.69 + +set group all spin/random 31 0.63 + 500 settings made for spin/random +#set group all spin 0.63 0.0 0.0 1.0 +velocity all create 100 4928459 rot yes dist gaussian + +pair_style hybrid/overlay eam/alloy spin/exchange 4.0 +pair_coeff * * eam/alloy Ni99.eam.alloy Ni +pair_coeff * * spin/exchange exchange 4.0 0.50 0.2280246862 1.229983475 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin cubic -0.0001 0.0 1.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 1.0 zeeman 0.0 0.0 0.0 1.0 +fix_modify 1 energy yes +fix 2 all langevin/spin 0.0 0.0 21 + +fix 3 all nve/spin lattice moving +timestep 0.0001 + +# compute and output options + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable magz equal c_out_mag[3] +variable magnorm equal c_out_mag[4] +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_magnorm pe v_emag temp v_tmag etotal +thermo 50 + +compute outsp all property/atom spx spy spz sp fmx fmy fmz +dump 1 all custom 50 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] c_outsp[4] c_outsp[5] c_outsp[6] c_outsp[7] + +run 1000 +Neighbor list info ... + update every 10 steps, delay 20 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 5.90375 + ghost atom cutoff = 5.90375 + binsize = 2.95187, bins = 6 6 6 + 2 neighbor lists, perpetual/occasional/extra = 2 0 0 + (1) pair eam/alloy, perpetual, half/full from (2) + attributes: half, newton on + pair build: halffull/newton + stencil: none + bin: none + (2) pair spin/exchange, perpetual + attributes: full, newton on + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 7.885 | 7.885 | 7.885 Mbytes +Step Time v_magnorm PotEng v_emag Temp v_tmag TotEng + 0 0 0.028733803 -2224.7478 0.21626519 100.03408 -4301.853 -2218.2955 + 50 0.005 0.028732018 -2224.6158 0.10507552 97.987527 -8506.178 -2218.2955 + 100 0.01 0.028730353 -2224.1105 -0.11492231 90.153208 9821.9545 -2218.2955 + 150 0.015 0.028729179 -2223.2871 -0.42247044 77.388252 2612.6762 -2218.2955 + 200 0.02 0.028730623 -2222.2924 -0.79199024 61.966499 1465.6684 -2218.2955 + 250 0.025 0.02873338 -2221.3383 -1.199896 47.175023 1028.0909 -2218.2955 + 300 0.03 0.028732667 -2220.6421 -1.6277461 36.381371 800.14526 -2218.2955 + 350 0.035 0.028726646 -2220.3547 -2.0613125 31.92537 654.96081 -2218.2955 + 400 0.04 0.028719587 -2220.5116 -2.4878332 34.358607 550.47697 -2218.2955 + 450 0.045 0.028713837 -2221.0303 -2.8957078 42.400515 472.35844 -2218.2955 + 500 0.05 0.028707813 -2221.7502 -3.2769128 53.560569 413.68656 -2218.2955 + 550 0.055 0.028706758 -2222.4909 -3.627757 65.045564 368.28172 -2218.2955 + 600 0.06 0.02871485 -2223.1098 -3.9496387 74.639348 332.20637 -2218.2955 + 650 0.065 0.028726143 -2223.5376 -4.2505578 81.273271 304.61479 -2218.2954 + 700 0.07 0.028732341 -2223.7831 -4.5429323 85.078158 284.47251 -2218.2954 + 750 0.075 0.028732637 -2223.9102 -4.8399235 87.049135 268.76484 -2218.2954 + 800 0.08 0.028731303 -2224.0055 -5.1527352 88.526667 255.14878 -2218.2954 + 850 0.085 0.028728379 -2224.1384 -5.4846326 90.587963 243.32753 -2218.2954 + 900 0.09 0.028721628 -2224.3304 -5.8258679 93.565089 233.2995 -2218.2954 + 950 0.095 0.028715574 -2224.5555 -6.1597678 97.054035 223.54492 -2218.2954 + 1000 0.1 0.02871582 -2224.7714 -6.4766454 100.40218 213.59853 -2218.2954 +Loop time of 4.15073 on 1 procs for 1000 steps with 500 atoms + +Performance: 2.082 ns/day, 11.530 hours/ns, 240.921 timesteps/s +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 1.8514 | 1.8514 | 1.8514 | 0.0 | 44.60 +Neigh | 0.012244 | 0.012244 | 0.012244 | 0.0 | 0.29 +Comm | 0.032971 | 0.032971 | 0.032971 | 0.0 | 0.79 +Output | 0.027815 | 0.027815 | 0.027815 | 0.0 | 0.67 +Modify | 2.2209 | 2.2209 | 2.2209 | 0.0 | 53.51 +Other | | 0.005408 | | | 0.13 + +Nlocal: 500 ave 500 max 500 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 1956 ave 1956 max 1956 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 19500 ave 19500 max 19500 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +FullNghs: 39000 ave 39000 max 39000 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 39000 +Ave neighs/atom = 78 +Neighbor list builds = 7 +Dangerous builds = 0 + + +Please see the log.cite file for references relevant to this simulation + +Total wall time: 0:00:04 diff --git a/examples/SPIN/nickel/log.14Apr20.spin.nickel_cubic.g++.4 b/examples/SPIN/nickel/log.14Apr20.spin.nickel_cubic.g++.4 new file mode 100644 index 0000000000000000000000000000000000000000..9a25cb1fb33cc19b3cd97124709858bd1ac28668 --- /dev/null +++ b/examples/SPIN/nickel/log.14Apr20.spin.nickel_cubic.g++.4 @@ -0,0 +1,142 @@ +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# fcc nickel in a 3d periodic box + +clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +units metal +atom_style spin + +dimension 3 +boundary p p p + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice fcc 3.524 +Lattice spacing in x,y,z = 3.524 3.524 3.524 +region box block 0.0 5.0 0.0 5.0 0.0 5.0 +create_box 1 box +Created orthogonal box = (0 0 0) to (17.62 17.62 17.62) + 1 by 2 by 2 MPI processor grid +create_atoms 1 box +Created 500 atoms + create_atoms CPU = 0.000750065 secs + +# setting mass, mag. moments, and interactions for cobalt + +mass 1 58.69 + +set group all spin/random 31 0.63 + 500 settings made for spin/random +#set group all spin 0.63 0.0 0.0 1.0 +velocity all create 100 4928459 rot yes dist gaussian + +pair_style hybrid/overlay eam/alloy spin/exchange 4.0 +pair_coeff * * eam/alloy Ni99.eam.alloy Ni +pair_coeff * * spin/exchange exchange 4.0 0.50 0.2280246862 1.229983475 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin cubic -0.0001 0.0 1.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 1.0 zeeman 0.0 0.0 0.0 1.0 +fix_modify 1 energy yes +fix 2 all langevin/spin 0.0 0.0 21 + +fix 3 all nve/spin lattice moving +timestep 0.0001 + +# compute and output options + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable magz equal c_out_mag[3] +variable magnorm equal c_out_mag[4] +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_magnorm pe v_emag temp v_tmag etotal +thermo 50 + +compute outsp all property/atom spx spy spz sp fmx fmy fmz +dump 1 all custom 50 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] c_outsp[4] c_outsp[5] c_outsp[6] c_outsp[7] + +run 1000 +Neighbor list info ... + update every 10 steps, delay 20 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 5.90375 + ghost atom cutoff = 5.90375 + binsize = 2.95187, bins = 6 6 6 + 2 neighbor lists, perpetual/occasional/extra = 2 0 0 + (1) pair eam/alloy, perpetual, half/full from (2) + attributes: half, newton on + pair build: halffull/newton + stencil: none + bin: none + (2) pair spin/exchange, perpetual + attributes: full, newton on + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 7.799 | 7.799 | 7.799 Mbytes +Step Time v_magnorm PotEng v_emag Temp v_tmag TotEng + 0 0 0.028733803 -2224.7478 0.21626519 100.03408 -4301.853 -2218.2955 + 50 0.005 0.028732084 -2224.5289 0.19650376 96.640616 -4777.5312 -2218.2955 + 100 0.01 0.028730714 -2223.9652 0.053583455 87.900827 -16071.152 -2218.2955 + 150 0.015 0.028730068 -2223.1163 -0.19237362 74.739735 6053.4731 -2218.2955 + 200 0.02 0.028732658 -2222.1255 -0.5148243 59.37938 2354.2236 -2218.2955 + 250 0.025 0.028736824 -2221.1989 -0.89021871 45.013765 1456.8539 -2218.2955 + 300 0.03 0.028734673 -2220.5467 -1.3008412 34.902634 1054.9996 -2218.2955 + 350 0.035 0.028726346 -2220.3085 -1.7298348 31.209321 816.00699 -2218.2955 + 400 0.04 0.028721022 -2220.5048 -2.1583012 34.253626 652.5234 -2218.2955 + 450 0.045 0.028722238 -2221.0433 -2.5698412 42.602344 536.67287 -2218.2955 + 500 0.05 0.02872462 -2221.767 -2.9571093 53.822165 454.37668 -2218.2955 + 550 0.055 0.02872545 -2222.5141 -3.3223962 65.405129 395.00593 -2218.2955 + 600 0.06 0.02873171 -2223.1579 -3.6706939 75.385253 351.37664 -2218.2955 + 650 0.065 0.028744439 -2223.6223 -4.0063211 82.586428 319.12437 -2218.2954 + 700 0.07 0.028756808 -2223.8952 -4.3392538 86.816357 295.88017 -2218.2954 + 750 0.075 0.028765368 -2224.0172 -4.6838051 88.708218 279.91792 -2218.2954 + 800 0.08 0.028773583 -2224.0515 -5.0453234 89.24056 268.55561 -2218.2954 + 850 0.085 0.028781239 -2224.0574 -5.4146399 89.331407 258.50935 -2218.2954 + 900 0.09 0.028787568 -2224.0847 -5.7774277 89.755159 247.86416 -2218.2954 + 950 0.095 0.028794703 -2224.1722 -6.1254539 91.111184 237.12719 -2218.2954 + 1000 0.1 0.028801253 -2224.3313 -6.4562841 93.577969 227.97114 -2218.2954 +Loop time of 2.49458 on 4 procs for 1000 steps with 500 atoms + +Performance: 3.464 ns/day, 6.929 hours/ns, 400.868 timesteps/s +100.0% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0.52895 | 0.53652 | 0.54117 | 0.6 | 21.51 +Neigh | 0.003077 | 0.0032379 | 0.003371 | 0.2 | 0.13 +Comm | 0.11315 | 0.1193 | 0.12956 | 1.8 | 4.78 +Output | 0.0093312 | 0.0093816 | 0.0094745 | 0.1 | 0.38 +Modify | 1.8206 | 1.8239 | 1.8254 | 0.1 | 73.11 +Other | | 0.00229 | | | 0.09 + +Nlocal: 125 ave 138 max 115 min +Histogram: 1 0 0 2 0 0 0 0 0 1 +Nghost: 1099 ave 1109 max 1086 min +Histogram: 1 0 0 0 0 0 2 0 0 1 +Neighs: 4875 ave 5337 max 4528 min +Histogram: 1 0 0 2 0 0 0 0 0 1 +FullNghs: 9750 ave 10764 max 8970 min +Histogram: 1 0 0 2 0 0 0 0 0 1 + +Total # of neighbors = 39000 +Ave neighs/atom = 78 +Neighbor list builds = 7 +Dangerous builds = 0 + + +Please see the log.cite file for references relevant to this simulation + +Total wall time: 0:00:02 diff --git a/examples/SPIN/nickel/log.19Nov19.spin.nickel.g++.1 b/examples/SPIN/nickel/log.19Nov19.spin.nickel.g++.1 deleted file mode 100644 index 7871d82ae9bbfc6588e5fd3f2dc2e57bcea59aea..0000000000000000000000000000000000000000 --- a/examples/SPIN/nickel/log.19Nov19.spin.nickel.g++.1 +++ /dev/null @@ -1,136 +0,0 @@ -LAMMPS (30 Oct 2019) -# fcc nickel in a 3d periodic box - -clear -units metal -atom_style spin - -dimension 3 -boundary p p p - -# necessary for the serial algorithm (sametag) -atom_modify map array - -lattice fcc 3.524 -Lattice spacing in x,y,z = 3.524 3.524 3.524 -region box block 0.0 5.0 0.0 5.0 0.0 5.0 -create_box 1 box -Created orthogonal box = (0 0 0) to (17.62 17.62 17.62) - 1 by 1 by 1 MPI processor grid -create_atoms 1 box -Created 500 atoms - create_atoms CPU = 0.000484943 secs - -# setting mass, mag. moments, and interactions for cobalt - -mass 1 58.69 - -set group all spin/random 31 0.63 - 500 settings made for spin/random -#set group all spin 0.63 0.0 0.0 1.0 -velocity all create 100 4928459 rot yes dist gaussian - -pair_style hybrid/overlay eam/alloy spin/exchange 4.0 -pair_coeff * * eam/alloy Ni99.eam.alloy Ni -pair_coeff * * spin/exchange exchange 4.0 0.50 0.2280246862 1.229983475 - -neighbor 0.1 bin -neigh_modify every 10 check yes delay 20 - -fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 -fix 2 all langevin/spin 0.0 0.0 21 - -fix 3 all nve/spin lattice moving -timestep 0.0001 - -# compute and output options - -compute out_mag all spin -compute out_pe all pe -compute out_ke all ke -compute out_temp all temp - -variable magz equal c_out_mag[3] -variable magnorm equal c_out_mag[4] -variable emag equal c_out_mag[5] -variable tmag equal c_out_mag[6] - -thermo_style custom step time v_magnorm v_emag temp v_tmag etotal -thermo 50 - -compute outsp all property/atom spx spy spz sp fmx fmy fmz -dump 1 all custom 100 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] c_outsp[4] c_outsp[5] c_outsp[6] c_outsp[7] - -run 1000 -Neighbor list info ... - update every 10 steps, delay 20 steps, check yes - max neighbors/atom: 2000, page size: 100000 - master list distance cutoff = 5.90375 - ghost atom cutoff = 5.90375 - binsize = 2.95187, bins = 6 6 6 - 2 neighbor lists, perpetual/occasional/extra = 2 0 0 - (1) pair eam/alloy, perpetual, half/full from (2) - attributes: half, newton on - pair build: halffull/newton - stencil: none - bin: none - (2) pair spin/exchange, perpetual - attributes: full, newton on - pair build: full/bin/atomonly - stencil: full/bin/3d - bin: standard -Per MPI rank memory allocation (min/avg/max) = 7.885 | 7.885 | 7.885 Mbytes -Step Time v_magnorm v_emag Temp v_tmag TotEng - 0 0 0.028733803 0.40997576 100.03408 -4775.0671 -2218.3068 - 50 0.005 0.028733807 0.070491717 101.47879 -28153.519 -2218.1371 - 100 0.01 0.028733815 -0.70937134 101.7311 2925.8177 -2217.7471 - 150 0.015 0.028733823 -1.853981 99.63039 1197.9339 -2217.1748 - 200 0.02 0.028733828 -3.2679239 94.850105 741.17431 -2216.4678 - 250 0.025 0.028733824 -4.863967 88.444584 550.36979 -2215.6698 - 300 0.03 0.028733807 -6.5763457 82.689581 449.78321 -2214.8136 - 350 0.035 0.028733783 -8.3489158 80.108798 384.32228 -2213.9273 - 400 0.04 0.028733763 -10.120216 82.374947 335.01545 -2213.0417 - 450 0.045 0.028733755 -11.828932 89.814597 296.88965 -2212.1873 - 500 0.05 0.028733762 -13.423712 101.39613 267.51686 -2211.39 - 550 0.055 0.028733783 -14.866724 115.07399 244.96012 -2210.6684 - 600 0.06 0.028733801 -16.135279 128.57849 229.33327 -2210.0342 - 650 0.065 0.028733804 -17.222838 140.22402 220.05718 -2209.4904 - 700 0.07 0.028733795 -18.154813 149.61295 212.95678 -2209.0244 - 750 0.075 0.028733781 -18.996903 157.5814 206.41327 -2208.6034 - 800 0.08 0.028733768 -19.804249 164.92075 203.88977 -2208.1997 - 850 0.085 0.028733752 -20.579151 171.67278 203.42363 -2207.8122 - 900 0.09 0.028733728 -21.294277 177.67238 199.84817 -2207.4547 - 950 0.095 0.028733715 -21.943945 183.2621 194.9614 -2207.1298 - 1000 0.1 0.02873374 -22.551277 188.99284 191.59796 -2206.8262 -Loop time of 4.30614 on 1 procs for 1000 steps with 500 atoms - -Performance: 2.006 ns/day, 11.961 hours/ns, 232.227 timesteps/s -100.0% CPU use with 1 MPI tasks x no OpenMP threads - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Pair | 2.038 | 2.038 | 2.038 | 0.0 | 47.33 -Neigh | 0.01566 | 0.01566 | 0.01566 | 0.0 | 0.36 -Comm | 0.032802 | 0.032802 | 0.032802 | 0.0 | 0.76 -Output | 0.014146 | 0.014146 | 0.014146 | 0.0 | 0.33 -Modify | 2.2003 | 2.2003 | 2.2003 | 0.0 | 51.10 -Other | | 0.005288 | | | 0.12 - -Nlocal: 500 ave 500 max 500 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Nghost: 1956 ave 1956 max 1956 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Neighs: 19504 ave 19504 max 19504 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -FullNghs: 39008 ave 39008 max 39008 min -Histogram: 1 0 0 0 0 0 0 0 0 0 - -Total # of neighbors = 39008 -Ave neighs/atom = 78.016 -Neighbor list builds = 9 -Dangerous builds = 0 - -Please see the log.cite file for references relevant to this simulation - -Total wall time: 0:00:04 diff --git a/examples/SPIN/nickel/log.19Nov19.spin.nickel.g++.4 b/examples/SPIN/nickel/log.19Nov19.spin.nickel.g++.4 deleted file mode 100644 index 638ab5e63516510e9a53e5d15ff54577a8cf9d27..0000000000000000000000000000000000000000 --- a/examples/SPIN/nickel/log.19Nov19.spin.nickel.g++.4 +++ /dev/null @@ -1,136 +0,0 @@ -LAMMPS (30 Oct 2019) -# fcc nickel in a 3d periodic box - -clear -units metal -atom_style spin - -dimension 3 -boundary p p p - -# necessary for the serial algorithm (sametag) -atom_modify map array - -lattice fcc 3.524 -Lattice spacing in x,y,z = 3.524 3.524 3.524 -region box block 0.0 5.0 0.0 5.0 0.0 5.0 -create_box 1 box -Created orthogonal box = (0 0 0) to (17.62 17.62 17.62) - 1 by 2 by 2 MPI processor grid -create_atoms 1 box -Created 500 atoms - create_atoms CPU = 0.000733852 secs - -# setting mass, mag. moments, and interactions for cobalt - -mass 1 58.69 - -set group all spin/random 31 0.63 - 500 settings made for spin/random -#set group all spin 0.63 0.0 0.0 1.0 -velocity all create 100 4928459 rot yes dist gaussian - -pair_style hybrid/overlay eam/alloy spin/exchange 4.0 -pair_coeff * * eam/alloy Ni99.eam.alloy Ni -pair_coeff * * spin/exchange exchange 4.0 0.50 0.2280246862 1.229983475 - -neighbor 0.1 bin -neigh_modify every 10 check yes delay 20 - -fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 -fix 2 all langevin/spin 0.0 0.0 21 - -fix 3 all nve/spin lattice moving -timestep 0.0001 - -# compute and output options - -compute out_mag all spin -compute out_pe all pe -compute out_ke all ke -compute out_temp all temp - -variable magz equal c_out_mag[3] -variable magnorm equal c_out_mag[4] -variable emag equal c_out_mag[5] -variable tmag equal c_out_mag[6] - -thermo_style custom step time v_magnorm v_emag temp v_tmag etotal -thermo 50 - -compute outsp all property/atom spx spy spz sp fmx fmy fmz -dump 1 all custom 100 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] c_outsp[4] c_outsp[5] c_outsp[6] c_outsp[7] - -run 1000 -Neighbor list info ... - update every 10 steps, delay 20 steps, check yes - max neighbors/atom: 2000, page size: 100000 - master list distance cutoff = 5.90375 - ghost atom cutoff = 5.90375 - binsize = 2.95187, bins = 6 6 6 - 2 neighbor lists, perpetual/occasional/extra = 2 0 0 - (1) pair eam/alloy, perpetual, half/full from (2) - attributes: half, newton on - pair build: halffull/newton - stencil: none - bin: none - (2) pair spin/exchange, perpetual - attributes: full, newton on - pair build: full/bin/atomonly - stencil: full/bin/3d - bin: standard -Per MPI rank memory allocation (min/avg/max) = 7.799 | 7.799 | 7.799 Mbytes -Step Time v_magnorm v_emag Temp v_tmag TotEng - 0 0 0.028733803 0.40997576 100.03408 -4775.0671 -2218.1018 - 50 0.005 0.028733805 0.25324083 98.741633 -7863.8744 -2218.1018 - 100 0.01 0.028733812 -0.37320751 97.073875 5622.1863 -2218.1018 - 150 0.015 0.028733819 -1.3971549 94.073447 1625.0258 -2218.1018 - 200 0.02 0.028733825 -2.7238372 89.419944 919.37601 -2218.1018 - 250 0.025 0.028733829 -4.2684428 84.07494 652.18375 -2218.1018 - 300 0.03 0.028733824 -5.9636712 80.06368 512.89077 -2218.1018 - 350 0.035 0.02873381 -7.7386326 79.366702 422.24864 -2218.1018 - 400 0.04 0.028733802 -9.5148059 83.052751 357.60379 -2218.1018 - 450 0.045 0.028733806 -11.234935 91.282747 310.87776 -2218.1018 - 500 0.05 0.02873381 -12.875184 103.49836 275.0224 -2218.1018 - 550 0.055 0.028733808 -14.413473 118.16526 247.85208 -2218.1018 - 600 0.06 0.028733803 -15.812466 132.83837 230.67903 -2218.1018 - 650 0.065 0.028733808 -17.061311 145.41049 222.19476 -2218.1018 - 700 0.07 0.028733818 -18.181903 154.83414 219.42933 -2218.1018 - 750 0.075 0.028733823 -19.176259 160.58645 218.45231 -2218.1018 - 800 0.08 0.028733825 -20.035157 163.02829 214.86596 -2218.1018 - 850 0.085 0.028733825 -20.806548 164.4197 209.86881 -2218.1018 - 900 0.09 0.028733829 -21.571419 167.8571 205.79849 -2218.1018 - 950 0.095 0.028733825 -22.365879 175.00875 201.33088 -2218.1018 - 1000 0.1 0.028733821 -23.133464 184.68305 195.52912 -2218.1018 -Loop time of 2.47211 on 4 procs for 1000 steps with 500 atoms - -Performance: 3.495 ns/day, 6.867 hours/ns, 404.513 timesteps/s -99.9% CPU use with 4 MPI tasks x no OpenMP threads - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Pair | 0.5549 | 0.56708 | 0.58627 | 1.6 | 22.94 -Neigh | 0.0039728 | 0.0041007 | 0.0042598 | 0.2 | 0.17 -Comm | 0.087296 | 0.10802 | 0.11917 | 3.8 | 4.37 -Output | 0.0046923 | 0.0047188 | 0.0047917 | 0.1 | 0.19 -Modify | 1.7832 | 1.7862 | 1.7879 | 0.1 | 72.25 -Other | | 0.002016 | | | 0.08 - -Nlocal: 125 ave 139 max 112 min -Histogram: 1 0 1 0 0 0 1 0 0 1 -Nghost: 1099 ave 1112 max 1085 min -Histogram: 1 0 0 1 0 0 0 1 0 1 -Neighs: 4876 ave 5386 max 4426 min -Histogram: 1 0 0 1 0 1 0 0 0 1 -FullNghs: 9752 ave 10845 max 8737 min -Histogram: 1 0 1 0 0 0 1 0 0 1 - -Total # of neighbors = 39008 -Ave neighs/atom = 78.016 -Neighbor list builds = 9 -Dangerous builds = 0 - -Please see the log.cite file for references relevant to this simulation - -Total wall time: 0:00:02 diff --git a/examples/SPIN/nickel/log.19Nov19.spin.nickel_cubic.g++.1 b/examples/SPIN/nickel/log.19Nov19.spin.nickel_cubic.g++.1 deleted file mode 100644 index 4f3b94df5308dfb1a203e0647fefea78edabbafe..0000000000000000000000000000000000000000 --- a/examples/SPIN/nickel/log.19Nov19.spin.nickel_cubic.g++.1 +++ /dev/null @@ -1,138 +0,0 @@ -LAMMPS (30 Oct 2019) -# fcc nickel in a 3d periodic box - -clear -units metal -atom_style spin - -dimension 3 -boundary p p p - -# necessary for the serial algorithm (sametag) -atom_modify map array - -lattice fcc 3.524 -Lattice spacing in x,y,z = 3.524 3.524 3.524 -region box block 0.0 5.0 0.0 5.0 0.0 5.0 -create_box 1 box -Created orthogonal box = (0 0 0) to (17.62 17.62 17.62) - 1 by 1 by 1 MPI processor grid -create_atoms 1 box -Created 500 atoms - create_atoms CPU = 0.00109196 secs - -# setting mass, mag. moments, and interactions for cobalt - -mass 1 58.69 - -set group all spin/random 31 0.63 - 500 settings made for spin/random -#set group all spin 0.63 0.0 0.0 1.0 -velocity all create 100 4928459 rot yes dist gaussian - -pair_style hybrid/overlay eam/alloy spin/exchange 4.0 -pair_coeff * * eam/alloy Ni99.eam.alloy Ni -pair_coeff * * spin/exchange exchange 4.0 0.50 0.2280246862 1.229983475 - -neighbor 0.1 bin -neigh_modify every 10 check yes delay 20 - -fix 1 all precession/spin cubic -0.0001 0.0 1.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 1.0 zeeman 0.0 0.0 0.0 1.0 -fix_modify 1 energy yes -fix 2 all langevin/spin 0.0 0.0 21 - -fix 3 all nve/spin lattice moving -timestep 0.0001 - -# compute and output options - -compute out_mag all spin -compute out_pe all pe -compute out_ke all ke -compute out_temp all temp - -variable magz equal c_out_mag[3] -variable magnorm equal c_out_mag[4] -variable emag equal c_out_mag[5] -variable tmag equal c_out_mag[6] - -thermo_style custom step time v_magnorm v_emag temp v_tmag etotal -thermo 50 - -compute outsp all property/atom spx spy spz sp fmx fmy fmz -dump 1 all custom 50 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] c_outsp[4] c_outsp[5] c_outsp[6] c_outsp[7] - -run 1000 -Neighbor list info ... - update every 10 steps, delay 20 steps, check yes - max neighbors/atom: 2000, page size: 100000 - master list distance cutoff = 5.90375 - ghost atom cutoff = 5.90375 - binsize = 2.95187, bins = 6 6 6 - 2 neighbor lists, perpetual/occasional/extra = 2 0 0 - (1) pair eam/alloy, perpetual, half/full from (2) - attributes: half, newton on - pair build: halffull/newton - stencil: none - bin: none - (2) pair spin/exchange, perpetual - attributes: full, newton on - pair build: full/bin/atomonly - stencil: full/bin/3d - bin: standard -Per MPI rank memory allocation (min/avg/max) = 7.885 | 7.885 | 7.885 Mbytes -Step Time v_magnorm v_emag Temp v_tmag TotEng - 0 0 0.028733803 0.455085 100.03408 -4301.853 -2218.2955 - 50 0.005 0.028732021 0.11535308 101.47887 -17203.944 -2218.1258 - 100 0.01 0.0287304 -0.665283 101.73105 3119.2267 -2217.7358 - 150 0.015 0.028729403 -1.8105707 99.629794 1226.3803 -2217.1636 - 200 0.02 0.028731067 -3.224763 94.849715 750.93124 -2216.4566 - 250 0.025 0.028732765 -4.8207784 88.447019 555.16456 -2215.6585 - 300 0.03 0.028728169 -6.5331538 82.697813 452.6101 -2214.8022 - 350 0.035 0.02871707 -8.3059526 80.122838 386.20109 -2213.9158 - 400 0.04 0.028706605 -10.077613 82.389555 336.36118 -2213.03 - 450 0.045 0.028701727 -11.78634 89.823176 297.91478 -2212.1758 - 500 0.05 0.028706691 -13.380919 101.39804 268.32933 -2211.3785 - 550 0.055 0.028714065 -14.824128 115.07511 245.62893 -2210.6569 - 600 0.06 0.028713691 -16.093505 128.58093 229.91054 -2210.0225 - 650 0.065 0.028713232 -17.181217 140.22137 220.57591 -2209.4787 - 700 0.07 0.02871245 -18.113035 149.60156 213.40077 -2209.0127 - 750 0.075 0.028712431 -18.954952 157.56849 206.80962 -2208.5916 - 800 0.08 0.02872489 -19.762756 164.91833 204.24742 -2208.1876 - 850 0.085 0.028733709 -20.538757 171.69348 203.73934 -2207.7993 - 900 0.09 0.028737031 -21.256095 177.71981 200.12043 -2207.4406 - 950 0.095 0.028743446 -21.908156 183.31613 195.23386 -2207.1149 - 1000 0.1 0.028751809 -22.516179 189.01672 191.90401 -2206.8111 -Loop time of 4.3661 on 1 procs for 1000 steps with 500 atoms - -Performance: 1.979 ns/day, 12.128 hours/ns, 229.037 timesteps/s -100.0% CPU use with 1 MPI tasks x no OpenMP threads - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Pair | 2.0467 | 2.0467 | 2.0467 | 0.0 | 46.88 -Neigh | 0.015636 | 0.015636 | 0.015636 | 0.0 | 0.36 -Comm | 0.032918 | 0.032918 | 0.032918 | 0.0 | 0.75 -Output | 0.027737 | 0.027737 | 0.027737 | 0.0 | 0.64 -Modify | 2.2379 | 2.2379 | 2.2379 | 0.0 | 51.26 -Other | | 0.005233 | | | 0.12 - -Nlocal: 500 ave 500 max 500 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Nghost: 1956 ave 1956 max 1956 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Neighs: 19504 ave 19504 max 19504 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -FullNghs: 39008 ave 39008 max 39008 min -Histogram: 1 0 0 0 0 0 0 0 0 0 - -Total # of neighbors = 39008 -Ave neighs/atom = 78.016 -Neighbor list builds = 9 -Dangerous builds = 0 - - -Please see the log.cite file for references relevant to this simulation - -Total wall time: 0:00:04 diff --git a/examples/SPIN/nickel/log.19Nov19.spin.nickel_cubic.g++.4 b/examples/SPIN/nickel/log.19Nov19.spin.nickel_cubic.g++.4 deleted file mode 100644 index 1724ef9df8eafd81bf5397a226c9e2e2c7a010f0..0000000000000000000000000000000000000000 --- a/examples/SPIN/nickel/log.19Nov19.spin.nickel_cubic.g++.4 +++ /dev/null @@ -1,138 +0,0 @@ -LAMMPS (30 Oct 2019) -# fcc nickel in a 3d periodic box - -clear -units metal -atom_style spin - -dimension 3 -boundary p p p - -# necessary for the serial algorithm (sametag) -atom_modify map array - -lattice fcc 3.524 -Lattice spacing in x,y,z = 3.524 3.524 3.524 -region box block 0.0 5.0 0.0 5.0 0.0 5.0 -create_box 1 box -Created orthogonal box = (0 0 0) to (17.62 17.62 17.62) - 1 by 2 by 2 MPI processor grid -create_atoms 1 box -Created 500 atoms - create_atoms CPU = 0.000827074 secs - -# setting mass, mag. moments, and interactions for cobalt - -mass 1 58.69 - -set group all spin/random 31 0.63 - 500 settings made for spin/random -#set group all spin 0.63 0.0 0.0 1.0 -velocity all create 100 4928459 rot yes dist gaussian - -pair_style hybrid/overlay eam/alloy spin/exchange 4.0 -pair_coeff * * eam/alloy Ni99.eam.alloy Ni -pair_coeff * * spin/exchange exchange 4.0 0.50 0.2280246862 1.229983475 - -neighbor 0.1 bin -neigh_modify every 10 check yes delay 20 - -fix 1 all precession/spin cubic -0.0001 0.0 1.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 1.0 zeeman 0.0 0.0 0.0 1.0 -fix_modify 1 energy yes -fix 2 all langevin/spin 0.0 0.0 21 - -fix 3 all nve/spin lattice moving -timestep 0.0001 - -# compute and output options - -compute out_mag all spin -compute out_pe all pe -compute out_ke all ke -compute out_temp all temp - -variable magz equal c_out_mag[3] -variable magnorm equal c_out_mag[4] -variable emag equal c_out_mag[5] -variable tmag equal c_out_mag[6] - -thermo_style custom step time v_magnorm v_emag temp v_tmag etotal -thermo 50 - -compute outsp all property/atom spx spy spz sp fmx fmy fmz -dump 1 all custom 50 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] c_outsp[4] c_outsp[5] c_outsp[6] c_outsp[7] - -run 1000 -Neighbor list info ... - update every 10 steps, delay 20 steps, check yes - max neighbors/atom: 2000, page size: 100000 - master list distance cutoff = 5.90375 - ghost atom cutoff = 5.90375 - binsize = 2.95187, bins = 6 6 6 - 2 neighbor lists, perpetual/occasional/extra = 2 0 0 - (1) pair eam/alloy, perpetual, half/full from (2) - attributes: half, newton on - pair build: halffull/newton - stencil: none - bin: none - (2) pair spin/exchange, perpetual - attributes: full, newton on - pair build: full/bin/atomonly - stencil: full/bin/3d - bin: standard -Per MPI rank memory allocation (min/avg/max) = 7.799 | 7.799 | 7.799 Mbytes -Step Time v_magnorm v_emag Temp v_tmag TotEng - 0 0 0.028733803 0.455085 100.03408 -4301.853 -2218.0905 - 50 0.005 0.028732088 0.2980989 98.74184 -6680.4308 -2218.0904 - 100 0.01 0.02873076 -0.32911738 97.074246 6374.7026 -2218.09 - 150 0.015 0.028730298 -1.3537059 94.073558 1676.9365 -2218.0897 - 200 0.02 0.028733079 -2.6807428 89.419616 934.03305 -2218.0895 - 250 0.025 0.028735725 -4.2256641 84.074249 658.72816 -2218.0893 - 300 0.03 0.028728939 -5.9209085 80.063263 516.58161 -2218.0893 - 350 0.035 0.028716731 -7.6957087 79.36782 424.59625 -2218.0893 - 400 0.04 0.02871114 -9.4720832 83.055773 359.18204 -2218.0893 - 450 0.045 0.02870879 -11.19254 91.28713 312.02076 -2218.0891 - 500 0.05 0.028708873 -12.832707 103.50343 275.92991 -2218.0892 - 550 0.055 0.028710315 -14.370603 118.16778 248.59763 -2218.0893 - 600 0.06 0.028707016 -15.769641 132.83264 231.2886 -2218.089 - 650 0.065 0.028706727 -17.018362 145.39247 222.70304 -2218.0888 - 700 0.07 0.028710482 -18.137792 154.80131 219.85838 -2218.0889 - 750 0.075 0.028705169 -19.130471 160.53663 218.83811 -2218.0892 - 800 0.08 0.028695336 -19.988452 162.95918 215.21456 -2218.089 - 850 0.085 0.028688393 -20.758389 164.33238 210.21496 -2218.0889 - 900 0.09 0.028684101 -21.521505 167.76167 206.14977 -2218.089 - 950 0.095 0.028684705 -22.314351 174.918 201.65878 -2218.0891 - 1000 0.1 0.028691284 -23.080026 184.60192 195.8385 -2218.0893 -Loop time of 2.5947 on 4 procs for 1000 steps with 500 atoms - -Performance: 3.330 ns/day, 7.207 hours/ns, 385.402 timesteps/s -99.9% CPU use with 4 MPI tasks x no OpenMP threads - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Pair | 0.56409 | 0.58139 | 0.605 | 2.0 | 22.41 -Neigh | 0.0039618 | 0.0041058 | 0.0042889 | 0.2 | 0.16 -Comm | 0.095324 | 0.12147 | 0.13892 | 4.8 | 4.68 -Output | 0.008945 | 0.0089793 | 0.0090477 | 0.0 | 0.35 -Modify | 1.8745 | 1.8765 | 1.8795 | 0.1 | 72.32 -Other | | 0.002217 | | | 0.09 - -Nlocal: 125 ave 139 max 112 min -Histogram: 1 0 1 0 0 0 1 0 0 1 -Nghost: 1099 ave 1112 max 1085 min -Histogram: 1 0 0 1 0 0 0 1 0 1 -Neighs: 4876 ave 5385 max 4427 min -Histogram: 1 0 0 1 0 1 0 0 0 1 -FullNghs: 9752 ave 10845 max 8737 min -Histogram: 1 0 1 0 0 0 1 0 0 1 - -Total # of neighbors = 39008 -Ave neighs/atom = 78.016 -Neighbor list builds = 9 -Dangerous builds = 0 - - -Please see the log.cite file for references relevant to this simulation - -Total wall time: 0:00:02 diff --git a/examples/SPIN/read_restart/in.spin.read_data b/examples/SPIN/read_restart/in.spin.read_data index e788ecf67eff4657a0cc7270178fd4fd59b0fddf..b2b55a9fcbc7b90669e472c2d2bc39545bf35aa1 100644 --- a/examples/SPIN/read_restart/in.spin.read_data +++ b/examples/SPIN/read_restart/in.spin.read_data @@ -35,8 +35,8 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -thermo 10 -thermo_style custom step time v_magnorm v_emag v_tmag temp etotal +thermo 20 +thermo_style custom step time v_magnorm pe v_emag v_tmag temp etotal thermo_modify format float %20.15g compute outsp all property/atom spx spy spz sp fmx fmy fmz diff --git a/examples/SPIN/read_restart/in.spin.restart b/examples/SPIN/read_restart/in.spin.restart index ccce25b2544a664875c98ac4e1db1e666fe1eb49..985da65eb484d586f99a9e2cccfe682934087a76 100644 --- a/examples/SPIN/read_restart/in.spin.restart +++ b/examples/SPIN/read_restart/in.spin.restart @@ -39,8 +39,8 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -thermo 10 -thermo_style custom step time v_magnorm v_emag v_tmag temp etotal +thermo 20 +thermo_style custom step time v_magnorm pe v_emag v_tmag temp etotal thermo_modify format float %20.15g compute outsp all property/atom spx spy spz sp fmx fmy fmz diff --git a/examples/SPIN/read_restart/in.spin.write_restart b/examples/SPIN/read_restart/in.spin.write_restart index c1271010937f37f2051240e4055b831657ef2987..19ab8e6b30abbe76bbcb7caf5b42a45985ef982a 100644 --- a/examples/SPIN/read_restart/in.spin.write_restart +++ b/examples/SPIN/read_restart/in.spin.write_restart @@ -44,7 +44,7 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -thermo_style custom step time v_magnorm v_emag temp etotal +thermo_style custom step time v_magnorm pe v_emag temp etotal thermo 100 compute outsp all property/atom spx spy spz sp fmx fmy fmz @@ -52,4 +52,3 @@ dump 100 all custom 1 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[ run 1000 write_restart restart_hcp_cobalt.equil - diff --git a/examples/SPIN/read_restart/log.19Nov19.spin.read_data.g++.1 b/examples/SPIN/read_restart/log.14Apr20.spin.read_data.g++.1 similarity index 51% rename from examples/SPIN/read_restart/log.19Nov19.spin.read_data.g++.1 rename to examples/SPIN/read_restart/log.14Apr20.spin.read_data.g++.1 index d5c38a996701ca7544ee2752824422a97c32a0b0..4a744700f9fd7e350ecb22fdd212dffd494863ca 100644 --- a/examples/SPIN/read_restart/log.19Nov19.spin.read_data.g++.1 +++ b/examples/SPIN/read_restart/log.14Apr20.spin.read_data.g++.1 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task units metal dimension 3 boundary p p p @@ -12,7 +14,7 @@ read_data Norm_randXY_8x8x32.data 1 by 1 by 1 MPI processor grid reading atoms ... 8192 atoms - read_data CPU = 0.0127251 secs + read_data CPU = 0.022048 secs mass 1 58.93 @@ -41,8 +43,8 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -thermo 10 -thermo_style custom step time v_magnorm v_emag v_tmag temp etotal +thermo 20 +thermo_style custom step time v_magnorm pe v_emag v_tmag temp etotal thermo_modify format float %20.15g compute outsp all property/atom spx spy spz sp fmx fmy fmz @@ -66,33 +68,28 @@ Neighbor list info ... pair build: full/bin/atomonly stencil: full/bin/3d bin: standard -Per MPI rank memory allocation (min/avg/max) = 19.68 | 19.68 | 19.68 Mbytes -Step Time v_magnorm v_emag v_tmag Temp TotEng - 0 0 0.0177864461018737 -1323.65841279979 1274.398774669 0 -36558.7284872918 - 10 0.001 0.0177864363786085 -1323.66900862123 1270.76616762926 0.0100007025152235 -36558.7231900452 - 20 0.002 0.0177864377251544 -1323.70032173151 1259.90270462032 0.0394803272360477 -36558.7075350597 - 30 0.003 0.0177864511986563 -1323.75117991179 1243.50772254923 0.0871132837928349 -36558.6821082609 - 40 0.004 0.0177864729727686 -1323.81992477224 1223.91535595806 0.150986538096776 -36558.6477386289 - 50 0.005 0.017786495620418 -1323.90456907402 1203.45497846157 0.22877054554493 -36558.6054195788 - 60 0.006 0.0177865119365897 -1324.00293472823 1183.95496070422 0.317876389336898 -36558.556239967 - 70 0.007 0.0177865186121948 -1324.11277680481 1166.52445270059 0.415601818818485 -36558.5013220755 - 80 0.008 0.0177865171615599 -1324.23190710734 1151.59958937508 0.519276751090729 -36558.4417598279 - 90 0.009 0.0177865117923882 -1324.35831839963 1139.14485136813 0.626407059487507 -36558.3785566998 - 100 0.01 0.0177865063215865 -1324.49029089774 1128.88117273962 0.734797362055872 -36558.3125725035 -Loop time of 14.8985 on 1 procs for 100 steps with 8192 atoms - -Performance: 0.058 ns/day, 413.847 hours/ns, 6.712 timesteps/s -99.6% CPU use with 1 MPI tasks x no OpenMP threads +Per MPI rank memory allocation (min/avg/max) = 19.99 | 19.99 | 19.99 Mbytes +Step Time v_magnorm PotEng v_emag v_tmag Temp TotEng + 0 0 0.0177864461018737 -36558.7284872918 -661.829206399896 1274.398774669 0 -36558.7284872918 + 20 0.002 0.0177864377256184 -36558.7389378387 -661.839683504936 1259.94171978912 0.00986992693139795 -36558.7284878577 + 40 0.004 0.017786472977471 -36558.7684525639 -661.869582914286 1224.05894016152 0.0377451568363827 -36558.7284891299 + 60 0.006 0.0177865119543331 -36558.8126238543 -661.915330492427 1184.24369688088 0.0794631076347515 -36558.728490712 + 80 0.008 0.0177865172048059 -36558.8659242367 -661.972562482488 1152.05459929593 0.129803482511904 -36558.7284922233 + 100 0.01 0.0177865063752424 -36558.9229549739 -662.037138807935 1129.51470280479 0.183667498513087 -36558.7284933644 +Loop time of 14.3276 on 1 procs for 100 steps with 8192 atoms + +Performance: 0.060 ns/day, 397.988 hours/ns, 6.980 timesteps/s +99.7% CPU use with 1 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 4.5996 | 4.5996 | 4.5996 | 0.0 | 30.87 -Neigh | 3.6 | 3.6 | 3.6 | 0.0 | 24.16 -Comm | 0.057512 | 0.057512 | 0.057512 | 0.0 | 0.39 -Output | 2.4463 | 2.4463 | 2.4463 | 0.0 | 16.42 -Modify | 4.1766 | 4.1766 | 4.1766 | 0.0 | 28.03 -Other | | 0.01854 | | | 0.12 +Pair | 4.0409 | 4.0409 | 4.0409 | 0.0 | 28.20 +Neigh | 3.6219 | 3.6219 | 3.6219 | 0.0 | 25.28 +Comm | 0.055327 | 0.055327 | 0.055327 | 0.0 | 0.39 +Output | 2.4259 | 2.4259 | 2.4259 | 0.0 | 16.93 +Modify | 4.1688 | 4.1688 | 4.1688 | 0.0 | 29.10 +Other | | 0.01477 | | | 0.10 Nlocal: 8192 ave 8192 max 8192 min Histogram: 1 0 0 0 0 0 0 0 0 0 @@ -110,4 +107,4 @@ Dangerous builds not checked Please see the log.cite file for references relevant to this simulation -Total wall time: 0:00:15 +Total wall time: 0:00:14 diff --git a/examples/SPIN/read_restart/log.19Nov19.spin.read_data.g++.4 b/examples/SPIN/read_restart/log.14Apr20.spin.read_data.g++.4 similarity index 51% rename from examples/SPIN/read_restart/log.19Nov19.spin.read_data.g++.4 rename to examples/SPIN/read_restart/log.14Apr20.spin.read_data.g++.4 index decd7f66de713e60b53f838854d2259bee0d94ea..6ce648abf35dabe4b4f685064d4e1e4337ddd072 100644 --- a/examples/SPIN/read_restart/log.19Nov19.spin.read_data.g++.4 +++ b/examples/SPIN/read_restart/log.14Apr20.spin.read_data.g++.4 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task units metal dimension 3 boundary p p p @@ -12,7 +14,7 @@ read_data Norm_randXY_8x8x32.data 1 by 1 by 4 MPI processor grid reading atoms ... 8192 atoms - read_data CPU = 0.0103889 secs + read_data CPU = 0.013634 secs mass 1 58.93 @@ -41,8 +43,8 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -thermo 10 -thermo_style custom step time v_magnorm v_emag v_tmag temp etotal +thermo 20 +thermo_style custom step time v_magnorm pe v_emag v_tmag temp etotal thermo_modify format float %20.15g compute outsp all property/atom spx spy spz sp fmx fmy fmz @@ -66,33 +68,28 @@ Neighbor list info ... pair build: full/bin/atomonly stencil: full/bin/3d bin: standard -Per MPI rank memory allocation (min/avg/max) = 8.422 | 8.508 | 8.751 Mbytes -Step Time v_magnorm v_emag v_tmag Temp TotEng - 0 0 0.0177864461018739 -1323.65841279979 1274.398774669 0 -37220.5576936996 - 10 0.001 0.0177864050983603 -1323.66900862096 1270.76618998865 0.0100007022583634 -37220.5576943555 - 20 0.002 0.0177863936833753 -1323.70032172864 1259.90276925185 0.0394803245313843 -37220.5576959254 - 30 0.003 0.0177864120892274 -1323.75117990111 1243.50783331745 0.087113273744231 -37220.5576982173 - 40 0.004 0.0177864533855652 -1323.8199247464 1223.91551103958 0.150986513868405 -37220.557701015 - 50 0.005 0.0177865078997919 -1323.90456902433 1203.45516787752 0.228770499151177 -37220.5577041159 - 60 0.006 0.0177865576955448 -1324.0029346455 1183.95517338662 0.317876312538184 -37220.5577073314 - 70 0.007 0.0177865860816348 -1324.11277667948 1166.52467969539 0.415601703342581 -37220.5577104775 - 80 0.008 0.0177865881669081 -1324.23190693081 1151.59982868413 0.519276589926842 -37220.557713381 - 90 0.009 0.0177865780982769 -1324.35831816525 1139.14509878533 0.626406847905203 -37220.557715901 - 100 0.01 0.017786564605084 -1324.49029060173 1128.88143013641 0.734797098519806 -37220.557717952 -Loop time of 4.32342 on 4 procs for 100 steps with 8192 atoms - -Performance: 0.200 ns/day, 120.095 hours/ns, 23.130 timesteps/s -99.8% CPU use with 4 MPI tasks x no OpenMP threads +Per MPI rank memory allocation (min/avg/max) = 8.961 | 9.047 | 9.29 Mbytes +Step Time v_magnorm PotEng v_emag v_tmag Temp TotEng + 0 0 0.0177864461018739 -36558.7284872997 -661.829206399894 1274.398774669 0 -36558.7284872997 + 20 0.002 0.0177863981273124 -36558.7389378386 -661.839683504262 1259.94177798388 0.00986992629371963 -36558.7284878582 + 40 0.004 0.0177864622701489 -36558.7684525586 -661.869582908114 1224.05908191331 0.0377451510479599 -36558.7284891308 + 60 0.006 0.0177865625037858 -36558.8126238326 -661.915330472361 1184.24389640891 0.0794630890177406 -36558.72849071 + 80 0.008 0.0177865898045059 -36558.8659241943 -661.972562439245 1152.05483020781 0.129803443061299 -36558.7284922226 + 100 0.01 0.017786565190115 -36558.9229549058 -662.037138735432 1129.51495182843 0.183667434061771 -36558.7284933646 +Loop time of 4.35911 on 4 procs for 100 steps with 8192 atoms + +Performance: 0.198 ns/day, 121.086 hours/ns, 22.940 timesteps/s +99.7% CPU use with 4 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 1.185 | 1.1925 | 1.1991 | 0.5 | 27.58 -Neigh | 0.93459 | 0.93934 | 0.94983 | 0.6 | 21.73 -Comm | 0.042462 | 0.059373 | 0.069249 | 4.1 | 1.37 -Output | 0.61823 | 0.63273 | 0.64528 | 1.3 | 14.63 -Modify | 1.4827 | 1.4955 | 1.5099 | 0.8 | 34.59 -Other | | 0.003968 | | | 0.09 +Pair | 1.0924 | 1.1043 | 1.1117 | 0.7 | 25.33 +Neigh | 0.93575 | 0.94926 | 0.98325 | 2.0 | 21.78 +Comm | 0.044663 | 0.088288 | 0.11128 | 8.7 | 2.03 +Output | 0.64199 | 0.6587 | 0.67226 | 1.4 | 15.11 +Modify | 1.5412 | 1.5535 | 1.5706 | 0.9 | 35.64 +Other | | 0.005046 | | | 0.12 Nlocal: 2048 ave 2061 max 2035 min Histogram: 1 0 0 1 0 0 1 0 0 1 diff --git a/examples/SPIN/read_restart/log.19Nov19.spin.restart.g++.1 b/examples/SPIN/read_restart/log.14Apr20.spin.restart.g++.1 similarity index 53% rename from examples/SPIN/read_restart/log.19Nov19.spin.restart.g++.1 rename to examples/SPIN/read_restart/log.14Apr20.spin.restart.g++.1 index c7544dedfad129b06f2f0a1bcdc9ee6454028fb4..652f7da4742f166c05d29cf6b2f1167886da1828 100644 --- a/examples/SPIN/read_restart/log.19Nov19.spin.restart.g++.1 +++ b/examples/SPIN/read_restart/log.14Apr20.spin.restart.g++.1 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # start a spin-lattice simulation from a data file units metal atom_style spin @@ -10,13 +12,13 @@ boundary p p p atom_modify map array read_restart restart_hcp_cobalt.equil -WARNING: Restart file used different # of processors: 4 vs. 1 (../read_restart.cpp:742) +WARNING: Restart file used different # of processors: 4 vs. 1 (../read_restart.cpp:736) restoring atom style spin from restart orthogonal box = (0 0 0) to (12.5355 21.7121 20.4704) 1 by 1 by 1 MPI processor grid restoring pair style spin/exchange from restart 500 atoms - read_restart CPU = 0.000396967 secs + read_restart CPU = 0.00179696 secs # setting mass, mag. moments, and interactions @@ -47,8 +49,8 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -thermo 10 -thermo_style custom step time v_magnorm v_emag v_tmag temp etotal +thermo 20 +thermo_style custom step time v_magnorm pe v_emag v_tmag temp etotal thermo_modify format float %20.15g compute outsp all property/atom spx spy spz sp fmx fmy fmz @@ -72,33 +74,28 @@ Neighbor list info ... pair build: full/bin/atomonly stencil: full/bin/3d bin: standard -Per MPI rank memory allocation (min/avg/max) = 6.816 | 6.816 | 6.816 Mbytes -Step Time v_magnorm v_emag v_tmag Temp TotEng - 1000 0 0.108317262557656 -10.7649197733649 2538.4247868621 0 -2200.38241212222 - 1010 0.001 0.108317281393701 -10.7743543303502 2527.22692799144 0.146167392153018 -2200.3776953858 - 1020 0.002 0.108317318482233 -10.8022550516195 2509.47863584151 0.577304300153637 -2200.36374625815 - 1030 0.003 0.108317366763426 -10.8476659807571 2487.5614649682 1.27529086243277 -2200.34104256596 - 1040 0.004 0.108317415532953 -10.9092708333549 2463.97963921611 2.21443906326453 -2200.31024227618 - 1050 0.005 0.108317453851058 -10.98553406179 2440.70473642157 3.36396898978859 -2200.27211302201 - 1060 0.006 0.108317473584086 -11.0748008072977 2418.66477599297 4.68991434259399 -2200.22748216359 - 1070 0.007 0.108317471632913 -11.175325501803 2397.59274785581 6.15596240129541 -2200.17722244953 - 1080 0.008 0.108317450667394 -11.2852665400894 2376.32871275528 7.7237909750654 -2200.12225459883 - 1090 0.009 0.108317417687893 -11.4027246787047 2353.52646917989 9.35409156720424 -2200.06352807392 - 1100 0.01 0.108317381194814 -11.52585602487 2328.41541723561 11.0087303030003 -2200.0019646458 -Loop time of 0.964681 on 1 procs for 100 steps with 500 atoms - -Performance: 0.896 ns/day, 26.797 hours/ns, 103.661 timesteps/s -100.0% CPU use with 1 MPI tasks x no OpenMP threads +Per MPI rank memory allocation (min/avg/max) = 7.422 | 7.422 | 7.422 Mbytes +Step Time v_magnorm PotEng v_emag v_tmag Temp TotEng + 1000 0 0.108317262557656 -2200.38241212222 -5.38245988668244 2538.4247868621 0 -2200.38241212222 + 1020 0.002 0.108317318495042 -2200.39172132133 -5.39179331134703 2513.42968070374 0.144319963844279 -2200.38241256643 + 1040 0.004 0.108317415558744 -2200.41811580407 -5.418541526637 2478.87571728648 0.553516420254567 -2200.38241354532 + 1060 0.006 0.108317473592946 -2200.45801216332 -5.45990062771403 2449.77257658726 1.17203792179707 -2200.38241476526 + 1080 0.008 0.108317450745396 -2200.5068824087 -5.51245983698347 2427.25022669715 1.92968606059505 -2200.3824160902 + 1100 0.01 0.108317381572202 -2200.55976028827 -5.57250071024394 2400.86131889957 2.74946927499959 -2200.38241728649 +Loop time of 0.954493 on 1 procs for 100 steps with 500 atoms + +Performance: 0.905 ns/day, 26.514 hours/ns, 104.768 timesteps/s +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 0.29842 | 0.29842 | 0.29842 | 0.0 | 30.93 -Neigh | 0.25359 | 0.25359 | 0.25359 | 0.0 | 26.29 -Comm | 0.0069926 | 0.0069926 | 0.0069926 | 0.0 | 0.72 -Output | 0.14398 | 0.14398 | 0.14398 | 0.0 | 14.93 -Modify | 0.26045 | 0.26045 | 0.26045 | 0.0 | 27.00 -Other | | 0.001249 | | | 0.13 +Pair | 0.27043 | 0.27043 | 0.27043 | 0.0 | 28.33 +Neigh | 0.26148 | 0.26148 | 0.26148 | 0.0 | 27.40 +Comm | 0.0071123 | 0.0071123 | 0.0071123 | 0.0 | 0.75 +Output | 0.14169 | 0.14169 | 0.14169 | 0.0 | 14.84 +Modify | 0.2726 | 0.2726 | 0.2726 | 0.0 | 28.56 +Other | | 0.001178 | | | 0.12 Nlocal: 500 ave 500 max 500 min Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/read_restart/log.19Nov19.spin.restart.g++.4 b/examples/SPIN/read_restart/log.14Apr20.spin.restart.g++.4 similarity index 53% rename from examples/SPIN/read_restart/log.19Nov19.spin.restart.g++.4 rename to examples/SPIN/read_restart/log.14Apr20.spin.restart.g++.4 index 6443d341cd06049525e3a8f1e9a46f09265bb24f..c22c672f6b82fdd030f3a971bc4161cdfc0562e8 100644 --- a/examples/SPIN/read_restart/log.19Nov19.spin.restart.g++.4 +++ b/examples/SPIN/read_restart/log.14Apr20.spin.restart.g++.4 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # start a spin-lattice simulation from a data file units metal atom_style spin @@ -15,7 +17,7 @@ read_restart restart_hcp_cobalt.equil 1 by 2 by 2 MPI processor grid restoring pair style spin/exchange from restart 500 atoms - read_restart CPU = 0.000922918 secs + read_restart CPU = 0.00173593 secs # setting mass, mag. moments, and interactions @@ -46,8 +48,8 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -thermo 10 -thermo_style custom step time v_magnorm v_emag v_tmag temp etotal +thermo 20 +thermo_style custom step time v_magnorm pe v_emag v_tmag temp etotal thermo_modify format float %20.15g compute outsp all property/atom spx spy spz sp fmx fmy fmz @@ -71,33 +73,28 @@ Neighbor list info ... pair build: full/bin/atomonly stencil: full/bin/3d bin: standard -Per MPI rank memory allocation (min/avg/max) = 6.704 | 6.704 | 6.704 Mbytes -Step Time v_magnorm v_emag v_tmag Temp TotEng - 1000 0 0.108317262557656 -10.7649197733649 2538.4247868621 0 -2205.7648720085 - 1010 0.001 0.108317290362529 -10.7743543303489 2527.22680531097 0.146167392137698 -2205.76487255096 - 1020 0.002 0.108317316207642 -10.8022550521284 2509.47840782645 0.577304308061779 -2205.76487378396 - 1030 0.003 0.108317335980455 -10.8476659832667 2487.56119588937 1.27529090130452 -2205.76487555634 - 1040 0.004 0.108317347902639 -10.9092708400684 2463.97936529674 2.21443916694928 -2205.76487769286 - 1050 0.005 0.108317349786635 -10.9855340757384 2440.7044253165 3.36396920446814 -2205.76488005291 - 1060 0.006 0.108317342445881 -11.0748008315013 2418.66438763214 4.68991471343994 -2205.76488256723 - 1070 0.007 0.10831733355314 -11.1753255362286 2397.59228728929 6.15596292529133 -2205.76488520046 - 1080 0.008 0.108317320750099 -11.2852665775656 2376.32820919279 7.7237915384778 -2205.76488786888 - 1090 0.009 0.108317304079233 -11.402724701646 2353.52588586648 9.35409189724323 -2205.7648904133 - 1100 0.01 0.108317284409678 -11.5258560062539 2328.41472376239 11.0087299868288 -2205.76489265829 -Loop time of 0.410707 on 4 procs for 100 steps with 500 atoms - -Performance: 2.104 ns/day, 11.409 hours/ns, 243.483 timesteps/s -99.7% CPU use with 4 MPI tasks x no OpenMP threads +Per MPI rank memory allocation (min/avg/max) = 7.324 | 7.324 | 7.324 Mbytes +Step Time v_magnorm PotEng v_emag v_tmag Temp TotEng + 1000 0 0.108317262557656 -2200.38241212182 -5.38245988668244 2538.4247868621 0 -2200.38241212182 + 1020 0.002 0.108317316216432 -2200.39172132147 -5.39179331147409 2513.42945241007 0.14431996581917 -2200.38241256644 + 1040 0.004 0.108317347939802 -2200.41811580574 -5.41854152831072 2478.87544274124 0.553516446104432 -2200.38241354532 + 1060 0.006 0.108317342440309 -2200.45801216927 -5.45990063373049 2449.77218633122 1.17203801398165 -2200.38241476526 + 1080 0.008 0.108317320345284 -2200.50688241767 -5.51245984623572 2427.2497145488 1.92968619968329 -2200.3824160902 + 1100 0.01 0.10831728372281 -2200.55976028296 -5.57250070536486 2400.86059511731 2.74946919265255 -2200.38241728649 +Loop time of 0.405615 on 4 procs for 100 steps with 500 atoms + +Performance: 2.130 ns/day, 11.267 hours/ns, 246.539 timesteps/s +99.7% CPU use with 4 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 0.083043 | 0.083714 | 0.084202 | 0.1 | 20.38 -Neigh | 0.063158 | 0.064429 | 0.065314 | 0.3 | 15.69 -Comm | 0.012237 | 0.013588 | 0.014798 | 0.8 | 3.31 -Output | 0.039088 | 0.040653 | 0.042176 | 0.6 | 9.90 -Modify | 0.20645 | 0.20795 | 0.20945 | 0.2 | 50.63 -Other | | 0.0003724 | | | 0.09 +Pair | 0.075661 | 0.076798 | 0.077343 | 0.2 | 18.93 +Neigh | 0.063154 | 0.064974 | 0.066991 | 0.5 | 16.02 +Comm | 0.012538 | 0.013787 | 0.015151 | 0.8 | 3.40 +Output | 0.039155 | 0.040842 | 0.042502 | 0.6 | 10.07 +Modify | 0.20709 | 0.20883 | 0.21036 | 0.3 | 51.49 +Other | | 0.0003826 | | | 0.09 Nlocal: 125 ave 127 max 122 min Histogram: 1 0 0 0 1 0 0 0 0 2 diff --git a/examples/SPIN/read_restart/log.19Nov19.spin.write_restart.g++.1 b/examples/SPIN/read_restart/log.14Apr20.spin.write_restart.g++.1 similarity index 59% rename from examples/SPIN/read_restart/log.19Nov19.spin.write_restart.g++.1 rename to examples/SPIN/read_restart/log.14Apr20.spin.write_restart.g++.1 index 9dbe1a354805b687d5f66aaddb5f66027fa36a24..8ad14c55c05d2d4bf43f751df6c75bc175bdf745 100644 --- a/examples/SPIN/read_restart/log.19Nov19.spin.write_restart.g++.1 +++ b/examples/SPIN/read_restart/log.14Apr20.spin.write_restart.g++.1 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # fcc cobalt in a 3d periodic box units metal @@ -18,7 +20,7 @@ Created orthogonal box = (0 0 0) to (12.5355 21.7121 20.4704) 1 by 1 by 1 MPI processor grid create_atoms 1 box Created 500 atoms - create_atoms CPU = 0.000552893 secs + create_atoms CPU = 0.000952005 secs # setting mass, mag. moments, and interactions for cobalt @@ -51,7 +53,7 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -thermo_style custom step time v_magnorm v_emag temp etotal +thermo_style custom step time v_magnorm pe v_emag temp etotal thermo 100 compute outsp all property/atom spx spy spz sp fmx fmy fmz @@ -71,32 +73,32 @@ Neighbor list info ... stencil: full/bin/3d bin: standard Per MPI rank memory allocation (min/avg/max) = 6.947 | 6.947 | 6.947 Mbytes -Step Time v_magnorm v_emag Temp TotEng - 0 0 0.076558814 1.7982359 0 0.89911794 - 100 0.01 0.077628154 0.73387834 0 0.36693917 - 200 0.02 0.076678996 -0.4048463 0 -0.20242315 - 300 0.03 0.079174837 -1.3519103 0 -0.67595514 - 400 0.04 0.085031632 -3.0345702 0 -1.5172851 - 500 0.05 0.08702747 -4.0853256 0 -2.0426628 - 600 0.06 0.087066482 -5.259549 0 -2.6297745 - 700 0.07 0.089788894 -6.629076 0 -3.314538 - 800 0.08 0.091699611 -8.0574087 0 -4.0287043 - 900 0.09 0.090038899 -9.2012019 0 -4.600601 - 1000 0.1 0.093257309 -10.470452 0 -5.2352261 -Loop time of 3.37852 on 1 procs for 1000 steps with 500 atoms - -Performance: 2.557 ns/day, 9.385 hours/ns, 295.987 timesteps/s -99.6% CPU use with 1 MPI tasks x no OpenMP threads +Step Time v_magnorm PotEng v_emag Temp TotEng + 0 0 0.076558814 0.89911794 0.89911794 0 0.89911794 + 100 0.01 0.077628154 0.36693917 0.36693917 0 0.36693917 + 200 0.02 0.076678996 -0.20242315 -0.20242315 0 -0.20242315 + 300 0.03 0.079174837 -0.67595514 -0.67595514 0 -0.67595514 + 400 0.04 0.085031632 -1.5172851 -1.5172851 0 -1.5172851 + 500 0.05 0.08702747 -2.0426628 -2.0426628 0 -2.0426628 + 600 0.06 0.087066482 -2.6297745 -2.6297745 0 -2.6297745 + 700 0.07 0.089788894 -3.314538 -3.314538 0 -3.314538 + 800 0.08 0.091699611 -4.0287043 -4.0287043 0 -4.0287043 + 900 0.09 0.090038899 -4.600601 -4.600601 0 -4.600601 + 1000 0.1 0.093257309 -5.2352261 -5.2352261 0 -5.2352261 +Loop time of 3.30071 on 1 procs for 1000 steps with 500 atoms + +Performance: 2.618 ns/day, 9.169 hours/ns, 302.965 timesteps/s +99.3% CPU use with 1 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 0.45808 | 0.45808 | 0.45808 | 0.0 | 13.56 +Pair | 0.3844 | 0.3844 | 0.3844 | 0.0 | 11.65 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.019707 | 0.019707 | 0.019707 | 0.0 | 0.58 -Output | 1.3865 | 1.3865 | 1.3865 | 0.0 | 41.04 -Modify | 1.5106 | 1.5106 | 1.5106 | 0.0 | 44.71 -Other | | 0.003624 | | | 0.11 +Comm | 0.019863 | 0.019863 | 0.019863 | 0.0 | 0.60 +Output | 1.3844 | 1.3844 | 1.3844 | 0.0 | 41.94 +Modify | 1.5084 | 1.5084 | 1.5084 | 0.0 | 45.70 +Other | | 0.00367 | | | 0.11 Nlocal: 500 ave 500 max 500 min Histogram: 1 0 0 0 0 0 0 0 0 0 @@ -113,7 +115,6 @@ Neighbor list builds = 0 Dangerous builds = 0 write_restart restart_hcp_cobalt.equil - Please see the log.cite file for references relevant to this simulation Total wall time: 0:00:03 diff --git a/examples/SPIN/read_restart/log.19Nov19.spin.write_restart.g++.4 b/examples/SPIN/read_restart/log.14Apr20.spin.write_restart.g++.4 similarity index 59% rename from examples/SPIN/read_restart/log.19Nov19.spin.write_restart.g++.4 rename to examples/SPIN/read_restart/log.14Apr20.spin.write_restart.g++.4 index cb9827460322a5edfb9b537bd771a6322255163b..c0c1ec130c9f0ca42f92fac4fd3a9f0c4cc61dff 100644 --- a/examples/SPIN/read_restart/log.19Nov19.spin.write_restart.g++.4 +++ b/examples/SPIN/read_restart/log.14Apr20.spin.write_restart.g++.4 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # fcc cobalt in a 3d periodic box units metal @@ -18,7 +20,7 @@ Created orthogonal box = (0 0 0) to (12.5355 21.7121 20.4704) 1 by 2 by 2 MPI processor grid create_atoms 1 box Created 500 atoms - create_atoms CPU = 0.000751972 secs + create_atoms CPU = 0.000663042 secs # setting mass, mag. moments, and interactions for cobalt @@ -51,7 +53,7 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -thermo_style custom step time v_magnorm v_emag temp etotal +thermo_style custom step time v_magnorm pe v_emag temp etotal thermo 100 compute outsp all property/atom spx spy spz sp fmx fmy fmz @@ -71,32 +73,32 @@ Neighbor list info ... stencil: full/bin/3d bin: standard Per MPI rank memory allocation (min/avg/max) = 6.868 | 6.868 | 6.868 Mbytes -Step Time v_magnorm v_emag Temp TotEng - 0 0 0.076558814 1.7982359 0 1.7982359 - 100 0.01 0.078299981 0.88259584 0 0.88259584 - 200 0.02 0.081260508 -0.43484722 0 -0.43484722 - 300 0.03 0.081195603 -1.7408209 0 -1.7408209 - 400 0.04 0.087298495 -3.4139038 0 -3.4139038 - 500 0.05 0.087663924 -4.3766089 0 -4.3766089 - 600 0.06 0.091713683 -5.8534921 0 -5.8534921 - 700 0.07 0.093779119 -6.706628 0 -6.706628 - 800 0.08 0.097960611 -7.8688568 0 -7.8688568 - 900 0.09 0.10193463 -9.5888008 0 -9.5888008 - 1000 0.1 0.10831726 -10.76492 0 -10.76492 -Loop time of 1.70473 on 4 procs for 1000 steps with 500 atoms - -Performance: 5.068 ns/day, 4.735 hours/ns, 586.602 timesteps/s -99.6% CPU use with 4 MPI tasks x no OpenMP threads +Step Time v_magnorm PotEng v_emag Temp TotEng + 0 0 0.076558814 0.89911794 0.89911794 0 0.89911794 + 100 0.01 0.078299981 0.44129792 0.44129792 0 0.44129792 + 200 0.02 0.081260508 -0.21742361 -0.21742361 0 -0.21742361 + 300 0.03 0.081195603 -0.87041046 -0.87041046 0 -0.87041046 + 400 0.04 0.087298495 -1.7069519 -1.7069519 0 -1.7069519 + 500 0.05 0.087663924 -2.1883045 -2.1883045 0 -2.1883045 + 600 0.06 0.091713683 -2.9267461 -2.9267461 0 -2.9267461 + 700 0.07 0.093779119 -3.353314 -3.353314 0 -3.353314 + 800 0.08 0.097960611 -3.9344284 -3.9344284 0 -3.9344284 + 900 0.09 0.10193463 -4.7944004 -4.7944004 0 -4.7944004 + 1000 0.1 0.10831726 -5.3824599 -5.3824599 0 -5.3824599 +Loop time of 1.7839 on 4 procs for 1000 steps with 500 atoms + +Performance: 4.843 ns/day, 4.955 hours/ns, 560.569 timesteps/s +99.5% CPU use with 4 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 0.11636 | 0.11927 | 0.12069 | 0.5 | 7.00 +Pair | 0.10068 | 0.10749 | 0.11461 | 1.5 | 6.03 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.049208 | 0.052445 | 0.057424 | 1.4 | 3.08 -Output | 0.38579 | 0.40345 | 0.4199 | 2.0 | 23.67 -Modify | 1.1138 | 1.1282 | 1.1436 | 1.1 | 66.18 -Other | | 0.001322 | | | 0.08 +Comm | 0.052378 | 0.062171 | 0.07177 | 2.8 | 3.49 +Output | 0.4054 | 0.42334 | 0.44025 | 2.0 | 23.73 +Modify | 1.174 | 1.1893 | 1.2043 | 1.1 | 66.67 +Other | | 0.001558 | | | 0.09 Nlocal: 125 ave 125 max 125 min Histogram: 4 0 0 0 0 0 0 0 0 0 @@ -113,7 +115,6 @@ Neighbor list builds = 0 Dangerous builds = 0 write_restart restart_hcp_cobalt.equil - Please see the log.cite file for references relevant to this simulation Total wall time: 0:00:01 diff --git a/examples/SPIN/read_restart/restart_hcp_cobalt.equil b/examples/SPIN/read_restart/restart_hcp_cobalt.equil new file mode 100644 index 0000000000000000000000000000000000000000..d7f92285db0bd7e3b4c36fcdf88d53180f491962 Binary files /dev/null and b/examples/SPIN/read_restart/restart_hcp_cobalt.equil differ diff --git a/examples/SPIN/run_spin_examples.sh b/examples/SPIN/run_spin_examples.sh index a71da82a04a8a5e03598363a7ca3faf27cffb844..432d70c2ba536bab6d0287ecfc6ea6bfce68e5a5 100755 --- a/examples/SPIN/run_spin_examples.sh +++ b/examples/SPIN/run_spin_examples.sh @@ -1,6 +1,6 @@ #!/bin/bash -DATE=19Nov19 +DATE=14Apr20 # bfo cd bfo/ diff --git a/examples/SPIN/run_spin_examples_serial.sh b/examples/SPIN/run_spin_examples_serial.sh new file mode 100755 index 0000000000000000000000000000000000000000..f367dfd8f494a63dc31cd19e141ef08af22d9ae1 --- /dev/null +++ b/examples/SPIN/run_spin_examples_serial.sh @@ -0,0 +1,120 @@ +#!/bin/bash + +DATE=21Fev20 + +# bfo +cd bfo/ +../../../src/lmp_serial -in in.spin.bfo +cp log.lammps log.${DATE}.spin.bfo.g++.1 +../../../src/lmp_serial -in in.spin.bfo +cp log.lammps log.${DATE}.spin.bfo.g++.4 +rm log.lammps log.cite dump*.lammpstrj +cd .. + +# fcc cobalt +cd cobalt_fcc/ +../../../src/lmp_serial -in in.spin.cobalt_fcc +cp log.lammps log.${DATE}.spin.cobalt_fcc.g++.1 +../../../src/lmp_serial -in in.spin.cobalt_fcc +cp log.lammps log.${DATE}.spin.cobalt_fcc.g++.4 +rm log.lammps log.cite dump*.lammpstrj +cd .. + +# hcp cobalt +cd cobalt_hcp/ +../../../src/lmp_serial -in in.spin.cobalt_hcp +cp log.lammps log.${DATE}.spin.cobalt_hcp.g++.1 +../../../src/lmp_serial -in in.spin.cobalt_hcp +cp log.lammps log.${DATE}.spin.cobalt_hcp.g++.4 +rm log.lammps log.cite dump*.lammpstrj +cd .. + +# dipole spin +cd dipole_spin/ +../../../src/lmp_serial -in in.spin.iron_dipole_cut +cp log.lammps log.${DATE}.spin.iron_dipole_cut.g++.1 +../../../src/lmp_serial -in in.spin.iron_dipole_cut +cp log.lammps log.${DATE}.spin.iron_dipole_cut.g++.4 +../../../src/lmp_serial -in in.spin.iron_dipole_ewald +cp log.lammps log.${DATE}.spin.iron_dipole_ewald.g++.1 +../../../src/lmp_serial -in in.spin.iron_dipole_ewald +cp log.lammps log.${DATE}.spin.iron_dipole_ewald.g++.4 +../../../src/lmp_serial -in in.spin.iron_dipole_pppm +cp log.lammps log.${DATE}.spin.iron_dipole_pppm.g++.1 +../../../src/lmp_serial -in in.spin.iron_dipole_pppm +cp log.lammps log.${DATE}.spin.iron_dipole_pppm.g++.4 +rm log.lammps log.cite dump*.lammpstrj +cd .. + +# bcc iron +cd iron/ +../../../src/lmp_serial -in in.spin.iron +cp log.lammps log.${DATE}.spin.iron.g++.1 +../../../src/lmp_serial -in in.spin.iron +cp log.lammps log.${DATE}.spin.iron.g++.4 +../../../src/lmp_serial -in in.spin.iron_cubic +cp log.lammps log.${DATE}.spin.iron_cubic.g++.1 +../../../src/lmp_serial -in in.spin.iron_cubic +cp log.lammps log.${DATE}.spin.iron_cubic.g++.4 +rm log.lammps log.cite dump*.lammpstrj +cd .. + +# fcc nickel +cd nickel/ +../../../src/lmp_serial -in in.spin.nickel +cp log.lammps log.${DATE}.spin.nickel.g++.1 +../../../src/lmp_serial -in in.spin.nickel +cp log.lammps log.${DATE}.spin.nickel.g++.4 +../../../src/lmp_serial -in in.spin.nickel_cubic +cp log.lammps log.${DATE}.spin.nickel_cubic.g++.1 +../../../src/lmp_serial -in in.spin.nickel_cubic +cp log.lammps log.${DATE}.spin.nickel_cubic.g++.4 +rm log.lammps log.cite dump*.lammpstrj +cd .. + +# read restart +cd read_restart/ +../../../src/lmp_serial -in in.spin.write_restart +cp log.lammps log.${DATE}.spin.write_restart.g++.1 +../../../src/lmp_serial -in in.spin.write_restart +cp log.lammps log.${DATE}.spin.write_restart.g++.4 +../../../src/lmp_serial -in in.spin.restart +cp log.lammps log.${DATE}.spin.restart.g++.1 +../../../src/lmp_serial -in in.spin.restart +cp log.lammps log.${DATE}.spin.restart.g++.4 +../../../src/lmp_serial -in in.spin.read_data +cp log.lammps log.${DATE}.spin.read_data.g++.1 +../../../src/lmp_serial -in in.spin.read_data +cp log.lammps log.${DATE}.spin.read_data.g++.4 +rm log.lammps log.cite dump*.lammpstrj +cd .. + +# setforce +cd setforce_spin/ +../../../src/lmp_serial -in in.spin.setforce +cp log.lammps log.${DATE}.spin.setforce.g++.1 +../../../src/lmp_serial -in in.spin.setforce +cp log.lammps log.${DATE}.spin.setforce.g++.4 +rm log.lammps log.cite dump*.lammpstrj +cd .. + +# spin minimizers +cd spinmin/ +../../../src/lmp_serial -in in.spin.bfo_min +cp log.lammps log.${DATE}.spin.bfo_min.g++.1 +../../../src/lmp_serial -in in.spin.bfo_min +cp log.lammps log.${DATE}.spin.bfo_min.g++.4 +../../../src/lmp_serial -in in.spin.bfo_min_cg +cp log.lammps log.${DATE}.spin.bfo_min_cg.g++.1 +../../../src/lmp_serial -in in.spin.bfo_min_cg +cp log.lammps log.${DATE}.spin.bfo_min_cg.g++.4 +../../../src/lmp_serial -in in.spin.bfo_min_lbfgs +cp log.lammps log.${DATE}.spin.bfo_min_lbfgs.g++.1 +../../../src/lmp_serial -in in.spin.bfo_min_lbfgs +cp log.lammps log.${DATE}.spin.bfo_min_lbfgs.g++.4 +../../../src/lmp_serial -in in.spin.iron_min +cp log.lammps log.${DATE}.spin.iron_min.g++.1 +../../../src/lmp_serial -in in.spin.iron_min +cp log.lammps log.${DATE}.spin.iron_min.g++.4 +rm log.lammps log.cite dump*.lammpstrj +cd .. diff --git a/examples/SPIN/setforce_spin/in.spin.setforce b/examples/SPIN/setforce_spin/in.spin.setforce index 0d65955a296b592f7403c3f2537cc9abcd5ef5b6..4edf70eb52d0f963a813be6bec297632923ca4f1 100644 --- a/examples/SPIN/setforce_spin/in.spin.setforce +++ b/examples/SPIN/setforce_spin/in.spin.setforce @@ -48,7 +48,7 @@ variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] thermo 100 -thermo_style custom step time v_magx v_magz v_magnorm v_tmag etotal +thermo_style custom step time v_magx v_magz v_magnorm v_tmag pe v_emag etotal thermo_modify format float %20.15g compute outsp all property/atom spx spy spz sp fmx fmy fmz diff --git a/examples/SPIN/setforce_spin/log.19Nov19.spin.setforce.g++.1 b/examples/SPIN/setforce_spin/log.14Apr20.spin.setforce.g++.1 similarity index 79% rename from examples/SPIN/setforce_spin/log.19Nov19.spin.setforce.g++.1 rename to examples/SPIN/setforce_spin/log.14Apr20.spin.setforce.g++.1 index ed5c037feb9f11dbb919924743cb96ee29ebfa10..ef5e86b0b3746acca22626e592bbfd88ad6cc830 100644 --- a/examples/SPIN/setforce_spin/log.19Nov19.spin.setforce.g++.1 +++ b/examples/SPIN/setforce_spin/log.14Apr20.spin.setforce.g++.1 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task units metal dimension 3 @@ -18,10 +20,10 @@ region reg1 block 0.0 10.0 0.0 5.0 0.0 1.0 region reg2 block 0.0 10.0 6.0 10.0 0.0 1.0 create_atoms 1 region reg1 Created 120 atoms - create_atoms CPU = 0.000998974 secs + create_atoms CPU = 0.000992775 secs create_atoms 2 region reg2 Created 80 atoms - create_atoms CPU = 4.1008e-05 secs + create_atoms CPU = 0.000215054 secs # setting mass, mag. moments, and interactions for bcc iron @@ -59,7 +61,7 @@ variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] thermo 100 -thermo_style custom step time v_magx v_magz v_magnorm v_tmag etotal +thermo_style custom step time v_magx v_magz v_magnorm v_tmag pe v_emag etotal thermo_modify format float %20.15g compute outsp all property/atom spx spy spz sp fmx fmy fmz @@ -68,7 +70,7 @@ dump 1 all custom 100 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[ min_style spin min_modify alpha_damp 1.0 discrete_factor 20.0 minimize 1.0e-16 1.0e-16 1000 100 -WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:177) +WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:190) Neighbor list info ... update every 1 steps, delay 0 steps, check yes max neighbors/atom: 2000, page size: 100000 @@ -87,21 +89,21 @@ Neighbor list info ... stencil: none bin: none Per MPI rank memory allocation (min/avg/max) = 7.215 | 7.215 | 7.215 Mbytes -Step Time v_magx v_magz v_magnorm v_tmag TotEng - 0 0 0.000143282585570239 0.615515043943419 0.615726279597251 24.9364200982478 0.121881906963737 - 100 0.01 0.000616167502203097 0.594467364025194 0.594498630048783 0.00188964439583802 0.0371335982020527 - 200 0.02 0.000498981016106215 0.595175581059792 0.595218717456538 0.000158614984300385 0.036877233648055 - 300 0.03 0.000211899815837572 0.595357874794342 0.595402442288391 1.44454891242177e-05 0.0368548794182375 - 400 0.04 7.98967577397158e-05 0.595395828381057 0.595440657806237 1.50721782707597e-06 0.0368527556548781 - 500 0.05 2.9121648914103e-05 0.595403174462525 0.595448064489507 1.74330474543395e-07 0.0368525254239539 - 600 0.06 1.04772320898497e-05 0.595404457003426 0.595449362424563 2.12204214498221e-08 0.0368524982492743 - 700 0.07 3.74634771616422e-06 0.595404627382825 0.595449536940641 2.63852407890463e-09 0.036852494912626 - 800 0.08 1.33525617457914e-06 0.595404626884198 0.595449537611055 3.30772506699851e-10 0.0368524944963445 - 900 0.09 4.75054785504803e-07 0.595404613763238 0.595449524836571 4.15940445257144e-11 0.0368524944440918 - 1000 0.1 1.68843135202601e-07 0.59540460640039 0.595449517580793 5.23632581178917e-12 0.036852494437518 -Loop time of 0.0966749 on 1 procs for 1000 steps with 200 atoms - -100.0% CPU use with 1 MPI tasks x no OpenMP threads +Step Time v_magx v_magz v_magnorm v_tmag PotEng v_emag TotEng + 0 0 0.000143282585570239 0.615515043943419 0.615726279597251 24.9364200982478 0.121881906963737 0.121881906963737 0.121881906963737 + 100 0.01 0.000616167502203097 0.594467364025194 0.594498630048783 0.00188964439583802 0.0371335982020527 0.0371335982020522 0.0371335982020527 + 200 0.02 0.000498981016106215 0.595175581059792 0.595218717456538 0.000158614984300385 0.036877233648055 0.0368772336480544 0.036877233648055 + 300 0.03 0.000211899815837572 0.595357874794342 0.595402442288391 1.44454891242177e-05 0.0368548794182375 0.0368548794182369 0.0368548794182375 + 400 0.04 7.98967577397158e-05 0.595395828381057 0.595440657806237 1.50721782707597e-06 0.0368527556548781 0.0368527556548775 0.0368527556548781 + 500 0.05 2.9121648914103e-05 0.595403174462525 0.595448064489507 1.74330474543395e-07 0.0368525254239539 0.0368525254239533 0.0368525254239539 + 600 0.06 1.04772320898497e-05 0.595404457003426 0.595449362424563 2.12204214498221e-08 0.0368524982492743 0.0368524982492735 0.0368524982492743 + 700 0.07 3.74634771616422e-06 0.595404627382825 0.595449536940641 2.63852407890463e-09 0.036852494912626 0.0368524949126256 0.036852494912626 + 800 0.08 1.33525617457914e-06 0.595404626884198 0.595449537611055 3.30772506699851e-10 0.0368524944963445 0.0368524944963439 0.0368524944963445 + 900 0.09 4.75054785504803e-07 0.595404613763238 0.595449524836571 4.15940445257144e-11 0.0368524944440918 0.0368524944440912 0.0368524944440918 + 1000 0.1 1.68843135202601e-07 0.59540460640039 0.595449517580793 5.23632581178917e-12 0.036852494437518 0.0368524944375173 0.036852494437518 +Loop time of 0.093256 on 1 procs for 1000 steps with 200 atoms + +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads Minimization stats: Stopping criterion = max iterations @@ -115,12 +117,12 @@ Minimization stats: MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 0.08679 | 0.08679 | 0.08679 | 0.0 | 89.77 +Pair | 0.08325 | 0.08325 | 0.08325 | 0.0 | 89.27 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 7.2956e-05 | 7.2956e-05 | 7.2956e-05 | 0.0 | 0.08 -Output | 0.0033231 | 0.0033231 | 0.0033231 | 0.0 | 3.44 -Modify | 0.0022919 | 0.0022919 | 0.0022919 | 0.0 | 2.37 -Other | | 0.004197 | | | 4.34 +Comm | 6.032e-05 | 6.032e-05 | 6.032e-05 | 0.0 | 0.06 +Output | 0.0033138 | 0.0033138 | 0.0033138 | 0.0 | 3.55 +Modify | 0.0023961 | 0.0023961 | 0.0023961 | 0.0 | 2.57 +Other | | 0.004236 | | | 4.54 Nlocal: 200 ave 200 max 200 min Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/setforce_spin/log.19Nov19.spin.setforce.g++.4 b/examples/SPIN/setforce_spin/log.14Apr20.spin.setforce.g++.4 similarity index 57% rename from examples/SPIN/setforce_spin/log.19Nov19.spin.setforce.g++.4 rename to examples/SPIN/setforce_spin/log.14Apr20.spin.setforce.g++.4 index 4ce1497957d374a4f43ca83d63b124997b0e2887..c99bad0b402d7e4f52a54248b0dc8089f7b16dae 100644 --- a/examples/SPIN/setforce_spin/log.19Nov19.spin.setforce.g++.4 +++ b/examples/SPIN/setforce_spin/log.14Apr20.spin.setforce.g++.4 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task units metal dimension 3 @@ -18,10 +20,10 @@ region reg1 block 0.0 10.0 0.0 5.0 0.0 1.0 region reg2 block 0.0 10.0 6.0 10.0 0.0 1.0 create_atoms 1 region reg1 Created 120 atoms - create_atoms CPU = 0.000770092 secs + create_atoms CPU = 0.000747204 secs create_atoms 2 region reg2 Created 80 atoms - create_atoms CPU = 7.9155e-05 secs + create_atoms CPU = 0.000154018 secs # setting mass, mag. moments, and interactions for bcc iron @@ -59,7 +61,7 @@ variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] thermo 100 -thermo_style custom step time v_magx v_magz v_magnorm v_tmag etotal +thermo_style custom step time v_magx v_magz v_magnorm v_tmag pe v_emag etotal thermo_modify format float %20.15g compute outsp all property/atom spx spy spz sp fmx fmy fmz @@ -68,7 +70,7 @@ dump 1 all custom 100 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[ min_style spin min_modify alpha_damp 1.0 discrete_factor 20.0 minimize 1.0e-16 1.0e-16 1000 100 -WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:177) +WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:190) Neighbor list info ... update every 1 steps, delay 0 steps, check yes max neighbors/atom: 2000, page size: 100000 @@ -87,26 +89,26 @@ Neighbor list info ... stencil: none bin: none Per MPI rank memory allocation (min/avg/max) = 7.2 | 7.2 | 7.2 Mbytes -Step Time v_magx v_magz v_magnorm v_tmag TotEng - 0 0 0.000143282585570239 0.615515043943419 0.615726279597251 21.735436446264 0.251043691626527 - 100 0.01 -0.00116085697754605 0.590264559350799 0.590315072966953 0.00283413081085227 0.0846048989956832 - 200 0.02 -0.000147570523236238 0.5896197627388 0.589686497206689 0.000451051163122381 0.0839054390713705 - 300 0.03 2.64982966536902e-05 0.59002969475615 0.590102003120244 5.22539631503911e-05 0.0838351677819014 - 400 0.04 1.77805448780033e-05 0.590195117338991 0.590268726215095 4.46490059775722e-06 0.0838382933245032 - 500 0.05 6.71566571038784e-06 0.590243842081075 0.590317756995865 3.63227563542099e-07 0.0838411433937997 - 600 0.06 2.2410340743112e-06 0.590257551861528 0.590331542128336 2.99360370345601e-08 0.0838420708305252 - 700 0.07 7.12179152897591e-07 0.5902614042958 0.590335413637884 2.51559188415894e-09 0.0838423375091767 - 800 0.08 2.20574733079126e-07 0.590262494529884 0.590336508799302 2.14455748236281e-10 0.0838424126463497 - 900 0.09 6.72564339365689e-08 0.590262805532644 0.590336821097688 1.84495767133404e-11 0.0838424338620728 - 1000 0.1 2.03001940390912e-08 0.590262894882646 0.590336910794094 1.5958531383517e-12 0.0838424398944951 -Loop time of 0.0617704 on 4 procs for 1000 steps with 200 atoms - -98.7% CPU use with 4 MPI tasks x no OpenMP threads +Step Time v_magx v_magz v_magnorm v_tmag PotEng v_emag TotEng + 0 0 0.000143282585570239 0.615515043943419 0.615726279597251 24.9364200982478 0.121881906963737 0.121881906963737 0.121881906963737 + 100 0.01 0.000616167502203098 0.594467364025194 0.594498630048784 0.00188964439583802 0.0371335982020525 0.037133598202052 0.0371335982020525 + 200 0.02 0.000498981016106215 0.595175581059791 0.595218717456538 0.000158614984300385 0.0368772336480548 0.0368772336480543 0.0368772336480548 + 300 0.03 0.000211899815837572 0.595357874794342 0.595402442288391 1.44454891242177e-05 0.0368548794182373 0.0368548794182367 0.0368548794182373 + 400 0.04 7.9896757739716e-05 0.595395828381057 0.595440657806237 1.50721782707597e-06 0.0368527556548779 0.0368527556548773 0.0368527556548779 + 500 0.05 2.91216489141033e-05 0.595403174462525 0.595448064489507 1.74330474543395e-07 0.0368525254239537 0.0368525254239531 0.0368525254239537 + 600 0.06 1.04772320898493e-05 0.595404457003426 0.595449362424563 2.12204214498221e-08 0.036852498249274 0.0368524982492735 0.036852498249274 + 700 0.07 3.74634771616408e-06 0.595404627382825 0.59544953694064 2.63852407890464e-09 0.0368524949126259 0.0368524949126254 0.0368524949126259 + 800 0.08 1.33525617457997e-06 0.595404626884198 0.595449537611055 3.30772506699851e-10 0.0368524944963442 0.0368524944963437 0.0368524944963442 + 900 0.09 4.75054785504803e-07 0.595404613763238 0.595449524836571 4.15940445257144e-11 0.0368524944440915 0.036852494444091 0.0368524944440915 + 1000 0.1 1.68843135202462e-07 0.59540460640039 0.595449517580793 5.23632581178917e-12 0.0368524944375178 0.0368524944375172 0.0368524944375178 +Loop time of 0.041718 on 4 procs for 1000 steps with 200 atoms + +97.4% CPU use with 4 MPI tasks x 1 OpenMP threads Minimization stats: Stopping criterion = max iterations Energy initial, next-to-last, final = - 0.251043691627 0.0838424398641 0.0838424398945 + 0.121881906964 0.0368524944375 0.0368524944375 Force two-norm initial, final = 0 0 Force max component initial, final = 0 0 Final line search alpha, max atom move = 0 0 @@ -115,12 +117,12 @@ Minimization stats: MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 0.023753 | 0.029762 | 0.035936 | 3.3 | 48.18 +Pair | 0.020462 | 0.021665 | 0.024967 | 1.3 | 51.93 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.011783 | 0.019131 | 0.025404 | 4.3 | 30.97 -Output | 0.0019517 | 0.0019774 | 0.0020368 | 0.1 | 3.20 -Modify | 0.0006361 | 0.00087249 | 0.0011525 | 0.0 | 1.41 -Other | | 0.01003 | | | 16.23 +Comm | 0.0074174 | 0.010855 | 0.012527 | 2.0 | 26.02 +Output | 0.0012774 | 0.0012916 | 0.0013213 | 0.0 | 3.10 +Modify | 0.00056291 | 0.00068498 | 0.00096655 | 0.0 | 1.64 +Other | | 0.007222 | | | 17.31 Nlocal: 50 ave 50 max 50 min Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/spinmin/log.19Nov19.spin.bfo_min.g++.1 b/examples/SPIN/spinmin/log.14Apr20.spin.bfo_min.g++.1 similarity index 54% rename from examples/SPIN/spinmin/log.19Nov19.spin.bfo_min.g++.1 rename to examples/SPIN/spinmin/log.14Apr20.spin.bfo_min.g++.1 index 61b6ad87006857e2aff6cb03a0844c34c83a5e29..49a5f4dd3fd58993ec8776c57e4395385dfb0ef2 100644 --- a/examples/SPIN/spinmin/log.19Nov19.spin.bfo_min.g++.1 +++ b/examples/SPIN/spinmin/log.14Apr20.spin.bfo_min.g++.1 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bfo in a 3d periodic box units metal @@ -17,7 +19,7 @@ Created orthogonal box = (0 0 0) to (134.64 134.64 19.8) 1 by 1 by 1 MPI processor grid create_atoms 1 box Created 5780 atoms - create_atoms CPU = 0.00107217 secs + create_atoms CPU = 0.00207901 secs # setting mass, mag. moments, and interactions for bcc iron @@ -60,10 +62,10 @@ dump 1 all custom 50 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3 min_style spin min_modify alpha_damp 1.0 discrete_factor 10.0 minimize 1.0e-10 0.0 1000 100 -WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:177) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) +WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:190) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) Neighbor list info ... update every 1 steps, delay 0 steps, check yes max neighbors/atom: 2000, page size: 100000 @@ -88,30 +90,30 @@ Neighbor list info ... bin: none Per MPI rank memory allocation (min/avg/max) = 8.331 | 8.331 | 8.331 Mbytes Step Time v_magnorm v_emag v_tmag Temp TotEng - 0 0 0.0100717228668283 -0.163834417271778 14831.3069413956 0 -0.0819172086358848 - 50 0.005 0.000106105812337003 -128.307447484203 0.104264818055985 0 -64.1537237421011 - 100 0.01 7.95347901119144e-06 -131.449389798071 0.0221943604064967 0 -65.7246948990356 - 150 0.015 5.63006161138875e-07 -132.296453030419 0.0085472877724348 0 -66.1482265152089 - 200 0.02 5.07390677383517e-07 -132.622857703805 0.00361380451198708 0 -66.3114288519012 - 250 0.025 3.28458336892231e-07 -132.774411992703 0.00187753161968493 0 -66.3872059963511 - 300 0.03 1.93294839202864e-07 -132.861283726084 0.00121374398924599 0 -66.4306418630428 - 350 0.035 1.13872157437693e-07 -132.923137019136 0.000954736871701507 0 -66.4615685095675 - 400 0.04 6.42075545620808e-08 -132.975239148591 0.000854064736183609 0 -66.4876195742954 - 450 0.045 3.44210513403008e-08 -133.023523287306 0.000812909459005007 0 -66.5117616436536 - 500 0.05 1.80394981485933e-08 -133.070071976252 0.000789742875305133 0 -66.5350359881254 - 550 0.055 9.54697157105863e-09 -133.11541233939 0.000769860218895372 0 -66.5577061696963 - 600 0.06 5.22455110720346e-09 -133.159676447906 0.000752941158466282 0 -66.5798382239526 - 650 0.065 2.95172977724016e-09 -133.203196195612 0.000745065216626277 0 -66.6015980978057 - 700 0.07 1.6727567441294e-09 -133.246696814329 0.000752898926000619 0 -66.6233484071653 - 750 0.075 9.17127001723567e-10 -133.291227007554 0.000780491405791262 0 -66.6456135037769 - 800 0.08 4.72669535949609e-10 -133.337962593396 0.000827942834401386 0 -66.6689812966976 - 850 0.085 2.25696738407094e-10 -133.387945245851 0.000890246383931885 0 -66.6939726229243 - 900 0.09 1.0030717061716e-10 -133.441737087546 0.000955403731484674 0 -66.720868543773 - 950 0.095 4.19867626359036e-11 -133.498969798312 0.00100352240545389 0 -66.7494848991554 - 1000 0.1 1.64283478182092e-11 -133.557979904763 0.00101162410316333 0 -66.7789899523816 -Loop time of 9.23057 on 1 procs for 1000 steps with 5780 atoms - -99.9% CPU use with 1 MPI tasks x no OpenMP threads + 0 0 0.0100717228668283 -0.0819172086358897 14831.3069413956 0 -0.0819172086358848 + 50 0.005 0.000106105812337003 -64.1537237421016 0.104264818055985 0 -64.1537237421011 + 100 0.01 7.95347901119144e-06 -65.7246948990358 0.0221943604064967 0 -65.7246948990356 + 150 0.015 5.63006161138875e-07 -66.1482265152101 0.0085472877724348 0 -66.1482265152089 + 200 0.02 5.07390677383517e-07 -66.3114288519027 0.00361380451198708 0 -66.3114288519012 + 250 0.025 3.28458336892231e-07 -66.3872059963515 0.00187753161968493 0 -66.3872059963511 + 300 0.03 1.93294839202864e-07 -66.430641863042 0.00121374398924599 0 -66.4306418630428 + 350 0.035 1.13872157437693e-07 -66.461568509568 0.000954736871701507 0 -66.4615685095675 + 400 0.04 6.42075545620808e-08 -66.4876195742958 0.000854064736183609 0 -66.4876195742954 + 450 0.045 3.44210513403008e-08 -66.5117616436528 0.000812909459005007 0 -66.5117616436536 + 500 0.05 1.80394981485933e-08 -66.535035988126 0.000789742875305133 0 -66.5350359881254 + 550 0.055 9.54697157105863e-09 -66.5577061696949 0.000769860218895372 0 -66.5577061696963 + 600 0.06 5.22455110720346e-09 -66.5798382239523 0.000752941158466282 0 -66.5798382239526 + 650 0.065 2.95172977724016e-09 -66.6015980978057 0.000745065216626277 0 -66.6015980978057 + 700 0.07 1.6727567441294e-09 -66.6233484071647 0.000752898926000619 0 -66.6233484071653 + 750 0.075 9.17127001723567e-10 -66.6456135037774 0.000780491405791262 0 -66.6456135037769 + 800 0.08 4.72669535949609e-10 -66.6689812966982 0.000827942834401386 0 -66.6689812966976 + 850 0.085 2.25696738407094e-10 -66.6939726229253 0.000890246383931885 0 -66.6939726229243 + 900 0.09 1.0030717061716e-10 -66.7208685437728 0.000955403731484674 0 -66.720868543773 + 950 0.095 4.19867626359036e-11 -66.7494848991559 0.00100352240545389 0 -66.7494848991554 + 1000 0.1 1.64283478182092e-11 -66.778989952382 0.00101162410316333 0 -66.7789899523816 +Loop time of 8.81793 on 1 procs for 1000 steps with 5780 atoms + +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads Minimization stats: Stopping criterion = max iterations @@ -125,12 +127,12 @@ Minimization stats: MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 8.7576 | 8.7576 | 8.7576 | 0.0 | 94.88 +Pair | 8.3388 | 8.3388 | 8.3388 | 0.0 | 94.57 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.023815 | 0.023815 | 0.023815 | 0.0 | 0.26 -Output | 0.31665 | 0.31665 | 0.31665 | 0.0 | 3.43 -Modify | 0.029446 | 0.029446 | 0.029446 | 0.0 | 0.32 -Other | | 0.1031 | | | 1.12 +Comm | 0.023951 | 0.023951 | 0.023951 | 0.0 | 0.27 +Output | 0.3172 | 0.3172 | 0.3172 | 0.0 | 3.60 +Modify | 0.034405 | 0.034405 | 0.034405 | 0.0 | 0.39 +Other | | 0.1036 | | | 1.17 Nlocal: 5780 ave 5780 max 5780 min Histogram: 1 0 0 0 0 0 0 0 0 0 @@ -145,4 +147,4 @@ Total # of neighbors = 92480 Ave neighs/atom = 16 Neighbor list builds = 0 Dangerous builds = 0 -Total wall time: 0:00:09 +Total wall time: 0:00:08 diff --git a/examples/SPIN/spinmin/log.19Nov19.spin.bfo_min.g++.4 b/examples/SPIN/spinmin/log.14Apr20.spin.bfo_min.g++.4 similarity index 51% rename from examples/SPIN/spinmin/log.19Nov19.spin.bfo_min.g++.4 rename to examples/SPIN/spinmin/log.14Apr20.spin.bfo_min.g++.4 index efc4628b9e578675e85718aa03e47393f13ce723..6ffa5eb86139ecf243dd33ec4eae72469f6c997d 100644 --- a/examples/SPIN/spinmin/log.19Nov19.spin.bfo_min.g++.4 +++ b/examples/SPIN/spinmin/log.14Apr20.spin.bfo_min.g++.4 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bfo in a 3d periodic box units metal @@ -17,7 +19,7 @@ Created orthogonal box = (0 0 0) to (134.64 134.64 19.8) 2 by 2 by 1 MPI processor grid create_atoms 1 box Created 5780 atoms - create_atoms CPU = 0.00102711 secs + create_atoms CPU = 0.00117207 secs # setting mass, mag. moments, and interactions for bcc iron @@ -60,10 +62,10 @@ dump 1 all custom 50 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3 min_style spin min_modify alpha_damp 1.0 discrete_factor 10.0 minimize 1.0e-10 0.0 1000 100 -WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:177) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) +WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:190) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) Neighbor list info ... update every 1 steps, delay 0 steps, check yes max neighbors/atom: 2000, page size: 100000 @@ -88,35 +90,35 @@ Neighbor list info ... bin: none Per MPI rank memory allocation (min/avg/max) = 7.788 | 7.788 | 7.788 Mbytes Step Time v_magnorm v_emag v_tmag Temp TotEng - 0 0 0.0100717228668283 -0.162177519662199 14970.709092345 0 -0.157514482753586 - 50 0.005 0.000134061770654521 -128.226118665465 0.102634444037434 0 -128.286182424672 - 100 0.01 7.67769618983817e-06 -131.374599259781 0.0222596977749883 0 -131.428418504308 - 150 0.015 6.02904602224806e-07 -132.224372015825 0.00974271828169071 0 -132.273190134602 - 200 0.02 6.50197247050491e-07 -132.573383315469 0.00374227079785921 0 -132.617565541034 - 250 0.025 4.4053438575152e-07 -132.729743470508 0.0019334097282578 0 -132.770567114743 - 300 0.03 2.78356316513274e-07 -132.819255077939 0.00124938353773497 0 -132.857574876413 - 350 0.035 1.79684785125388e-07 -132.882714312877 0.000973166792896165 0 -132.919261229742 - 400 0.04 1.10949878459078e-07 -132.935357748213 0.000852955460997588 0 -132.970786605995 - 450 0.045 6.49064465617817e-08 -132.982991683198 0.000790741148426224 0 -133.017887798927 - 500 0.05 3.70514666559952e-08 -133.027689959766 0.00074794913288275 0 -133.06256199189 - 550 0.055 2.12433814830885e-08 -133.070148920145 0.00071263732127117 0 -133.105417593745 - 600 0.06 1.24676590171361e-08 -133.110772798503 0.000685051841817325 0 -133.14676746928 - 650 0.065 7.53611859129351e-09 -133.150126417754 0.000669443562813208 0 -133.187094895708 - 700 0.07 4.63539338651321e-09 -133.189024073453 0.000669619853917953 0 -133.227152349439 - 750 0.075 2.82145833974835e-09 -133.22844627026 0.000687338035086961 0 -133.267881315198 - 800 0.08 1.64378151566173e-09 -133.269413776733 0.000722197692175127 0 -133.310284062463 - 850 0.085 8.883310104497e-10 -133.312863108454 0.000771645398804486 0 -133.355293578462 - 900 0.09 4.33874801863461e-10 -133.359507749172 0.000830255722998156 0 -133.403626236688 - 950 0.095 1.8812784924272e-10 -133.409630495316 0.000888348219681112 0 -133.455560507802 - 1000 0.1 7.17748875671948e-11 -133.462806227865 0.000931427722404681 0 -133.510640942679 -Loop time of 2.53419 on 4 procs for 1000 steps with 5780 atoms - -99.9% CPU use with 4 MPI tasks x no OpenMP threads + 0 0 0.0100717228668283 -0.0819172086358889 14831.3069413956 0 -0.0819172086358937 + 50 0.005 0.000106105812337003 -64.1537237421014 0.104264818055985 0 -64.1537237421015 + 100 0.01 7.95347901119157e-06 -65.7246948990355 0.0221943604064966 0 -65.7246948990356 + 150 0.015 5.63006161138918e-07 -66.1482265152097 0.0085472877724348 0 -66.1482265152095 + 200 0.02 5.07390677383159e-07 -66.3114288519023 0.00361380451198709 0 -66.3114288519024 + 250 0.025 3.28458336892466e-07 -66.3872059963513 0.00187753161968492 0 -66.3872059963514 + 300 0.03 1.93294839202718e-07 -66.4306418630421 0.00121374398924599 0 -66.4306418630421 + 350 0.035 1.1387215743759e-07 -66.461568509568 0.00095473687170151 0 -66.4615685095675 + 400 0.04 6.42075545625297e-08 -66.4876195742954 0.000854064736183607 0 -66.4876195742956 + 450 0.045 3.44210513402635e-08 -66.5117616436528 0.000812909459005008 0 -66.5117616436531 + 500 0.05 1.80394981487191e-08 -66.5350359881262 0.000789742875305131 0 -66.5350359881262 + 550 0.055 9.54697157097866e-09 -66.5577061696951 0.000769860218895372 0 -66.5577061696951 + 600 0.06 5.22455110708009e-09 -66.5798382239529 0.000752941158466282 0 -66.5798382239528 + 650 0.065 2.95172977729162e-09 -66.601598097806 0.000745065216626281 0 -66.6015980978062 + 700 0.07 1.67275674393186e-09 -66.6233484071645 0.000752898926000615 0 -66.6233484071643 + 750 0.075 9.17127001716818e-10 -66.6456135037772 0.000780491405791264 0 -66.6456135037769 + 800 0.08 4.72669535990986e-10 -66.6689812966981 0.000827942834401387 0 -66.668981296698 + 850 0.085 2.25696738737671e-10 -66.6939726229253 0.000890246383931887 0 -66.6939726229253 + 900 0.09 1.00307170296094e-10 -66.7208685437728 0.000955403731484673 0 -66.7208685437728 + 950 0.095 4.19867626693269e-11 -66.7494848991562 0.00100352240545389 0 -66.7494848991562 + 1000 0.1 1.64283481770759e-11 -66.7789899523816 0.00101162410316333 0 -66.7789899523818 +Loop time of 2.42884 on 4 procs for 1000 steps with 5780 atoms + +99.9% CPU use with 4 MPI tasks x 1 OpenMP threads Minimization stats: Stopping criterion = max iterations Energy initial, next-to-last, final = - -0.157514482754 -133.509516066 -133.510640943 + -0.0819172086359 -66.778399627 -66.7789899524 Force two-norm initial, final = 0 0 Force max component initial, final = 0 0 Final line search alpha, max atom move = 0 0 @@ -125,12 +127,12 @@ Minimization stats: MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 2.2697 | 2.2995 | 2.3564 | 2.2 | 90.74 +Pair | 2.1221 | 2.164 | 2.2349 | 2.9 | 89.10 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.05711 | 0.11414 | 0.14368 | 10.1 | 4.50 -Output | 0.084883 | 0.084915 | 0.084985 | 0.0 | 3.35 -Modify | 0.0071826 | 0.0072024 | 0.0072234 | 0.0 | 0.28 -Other | | 0.02847 | | | 1.12 +Comm | 0.065814 | 0.13626 | 0.17868 | 11.7 | 5.61 +Output | 0.090046 | 0.090083 | 0.090171 | 0.0 | 3.71 +Modify | 0.0081615 | 0.0083774 | 0.008673 | 0.2 | 0.34 +Other | | 0.03009 | | | 1.24 Nlocal: 1445 ave 1445 max 1445 min Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/spinmin/log.19Nov19.spin.bfo_min_cg.g++.1 b/examples/SPIN/spinmin/log.14Apr20.spin.bfo_min_cg.g++.1 similarity index 65% rename from examples/SPIN/spinmin/log.19Nov19.spin.bfo_min_cg.g++.1 rename to examples/SPIN/spinmin/log.14Apr20.spin.bfo_min_cg.g++.1 index ffd244b146e487cff3f6d9e287fc6490e2e821cb..cef54797eb842f6506559b1908ded82f5925cfdd 100644 --- a/examples/SPIN/spinmin/log.19Nov19.spin.bfo_min_cg.g++.1 +++ b/examples/SPIN/spinmin/log.14Apr20.spin.bfo_min_cg.g++.1 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bfo in a 3d periodic box units metal @@ -17,7 +19,7 @@ Created orthogonal box = (0 0 0) to (134.64 134.64 19.8) 1 by 1 by 1 MPI processor grid create_atoms 1 box Created 5780 atoms - create_atoms CPU = 0.00135589 secs + create_atoms CPU = 0.00267482 secs # setting mass, mag. moments, and interactions for bcc iron @@ -59,11 +61,11 @@ dump 1 all custom 50 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3 min_style spin/cg # min_modify line spin_none discrete_factor 10.0 minimize 1.0e-10 1.0e-10 1000 100 -WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:177) +WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:190) WARNING: Line search incompatible gneb (../min_spin_cg.cpp:105) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) Neighbor list info ... update every 1 steps, delay 0 steps, check yes max neighbors/atom: 2000, page size: 100000 @@ -88,20 +90,20 @@ Neighbor list info ... bin: none Per MPI rank memory allocation (min/avg/max) = 8.331 | 8.331 | 8.331 Mbytes Step Time v_magnorm v_emag v_tmag Temp TotEng - 0 0 0.0100717228668283 -0.163834417271778 14831.3069413956 0 -0.0819172086358848 - 100 0.01 8.80197005314557e-06 -132.80634639767 0.00226660536216922 0 -66.4031731988347 - 200 0.02 6.70903250218956e-06 -133.127078243354 0.00103783628361662 0 -66.563539121677 - 300 0.03 4.5381603452565e-06 -133.471076972345 0.00144833375067451 0 -66.7355384861738 - 400 0.04 9.04820921016732e-07 -133.767843456662 0.000682239601485924 0 -66.8839217283314 - 500 0.05 1.6866160174916e-06 -133.893922160731 0.00032462625992713 0 -66.946961080365 - 600 0.06 1.78038217785001e-06 -133.957222680701 0.000160730704849448 0 -66.9786113403509 - 700 0.07 1.49199057723078e-06 -133.987255887786 7.39864656758093e-05 0 -66.9936279438931 - 800 0.08 1.15173756711067e-06 -134.000921126053 3.33959465206462e-05 0 -67.0004605630278 - 900 0.09 8.48526364752965e-07 -134.007049858868 1.49345737358251e-05 0 -67.0035249294347 - 1000 0.1 6.10346492876059e-07 -134.009791515671 6.71648807105468e-06 0 -67.0048957578347 -Loop time of 9.4449 on 1 procs for 1000 steps with 5780 atoms - -100.0% CPU use with 1 MPI tasks x no OpenMP threads + 0 0 0.0100717228668283 -0.0819172086358897 14831.3069413956 0 -0.0819172086358848 + 100 0.01 8.80197005314557e-06 -66.4031731988352 0.00226660536216922 0 -66.4031731988347 + 200 0.02 6.70903250218956e-06 -66.5635391216766 0.00103783628361662 0 -66.563539121677 + 300 0.03 4.5381603452565e-06 -66.7355384861724 0.00144833375067451 0 -66.7355384861738 + 400 0.04 9.04820921016732e-07 -66.8839217283316 0.000682239601485924 0 -66.8839217283314 + 500 0.05 1.6866160174916e-06 -66.9469610803664 0.00032462625992713 0 -66.946961080365 + 600 0.06 1.78038217785001e-06 -66.9786113403508 0.000160730704849448 0 -66.9786113403509 + 700 0.07 1.49199057723078e-06 -66.9936279438935 7.39864656758093e-05 0 -66.9936279438931 + 800 0.08 1.15173756711067e-06 -67.0004605630263 3.33959465206462e-05 0 -67.0004605630278 + 900 0.09 8.48526364752965e-07 -67.0035249294341 1.49345737358251e-05 0 -67.0035249294347 + 1000 0.1 6.10346492876059e-07 -67.004895757836 6.71648807105468e-06 0 -67.0048957578347 +Loop time of 9.0061 on 1 procs for 1000 steps with 5780 atoms + +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads Minimization stats: Stopping criterion = max iterations @@ -115,12 +117,12 @@ Minimization stats: MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 8.7686 | 8.7686 | 8.7686 | 0.0 | 92.84 +Pair | 8.3237 | 8.3237 | 8.3237 | 0.0 | 92.42 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.02386 | 0.02386 | 0.02386 | 0.0 | 0.25 -Output | 0.31808 | 0.31808 | 0.31808 | 0.0 | 3.37 -Modify | 0.029416 | 0.029416 | 0.029416 | 0.0 | 0.31 -Other | | 0.305 | | | 3.23 +Comm | 0.023927 | 0.023927 | 0.023927 | 0.0 | 0.27 +Output | 0.31783 | 0.31783 | 0.31783 | 0.0 | 3.53 +Modify | 0.034633 | 0.034633 | 0.034633 | 0.0 | 0.38 +Other | | 0.306 | | | 3.40 Nlocal: 5780 ave 5780 max 5780 min Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/spinmin/log.19Nov19.spin.bfo_min_cg.g++.4 b/examples/SPIN/spinmin/log.14Apr20.spin.bfo_min_cg.g++.4 similarity index 62% rename from examples/SPIN/spinmin/log.19Nov19.spin.bfo_min_cg.g++.4 rename to examples/SPIN/spinmin/log.14Apr20.spin.bfo_min_cg.g++.4 index 730bf477d437f37cff0bab75b42769d081e347ca..26dd99046bd4b8c6244900456e56d36edcbede59 100644 --- a/examples/SPIN/spinmin/log.19Nov19.spin.bfo_min_cg.g++.4 +++ b/examples/SPIN/spinmin/log.14Apr20.spin.bfo_min_cg.g++.4 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bfo in a 3d periodic box units metal @@ -17,7 +19,7 @@ Created orthogonal box = (0 0 0) to (134.64 134.64 19.8) 2 by 2 by 1 MPI processor grid create_atoms 1 box Created 5780 atoms - create_atoms CPU = 0.00138712 secs + create_atoms CPU = 0.000908136 secs # setting mass, mag. moments, and interactions for bcc iron @@ -59,11 +61,11 @@ dump 1 all custom 50 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3 min_style spin/cg # min_modify line spin_none discrete_factor 10.0 minimize 1.0e-10 1.0e-10 1000 100 -WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:177) +WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:190) WARNING: Line search incompatible gneb (../min_spin_cg.cpp:105) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) Neighbor list info ... update every 1 steps, delay 0 steps, check yes max neighbors/atom: 2000, page size: 100000 @@ -88,25 +90,25 @@ Neighbor list info ... bin: none Per MPI rank memory allocation (min/avg/max) = 7.788 | 7.788 | 7.788 Mbytes Step Time v_magnorm v_emag v_tmag Temp TotEng - 0 0 0.0100717228668283 -0.162177519662199 14970.709092345 0 -0.157514482753586 - 100 0.01 8.97646420936397e-06 -132.756468673032 0.00226858475243123 0 -132.79881239587 - 200 0.02 5.7049674439631e-06 -133.065966570145 0.000924384747875186 0 -133.105411060402 - 300 0.03 7.08166486348038e-06 -133.359072681024 0.00128114254070688 0 -133.406669528642 - 400 0.04 4.60224970353229e-06 -133.668643035703 0.000822334798448062 0 -133.725353643022 - 500 0.05 3.13737045264193e-06 -133.819548711647 0.000369678417461456 0 -133.878037514585 - 600 0.06 2.55239214469856e-06 -133.889302880669 0.0001696142482835 0 -133.948327309746 - 700 0.07 1.92236411979341e-06 -133.920147501261 7.31985644003847e-05 0 -133.979597440788 - 800 0.08 1.40879742055238e-06 -133.933445418833 3.19349095035109e-05 0 -133.993344750158 - 900 0.09 1.02629246257047e-06 -133.939321574068 1.44399877051467e-05 0 -133.999611147322 - 1000 0.1 7.52253147824893e-07 -133.942032102451 6.85789018963965e-06 0 -134.002604512509 -Loop time of 2.49676 on 4 procs for 1000 steps with 5780 atoms - -100.0% CPU use with 4 MPI tasks x no OpenMP threads + 0 0 0.0100717228668283 -0.0819172086358889 14831.3069413956 0 -0.0819172086358937 + 100 0.01 8.80197005315463e-06 -66.403173198835 0.00226660536216922 0 -66.4031731988351 + 200 0.02 6.7090325022051e-06 -66.563539121677 0.00103783628361663 0 -66.5635391216769 + 300 0.03 4.53816034526952e-06 -66.7355384861727 0.00144833375067451 0 -66.7355384861724 + 400 0.04 9.04820921046463e-07 -66.8839217283311 0.000682239601485917 0 -66.883921728331 + 500 0.05 1.68661601751279e-06 -66.9469610803655 0.00032462625992713 0 -66.9469610803655 + 600 0.06 1.78038217786708e-06 -66.9786113403506 0.000160730704849447 0 -66.9786113403508 + 700 0.07 1.4919905772389e-06 -66.9936279438931 7.39864656758075e-05 0 -66.9936279438932 + 800 0.08 1.15173756711682e-06 -67.0004605630262 3.33959465206457e-05 0 -67.0004605630265 + 900 0.09 8.48526364761828e-07 -67.0035249294341 1.49345737358251e-05 0 -67.0035249294341 + 1000 0.1 6.10346492874848e-07 -67.0048957578355 6.71648807105468e-06 0 -67.0048957578356 +Loop time of 2.40179 on 4 procs for 1000 steps with 5780 atoms + +100.0% CPU use with 4 MPI tasks x 1 OpenMP threads Minimization stats: Stopping criterion = max iterations Energy initial, next-to-last, final = - -0.157514482754 -134.00257032 -134.002604513 + -0.0819172086359 -67.0048809251 -67.0048957578 Force two-norm initial, final = 0 0 Force max component initial, final = 0 0 Final line search alpha, max atom move = 0 0 @@ -115,12 +117,12 @@ Minimization stats: MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 2.2509 | 2.2589 | 2.2629 | 0.3 | 90.47 +Pair | 2.1174 | 2.1533 | 2.179 | 1.6 | 89.66 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.06036 | 0.064254 | 0.072356 | 1.9 | 2.57 -Output | 0.084002 | 0.085009 | 0.085985 | 0.3 | 3.40 -Modify | 0.0072496 | 0.0072694 | 0.0073116 | 0.0 | 0.29 -Other | | 0.08134 | | | 3.26 +Comm | 0.049849 | 0.075596 | 0.11152 | 8.4 | 3.15 +Output | 0.082974 | 0.084081 | 0.085145 | 0.3 | 3.50 +Modify | 0.0082345 | 0.008339 | 0.008424 | 0.1 | 0.35 +Other | | 0.08043 | | | 3.35 Nlocal: 1445 ave 1445 max 1445 min Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/spinmin/log.19Nov19.spin.bfo_min_lbfgs.g++.1 b/examples/SPIN/spinmin/log.14Apr20.spin.bfo_min_lbfgs.g++.1 similarity index 69% rename from examples/SPIN/spinmin/log.19Nov19.spin.bfo_min_lbfgs.g++.1 rename to examples/SPIN/spinmin/log.14Apr20.spin.bfo_min_lbfgs.g++.1 index a8685eb2cf9b337e849c72d14d18d6d5819a03bb..2d8b319b3cff54162ec41037ea95ce7879c611e1 100644 --- a/examples/SPIN/spinmin/log.19Nov19.spin.bfo_min_lbfgs.g++.1 +++ b/examples/SPIN/spinmin/log.14Apr20.spin.bfo_min_lbfgs.g++.1 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bfo in a 3d periodic box units metal @@ -17,7 +19,7 @@ Created orthogonal box = (0 0 0) to (134.64 134.64 3.96) 1 by 1 by 1 MPI processor grid create_atoms 1 box Created 1156 atoms - create_atoms CPU = 0.00136805 secs + create_atoms CPU = 0.00159788 secs # setting mass, mag. moments, and interactions for bcc iron @@ -60,11 +62,11 @@ min_style spin/lbfgs # min_modify line spin_cubic discrete_factor 10.0 min_modify norm max minimize 1.0e-15 1.0e-10 1000 100 -WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:177) +WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:190) WARNING: Line search incompatible gneb (../min_spin_lbfgs.cpp:109) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) Neighbor list info ... update every 1 steps, delay 0 steps, check yes max neighbors/atom: 2000, page size: 100000 @@ -89,17 +91,17 @@ Neighbor list info ... bin: none Per MPI rank memory allocation (min/avg/max) = 7.748 | 7.748 | 7.748 Mbytes Step Time v_magnorm v_emag v_tmag Temp TotEng - 0 0 0.0205636053306396 -0.218504643888467 1537.40479337332 0 -0.109252321944233 - 50 0.005 0.000800557938107919 -19.2937186217138 0.293526226015746 0 -9.65918446070018 - 100 0.01 0.000434178067296136 -19.6225314972776 0.136842093090897 0 -9.81803976806459 - 150 0.015 9.48307628510239e-06 -19.7062424007137 0.000835148627123792 0 -9.85315267460932 - 200 0.02 9.40072944704056e-06 -19.7072931204571 7.72334770010361e-06 0 -9.85364693487844 - 250 0.025 5.05117500164935e-07 -19.7072952885901 5.72437821949831e-08 0 -9.85364764712939 - 300 0.03 2.15063977474981e-09 -19.707295295749 2.09970244523395e-12 0 -9.8536476478746 - 303 0.0303 1.43831710574092e-09 -19.7072952957498 1.70336397715489e-13 0 -9.85364764787493 -Loop time of 0.335897 on 1 procs for 303 steps with 1156 atoms - -100.0% CPU use with 1 MPI tasks x no OpenMP threads + 0 0 0.0205636053306396 -0.109252321944233 1537.40479337332 0 -0.109252321944233 + 50 0.005 0.000800557938107919 -9.65918446070017 0.293526226015746 0 -9.65918446070018 + 100 0.01 0.000434178067296136 -9.81803976806455 0.136842093090897 0 -9.81803976806459 + 150 0.015 9.48307628510239e-06 -9.85315267460926 0.000835148627123792 0 -9.85315267460932 + 200 0.02 9.40072944704056e-06 -9.85364693487847 7.72334770010361e-06 0 -9.85364693487844 + 250 0.025 5.05117500164935e-07 -9.85364764712935 5.72437821949831e-08 0 -9.85364764712939 + 300 0.03 2.15063977474981e-09 -9.85364764787459 2.09970244523395e-12 0 -9.8536476478746 + 303 0.0303 1.43831710574092e-09 -9.85364764787493 1.70336397715489e-13 0 -9.85364764787493 +Loop time of 0.328641 on 1 procs for 303 steps with 1156 atoms + +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads Minimization stats: Stopping criterion = energy tolerance @@ -113,12 +115,12 @@ Minimization stats: MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 0.28129 | 0.28129 | 0.28129 | 0.0 | 83.74 +Pair | 0.27332 | 0.27332 | 0.27332 | 0.0 | 83.17 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.0010796 | 0.0010796 | 0.0010796 | 0.0 | 0.32 -Output | 0.017942 | 0.017942 | 0.017942 | 0.0 | 5.34 -Modify | 0.001771 | 0.001771 | 0.001771 | 0.0 | 0.53 -Other | | 0.03382 | | | 10.07 +Comm | 0.001117 | 0.001117 | 0.001117 | 0.0 | 0.34 +Output | 0.018038 | 0.018038 | 0.018038 | 0.0 | 5.49 +Modify | 0.0020421 | 0.0020421 | 0.0020421 | 0.0 | 0.62 +Other | | 0.03412 | | | 10.38 Nlocal: 1156 ave 1156 max 1156 min Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/spinmin/log.19Nov19.spin.bfo_min_lbfgs.g++.4 b/examples/SPIN/spinmin/log.14Apr20.spin.bfo_min_lbfgs.g++.4 similarity index 66% rename from examples/SPIN/spinmin/log.19Nov19.spin.bfo_min_lbfgs.g++.4 rename to examples/SPIN/spinmin/log.14Apr20.spin.bfo_min_lbfgs.g++.4 index f2756d0f5ed9f0d9450edae668125ecfa82941d9..2c26f64b564c7af82ab8a81d7184c242e179f26c 100644 --- a/examples/SPIN/spinmin/log.19Nov19.spin.bfo_min_lbfgs.g++.4 +++ b/examples/SPIN/spinmin/log.14Apr20.spin.bfo_min_lbfgs.g++.4 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bfo in a 3d periodic box units metal @@ -17,7 +19,7 @@ Created orthogonal box = (0 0 0) to (134.64 134.64 3.96) 2 by 2 by 1 MPI processor grid create_atoms 1 box Created 1156 atoms - create_atoms CPU = 0.000981808 secs + create_atoms CPU = 0.00098896 secs # setting mass, mag. moments, and interactions for bcc iron @@ -60,11 +62,11 @@ min_style spin/lbfgs # min_modify line spin_cubic discrete_factor 10.0 min_modify norm max minimize 1.0e-15 1.0e-10 1000 100 -WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:177) +WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:190) WARNING: Line search incompatible gneb (../min_spin_lbfgs.cpp:109) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) Neighbor list info ... update every 1 steps, delay 0 steps, check yes max neighbors/atom: 2000, page size: 100000 @@ -89,36 +91,36 @@ Neighbor list info ... bin: none Per MPI rank memory allocation (min/avg/max) = 7.619 | 7.619 | 7.619 Mbytes Step Time v_magnorm v_emag v_tmag Temp TotEng - 0 0 0.0205636053306396 -0.217760509274282 1541.29975585882 0 -0.217230771393012 - 50 0.005 0.000966655616837406 -19.2878369426356 0.312860071233841 0 -19.3229939390148 - 100 0.01 0.00154452800191107 -19.5948898197917 0.365367666925029 0 -19.6389064900413 - 150 0.015 4.89955946750017e-05 -19.6962580067431 0.000385536538802408 0 -19.7047140195852 - 200 0.02 5.66300530875654e-05 -19.6975252647309 9.8679922880911e-05 0 -19.7059140354146 - 250 0.025 5.21141123128679e-08 -19.6975359469038 2.52554968535685e-09 0 -19.7059189333986 - 300 0.03 2.9845103782958e-09 -19.6975359475094 2.31782597655471e-11 0 -19.7059191124033 - 342 0.0342 1.0526549233076e-10 -19.6975359475123 3.65641352240487e-16 0 -19.7059191178145 -Loop time of 0.117672 on 4 procs for 342 steps with 1156 atoms - -99.4% CPU use with 4 MPI tasks x no OpenMP threads + 0 0 0.0205636053306396 -0.109252321944234 1537.40479337332 0 -0.109252321944234 + 50 0.005 0.00080055793810568 -9.65918446070018 0.293526226015742 0 -9.65918446070018 + 100 0.01 0.000434178089299568 -9.81803976800935 0.136842091062742 0 -9.81803976800936 + 150 0.015 9.48305696976556e-06 -9.85315267463944 0.000835124161214539 0 -9.85315267463943 + 200 0.02 9.48404729540188e-06 -9.85364693589303 7.83955637621785e-06 0 -9.85364693589302 + 250 0.025 2.53129638591035e-07 -9.85364764689316 5.38044599466511e-08 0 -9.85364764689315 + 300 0.03 1.70353573452093e-08 -9.85364764787448 5.65371929906417e-11 0 -9.85364764787448 + 318 0.0318 1.78606698438076e-09 -9.85364764787558 5.01580212144594e-14 0 -9.85364764787559 +Loop time of 0.107177 on 4 procs for 318 steps with 1156 atoms + +99.6% CPU use with 4 MPI tasks x 1 OpenMP threads Minimization stats: - Stopping criterion = force tolerance + Stopping criterion = energy tolerance Energy initial, next-to-last, final = - -0.217230771393 -19.7059191178 -19.7059191178 + -0.109252321944 -9.85364764788 -9.85364764788 Force two-norm initial, final = 0 0 Force max component initial, final = 0 0 Final line search alpha, max atom move = 0 0 - Iterations, force evaluations = 342 342 + Iterations, force evaluations = 318 318 MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 0.084558 | 0.086668 | 0.091471 | 1.0 | 73.65 +Pair | 0.074155 | 0.077992 | 0.081876 | 1.2 | 72.77 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.0052197 | 0.010042 | 0.012191 | 2.8 | 8.53 -Output | 0.0050647 | 0.0050726 | 0.0050921 | 0.0 | 4.31 -Modify | 0.00052595 | 0.00053537 | 0.00055242 | 0.0 | 0.45 -Other | | 0.01535 | | | 13.05 +Comm | 0.005573 | 0.0094561 | 0.013285 | 3.4 | 8.82 +Output | 0.005054 | 0.0050632 | 0.0050869 | 0.0 | 4.72 +Modify | 0.00055051 | 0.00058657 | 0.00066018 | 0.0 | 0.55 +Other | | 0.01408 | | | 13.14 Nlocal: 289 ave 289 max 289 min Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/spinmin/log.19Nov19.spin.iron_min.g++.1 b/examples/SPIN/spinmin/log.14Apr20.spin.iron_min.g++.1 similarity index 87% rename from examples/SPIN/spinmin/log.19Nov19.spin.iron_min.g++.1 rename to examples/SPIN/spinmin/log.14Apr20.spin.iron_min.g++.1 index 6df5959c0b371641644cfb43625da94c2f972661..7fe5920c1c2093c4cafd02c227347dde117ff685 100644 --- a/examples/SPIN/spinmin/log.19Nov19.spin.iron_min.g++.1 +++ b/examples/SPIN/spinmin/log.14Apr20.spin.iron_min.g++.1 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bcc iron in a 3d periodic box units metal @@ -17,7 +19,7 @@ Created orthogonal box = (0 0 0) to (14.3325 14.3325 14.3325) 1 by 1 by 1 MPI processor grid create_atoms 1 box Created 250 atoms - create_atoms CPU = 0.000965834 secs + create_atoms CPU = 0.00106812 secs # setting mass, mag. moments, and interactions for bcc iron @@ -60,8 +62,8 @@ dump 1 all custom 100 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[ min_style spin min_modify alpha_damp 1.0 discrete_factor 10.0 minimize 1.0e-10 1.0e-10 1000 100 -WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:177) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) +WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:190) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) Neighbor list info ... update every 1 steps, delay 0 steps, check yes max neighbors/atom: 2000, page size: 100000 @@ -87,9 +89,9 @@ Step Time v_magx v_magz v_magnorm v_tmag TotEng 800 0.08 -0.584017381477007 -0.0766850043611195 0.999999999999999 6.70634191991825e-06 -25.289508192325 900 0.09 -0.583812040722351 -0.0811143180675364 0.999999999999999 7.49814943594148e-06 -25.2895256658925 1000 0.1 -0.583582389243979 -0.0857958823565731 0.999999999999998 8.38204259112222e-06 -25.2895452009133 -Loop time of 0.195254 on 1 procs for 1000 steps with 250 atoms +Loop time of 0.199799 on 1 procs for 1000 steps with 250 atoms -100.0% CPU use with 1 MPI tasks x no OpenMP threads +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads Minimization stats: Stopping criterion = max iterations @@ -103,12 +105,12 @@ Minimization stats: MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 0.17668 | 0.17668 | 0.17668 | 0.0 | 90.49 +Pair | 0.1795 | 0.1795 | 0.1795 | 0.0 | 89.84 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.0052176 | 0.0052176 | 0.0052176 | 0.0 | 2.67 -Output | 0.0067751 | 0.0067751 | 0.0067751 | 0.0 | 3.47 -Modify | 0.0013788 | 0.0013788 | 0.0013788 | 0.0 | 0.71 -Other | | 0.005203 | | | 2.66 +Comm | 0.0056038 | 0.0056038 | 0.0056038 | 0.0 | 2.80 +Output | 0.0074773 | 0.0074773 | 0.0074773 | 0.0 | 3.74 +Modify | 0.0016823 | 0.0016823 | 0.0016823 | 0.0 | 0.84 +Other | | 0.005533 | | | 2.77 Nlocal: 250 ave 250 max 250 min Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/spinmin/log.19Nov19.spin.iron_min.g++.4 b/examples/SPIN/spinmin/log.14Apr20.spin.iron_min.g++.4 similarity index 72% rename from examples/SPIN/spinmin/log.19Nov19.spin.iron_min.g++.4 rename to examples/SPIN/spinmin/log.14Apr20.spin.iron_min.g++.4 index b0cdd4f94adfbce6ae64b45bd546e0b6de6f3373..0eb7bd1783058f547ee4c1499e24041bba305edb 100644 --- a/examples/SPIN/spinmin/log.19Nov19.spin.iron_min.g++.4 +++ b/examples/SPIN/spinmin/log.14Apr20.spin.iron_min.g++.4 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bcc iron in a 3d periodic box units metal @@ -17,7 +19,7 @@ Created orthogonal box = (0 0 0) to (14.3325 14.3325 14.3325) 1 by 2 by 2 MPI processor grid create_atoms 1 box Created 250 atoms - create_atoms CPU = 0.000759125 secs + create_atoms CPU = 0.000731945 secs # setting mass, mag. moments, and interactions for bcc iron @@ -60,8 +62,8 @@ dump 1 all custom 100 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[ min_style spin min_modify alpha_damp 1.0 discrete_factor 10.0 minimize 1.0e-10 1.0e-10 1000 100 -WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:177) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) +WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:190) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) Neighbor list info ... update every 1 steps, delay 0 steps, check yes max neighbors/atom: 2000, page size: 100000 @@ -76,25 +78,25 @@ Neighbor list info ... bin: standard Per MPI rank memory allocation (min/avg/max) = 6.828 | 6.829 | 6.829 Mbytes Step Time v_magx v_magz v_magnorm v_tmag TotEng - 0 0 -0.0285071136621457 -0.00948990815281275 0.0764569750905723 5048.5607623768 -0.701465876910695 - 100 0.01 -0.584953861980204 -0.0517163256267969 0.999992350892306 6.25556948778489e-05 -50.5787443620229 - 200 0.02 -0.584864756506845 -0.0547143484057154 0.999999990495506 3.49782260454051e-06 -50.5787971409246 - 300 0.03 -0.5847600493607 -0.0578846348986585 0.999999999988173 3.83095226804998e-06 -50.5788061208592 - 400 0.04 -0.584642875238891 -0.06123730753627 0.999999999999984 4.28575832708228e-06 -50.5788161053499 - 500 0.05 -0.584511765589526 -0.0647826190376232 0.999999999999999 4.79421486949061e-06 -50.5788272748473 - 600 0.06 -0.584365074206158 -0.0685313536438759 0.999999999999999 5.36242072641826e-06 -50.5788397688148 - 700 0.07 -0.584200963215272 -0.0724948469588718 1 5.99725249459218e-06 -50.5788537427249 - 800 0.08 -0.584017381477007 -0.0766850043611196 1 6.7063419199184e-06 -50.5788693699014 - 900 0.09 -0.583812040722352 -0.0811143180675365 0.999999999999998 7.49814943594153e-06 -50.5788868434688 - 1000 0.1 -0.583582389243979 -0.0857958823565732 0.999999999999999 8.38204259112203e-06 -50.5789063784897 -Loop time of 0.0845464 on 4 procs for 1000 steps with 250 atoms - -99.8% CPU use with 4 MPI tasks x no OpenMP threads + 0 0 -0.0285071136621457 -0.00948990815281275 0.0764569750905723 5048.5607623768 -0.354774619362399 + 100 0.01 -0.584953861980204 -0.0517163256267969 0.999992350892306 6.25556948778489e-05 -25.2894057771132 + 200 0.02 -0.584864756506845 -0.0547143484057154 0.999999990495506 3.49782260454051e-06 -25.2894359914181 + 300 0.03 -0.5847600493607 -0.0578846348986585 0.999999999988173 3.83095226804998e-06 -25.2894449433168 + 400 0.04 -0.584642875238891 -0.06123730753627 0.999999999999984 4.28575832708228e-06 -25.2894549277729 + 500 0.05 -0.584511765589526 -0.0647826190376232 0.999999999999999 4.79421486949061e-06 -25.2894660972702 + 600 0.06 -0.584365074206158 -0.0685313536438759 0.999999999999999 5.36242072641826e-06 -25.2894785912378 + 700 0.07 -0.584200963215272 -0.0724948469588718 1 5.99725249459218e-06 -25.2894925651479 + 800 0.08 -0.584017381477007 -0.0766850043611196 1 6.7063419199184e-06 -25.2895081923244 + 900 0.09 -0.583812040722352 -0.0811143180675365 0.999999999999998 7.49814943594153e-06 -25.2895256658919 + 1000 0.1 -0.583582389243979 -0.0857958823565732 0.999999999999999 8.38204259112203e-06 -25.2895452009127 +Loop time of 0.086317 on 4 procs for 1000 steps with 250 atoms + +99.9% CPU use with 4 MPI tasks x 1 OpenMP threads Minimization stats: Stopping criterion = max iterations Energy initial, next-to-last, final = - -0.701465876911 -50.5789061722 -50.5789063785 + -0.354774619362 -25.2895449946 -25.2895452009 Force two-norm initial, final = 0 0 Force max component initial, final = 0 0 Final line search alpha, max atom move = 0 0 @@ -103,12 +105,12 @@ Minimization stats: MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 0.043007 | 0.045307 | 0.04776 | 0.8 | 53.59 +Pair | 0.042744 | 0.045964 | 0.049416 | 1.2 | 53.25 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.026827 | 0.029139 | 0.031438 | 1.0 | 34.47 -Output | 0.0023198 | 0.0023302 | 0.0023572 | 0.0 | 2.76 -Modify | 0.00041103 | 0.0004673 | 0.00054026 | 0.0 | 0.55 -Other | | 0.007303 | | | 8.64 +Comm | 0.026905 | 0.030113 | 0.033464 | 1.5 | 34.89 +Output | 0.0023746 | 0.0023909 | 0.0024335 | 0.1 | 2.77 +Modify | 0.00052047 | 0.00055218 | 0.00058579 | 0.0 | 0.64 +Other | | 0.007297 | | | 8.45 Nlocal: 62.5 ave 65 max 60 min Histogram: 2 0 0 0 0 0 0 0 0 2 diff --git a/examples/SPIN/test_problems/README b/examples/SPIN/test_problems/README index 5557e3d42b688428929b87ad3d31cef9f3e39baa..0a1362ec9c54f188ac7a56e766bb4e50890c2428 100644 --- a/examples/SPIN/test_problems/README +++ b/examples/SPIN/test_problems/README @@ -1,4 +1,4 @@ -** The objective of the benchmark examples in this directory +** The objective of the test problems in this directory is the following twofold: - verify the implementation of the LAMMPS' SPIN package by comparing its results to well-known analytic results, or @@ -6,39 +6,39 @@ - provide users with simple comparisons, allowing them to better understand what is implemented in the code. -The LAMMPS input file (bench-*) can be modified, as well as the +The LAMMPS input file (test-*) can be modified, as well as the associated python script, in order to try different comparisons. All scripts can be run by executing the shell script from its directory. Example: -./run-bench-exchange.sh from the benchmarck_damped_exchange/ +./run-test-exchange.sh from the validation_damped_exchange/ directory. -** Below a brief description of the different benchmark +** Below a brief description of the different validation problems: -- benchmarck_damped_precession: +- validation_damped_precession: simulates the damped precession of a single spin in a magnetic field. - Run as: ./run-bench-prec.sh + Run as: ./run-test-prec.sh Output: x, y and z components of the magnetization, and magnetic energy. -- benchmarck_damped_exchange: +- validation_damped_exchange: simulates two spins interacting through the exchange interaction. The parameters in the LAMMPS input script - (bench-spin-precession.in) are calibrated to match the + (test-spin-precession.in) are calibrated to match the exchange definition in the python script (llg_exchange.py). - Run as: ./run-bench-exchange.sh + Run as: ./run-test-exchange.sh Output: average magnetization resulting from the damped precession of the two interacting spins. Also plots the evolution of the magnetic energy. -- benchmarck_langevin_precession: +- validation_langevin_precession: simulates a single spin in a magnetic field and in contact with a thermal bath, and compares the statistical averages of the output to the analytical result of the Langevin function. - Run as: ./run-bench-prec.sh + Run as: ./run-test-prec.sh Output: statistical average of the z-component of the magnetization (along the applied field) and of the magnetic energy versus temperature. Comparison to the Langevin function diff --git a/examples/SPIN/test_problems/validation_damped_exchange/llg_exchange.py b/examples/SPIN/test_problems/validation_damped_exchange/llg_exchange.py index dd1c543bb3558f2b811d4c73e9210ad259d3329f..49eecb5b44d08c5d880c1f34bb6cd4d23476fbd0 100755 --- a/examples/SPIN/test_problems/validation_damped_exchange/llg_exchange.py +++ b/examples/SPIN/test_problems/validation_damped_exchange/llg_exchange.py @@ -65,6 +65,6 @@ for t in range (0,N): # calc. average magnetization Sm = (S1+S2)*0.5 # calc. energy - en = -J0*(np.dot(S1,S2)) + en = -2.0*J0*(np.dot(S1,S2)) # print res. in ps for comparison with LAMMPS print(t*dt/1000.0,Sm[0],Sm[1],Sm[2],en) diff --git a/examples/SPIN/test_problems/validation_damped_exchange/run-test-exchange.sh b/examples/SPIN/test_problems/validation_damped_exchange/run-test-exchange.sh index 15de8d350e09431dc6d3ef6932ebe446d635af68..599730fe7b88e55c91cb1ef87e93b4ad05ca2eb1 100755 --- a/examples/SPIN/test_problems/validation_damped_exchange/run-test-exchange.sh +++ b/examples/SPIN/test_problems/validation_damped_exchange/run-test-exchange.sh @@ -5,7 +5,7 @@ rm res_*.dat # compute Lammps ./../../../../src/lmp_serial \ - -in bench-spin-precession.in + -in test-spin-precession.in in="$(grep -n Step log.lammps | awk -F ':' '{print $1}')" en="$(grep -n Loop log.lammps | awk -F ':' '{print $1}')" in="$(echo "$in+1" | bc -l)" diff --git a/examples/SPIN/test_problems/validation_damped_exchange/bench-spin-precession.in b/examples/SPIN/test_problems/validation_damped_exchange/test-spin-precession.in similarity index 100% rename from examples/SPIN/test_problems/validation_damped_exchange/bench-spin-precession.in rename to examples/SPIN/test_problems/validation_damped_exchange/test-spin-precession.in diff --git a/examples/SPIN/test_problems/validation_damped_exchange/two_spins.data b/examples/SPIN/test_problems/validation_damped_exchange/two_spins.data new file mode 100644 index 0000000000000000000000000000000000000000..013f8137511fd02e1d16d0f2a4f7106cc4300209 --- /dev/null +++ b/examples/SPIN/test_problems/validation_damped_exchange/two_spins.data @@ -0,0 +1,22 @@ +LAMMPS data file via write_data, version 19 Sep 2019, timestep = 0 + +2 atoms +1 atom types + +0.0 6.0 xlo xhi +0.0 3.0 ylo yhi +0.0 3.0 zlo zhi + +Masses + +1 1 + +Atoms # spin + +1 1 2.0 0.0 0.0 0.0 1.0 0.0 0.0 0 0 0 +2 1 2.0 3.0 0.0 0.0 0.0 1.0 0.0 0 0 0 + +Velocities + +1 0.0 0.0 0.0 +2 0.0 0.0 0.0 diff --git a/examples/SPIN/test_problems/validation_damped_precession/run-test-prec.sh b/examples/SPIN/test_problems/validation_damped_precession/run-test-prec.sh index 49ebc2ac4efd19895877fe847d5e728b0c731096..86091793767c7300328e9a76252b54143410d79e 100755 --- a/examples/SPIN/test_problems/validation_damped_precession/run-test-prec.sh +++ b/examples/SPIN/test_problems/validation_damped_precession/run-test-prec.sh @@ -5,7 +5,7 @@ rm res_*.dat # compute Lammps ./../../../../src/lmp_serial \ - -in bench-spin-precession.in + -in test-spin-precession.in in="$(grep -n Step log.lammps | awk -F ':' '{print $1}')" en="$(grep -n Loop log.lammps | awk -F ':' '{print $1}')" in="$(echo "$in+1" | bc -l)" @@ -13,7 +13,7 @@ en="$(echo "$en-$in" | bc -l)" tail -n +$in log.lammps | head -n $en > res_lammps.dat # compute Langevin -python3 -m llg_precession.py > res_llg.dat +python3 llg_precession.py > res_llg.dat # plot results -python3 -m plot_precession.py res_lammps.dat res_llg.dat +python3 plot_precession.py res_lammps.dat res_llg.dat diff --git a/examples/SPIN/test_problems/validation_damped_precession/bench-spin-precession.in b/examples/SPIN/test_problems/validation_damped_precession/test-spin-precession.in similarity index 90% rename from examples/SPIN/test_problems/validation_damped_precession/bench-spin-precession.in rename to examples/SPIN/test_problems/validation_damped_precession/test-spin-precession.in index ed8a5caeaf747f8bc77c83a77ad74828c865255c..b8e943e579cc6caa67417ea21f534eb316e67137 100644 --- a/examples/SPIN/test_problems/validation_damped_precession/bench-spin-precession.in +++ b/examples/SPIN/test_problems/validation_damped_precession/test-spin-precession.in @@ -3,7 +3,7 @@ units metal atom_style spin atom_modify map array -boundary p p p +boundary f f f # read_data singlespin.data @@ -26,6 +26,7 @@ variable Temperature equal 0.0 variable Nsteps equal 500000 fix 1 all nve/spin lattice no +# fix 2 all precession/spin zeeman ${H} 0.0 0.0 1.0 anisotropy ${Kan} 0.0 0.0 1.0 fix 2 all precession/spin zeeman ${H} 0.0 0.0 1.0 anisotropy ${Kan} 0.0 0.0 1.0 fix_modify 2 energy yes fix 3 all langevin/spin ${Temperature} 0.01 12345 diff --git a/examples/SPIN/test_problems/validation_langevin_precession/langevin.py b/examples/SPIN/test_problems/validation_langevin_precession/langevin.py index 7acb78014316abf67ace7355b7899c4e5e983a43..0e9b634eeba7f0c1997d27c0c7640423a6b8689a 100755 --- a/examples/SPIN/test_problems/validation_langevin_precession/langevin.py +++ b/examples/SPIN/test_problems/validation_langevin_precession/langevin.py @@ -1,12 +1,8 @@ #!/usr/bin/env python3 -#Program fitting the exchange interaction -#Model curve: Bethe-Slater function import numpy as np, pylab, tkinter import matplotlib.pyplot as plt import mpmath as mp -# from scipy.optimize import curve_fit -# from decimal import * mub=5.78901e-5 # Bohr magneton (eV/T) kb=8.617333262145e-5 # Boltzman constant (eV/K) @@ -24,4 +20,3 @@ tf=20.0 for i in range (0,npoints): temp=ti+i*(tf-ti)/npoints print('%lf %lf %lf' % (temp,func(temp),-g*mub*Hz*func(temp))) - diff --git a/examples/SPIN/test_problems/validation_langevin_precession/run-test-prec.sh b/examples/SPIN/test_problems/validation_langevin_precession/run-test-prec.sh index 98fceeca956f26661abc03fca7ffb1760e0526de..bcdd33790380e04e94e965b262dc6a95ec737f03 100755 --- a/examples/SPIN/test_problems/validation_langevin_precession/run-test-prec.sh +++ b/examples/SPIN/test_problems/validation_langevin_precession/run-test-prec.sh @@ -10,10 +10,9 @@ N=20 for (( i=0; i<$N; i++ )) do temp="$(echo "$tempi+$i*($tempf-$tempi)/$N" | bc -l)" - sed s/temperature/${temp}/g bench-prec-spin.template > \ - bench-prec-spin.in - ./../../../../src/lmp_serial \ - -in bench-prec-spin.in + sed s/temperature/${temp}/g test-prec-spin.template > \ + test-prec-spin.in + ./../../../../src/lmp_serial -in test-prec-spin.in Hz="$(tail -n 1 average_spin | awk -F " " '{print $3}')" sz="$(tail -n 1 average_spin | awk -F " " '{print $5}')" en="$(tail -n 1 average_spin | awk -F " " '{print $6}')" @@ -21,7 +20,7 @@ do done # compute Langevin -python3 -m langevin.py > res_langevin.dat +python3 langevin.py > res_langevin.dat # plot results -python3 -m plot_precession.py res_lammps.dat res_langevin.dat +python3 plot_precession.py res_lammps.dat res_langevin.dat diff --git a/examples/SPIN/test_problems/validation_langevin_precession/bench-prec-spin.template b/examples/SPIN/test_problems/validation_langevin_precession/test-prec-spin.template similarity index 100% rename from examples/SPIN/test_problems/validation_langevin_precession/bench-prec-spin.template rename to examples/SPIN/test_problems/validation_langevin_precession/test-prec-spin.template diff --git a/examples/SPIN/test_problems/validation_nve/Fe_Mishin2006.eam.alloy b/examples/SPIN/test_problems/validation_nve/Fe_Mishin2006.eam.alloy new file mode 100644 index 0000000000000000000000000000000000000000..69231bb7eeed5677009482896fe520f4445413dd --- /dev/null +++ b/examples/SPIN/test_problems/validation_nve/Fe_Mishin2006.eam.alloy @@ -0,0 +1,15009 @@ +comment 1 +comment 2 +Converted by Ganga P Purja Pun using C++ code on Mon Nov 3 10:48:17 2014 +1 Fe +5001 2.400000000000000e-03 5001 1.134673400048920e-03 5.673367000244601e+00 +26 5.584700000000000e+01 2.855300000000000e+00 BCC + 0.000000000000000e+00 + -2.338738741480766e-02 + -4.628214468925276e-02 + -6.912258036387915e-02 + -9.175603963501618e-02 + -1.141497214000000e-01 + -1.363388222824136e-01 + -1.583226111166723e-01 + -1.800965752913192e-01 + -2.016645989796093e-01 + -2.230289901000000e-01 + -2.441907391820846e-01 + -2.651515164004249e-01 + -2.859130554412839e-01 + -3.064769176965011e-01 + -3.268446844000000e-01 + -3.470179531091855e-01 + -3.669982968636285e-01 + -3.867872784635140e-01 + -4.063864535839295e-01 + -4.257973671999999e-01 + -4.450215551034667e-01 + -4.640605423684923e-01 + -4.829158454463851e-01 + -5.015889707095635e-01 + -5.200814146000000e-01 + -5.383946650297390e-01 + -5.565301991603658e-01 + -5.744894857268537e-01 + -5.922739837155686e-01 + -6.098851423000000e-01 + -6.273244022645037e-01 + -6.445931939756846e-01 + -6.616929394780281e-01 + -6.786250511352716e-01 + -6.953909318999999e-01 + -7.119919765952718e-01 + -7.284295696432279e-01 + -7.447050873484842e-01 + -7.608198966551055e-01 + -7.767753551999997e-01 + -7.925728126189590e-01 + -8.082136084644670e-01 + -8.236990743647939e-01 + -8.390305327768360e-01 + -8.542092970000001e-01 + -8.692366724924486e-01 + -8.841139549244775e-01 + -8.988424321942350e-01 + -9.134233831702263e-01 + -9.278580778000000e-01 + -9.421477783833053e-01 + -9.562937376266863e-01 + -9.702972006149804e-01 + -9.841594035897939e-01 + -9.978815741999999e-01 + -1.011464932626877e+00 + -1.024910689374227e+00 + -1.038220047492379e+00 + -1.051394201530088e+00 + -1.064434338000000e+00 + -1.077341636148692e+00 + -1.090117264995124e+00 + -1.102762386491716e+00 + -1.115278154872840e+00 + -1.127665716000000e+00 + -1.139926208397942e+00 + -1.152060761127338e+00 + -1.164070496370367e+00 + -1.175956528594255e+00 + -1.187719964000000e+00 + -1.199361901614307e+00 + -1.210883431375302e+00 + -1.222285636497208e+00 + -1.233569592577916e+00 + -1.244736367000000e+00 + -1.255787020138112e+00 + -1.266722603828963e+00 + -1.277544163022812e+00 + -1.288252734706093e+00 + -1.298849349000000e+00 + -1.309335029515858e+00 + -1.319710789766606e+00 + -1.329977637188551e+00 + -1.340136572985217e+00 + -1.350188590000000e+00 + -1.360134674013246e+00 + -1.369975802739515e+00 + -1.379712947700534e+00 + -1.389347073262624e+00 + -1.398879136000000e+00 + -1.408310086019347e+00 + -1.417640866058629e+00 + -1.426872412303706e+00 + -1.436005653062347e+00 + -1.445041510000000e+00 + -1.453980898788701e+00 + -1.462824726507764e+00 + -1.471573894410746e+00 + -1.480229297407323e+00 + -1.488791823000000e+00 + -1.497262352264216e+00 + -1.505641758273629e+00 + -1.513930908641532e+00 + -1.522130664940848e+00 + -1.530241881000000e+00 + -1.538265404312065e+00 + -1.546202075663511e+00 + -1.554052729949587e+00 + -1.561818194964572e+00 + -1.569499292000000e+00 + -1.577096836740392e+00 + -1.584611637631811e+00 + -1.592044497451960e+00 + -1.599396212413425e+00 + -1.606667572000000e+00 + -1.613859360067553e+00 + -1.620972353850301e+00 + -1.628007324820493e+00 + -1.634965037585305e+00 + -1.641846251000000e+00 + -1.648651718746855e+00 + -1.655382187120556e+00 + -1.662038397074428e+00 + -1.668621083574689e+00 + -1.675130975000000e+00 + -1.681568794320005e+00 + -1.687935258528783e+00 + -1.694231079461267e+00 + -1.700456962682860e+00 + -1.706613607000000e+00 + -1.712701705964896e+00 + -1.718721948142722e+00 + -1.724675016499694e+00 + -1.730561586828580e+00 + -1.736382330000000e+00 + -1.742137912457411e+00 + -1.747828994017656e+00 + -1.753456229026646e+00 + -1.759020265526974e+00 + -1.764521747000000e+00 + -1.769961312537754e+00 + -1.775339594027171e+00 + -1.780657218386706e+00 + -1.785914807192779e+00 + -1.791112977000000e+00 + -1.796252339747382e+00 + -1.801333500673453e+00 + -1.806357060360666e+00 + -1.811323614328498e+00 + -1.816233753000000e+00 + -1.821088062150410e+00 + -1.825887120985276e+00 + -1.830631504346673e+00 + -1.835321782397329e+00 + -1.839958520000000e+00 + -1.844542277375300e+00 + -1.849073608812209e+00 + -1.853553064572349e+00 + -1.857981190434394e+00 + -1.862358526000000e+00 + -1.866685606181999e+00 + -1.870962962067843e+00 + -1.875191119963197e+00 + -1.879370599902195e+00 + -1.883501918000000e+00 + -1.887585586736770e+00 + -1.891622112755978e+00 + -1.895611998485404e+00 + -1.899555741651499e+00 + -1.903453835000000e+00 + -1.907306767129976e+00 + -1.911115021853308e+00 + -1.914879078883785e+00 + -1.918599413067872e+00 + -1.922276495000000e+00 + -1.925910791456493e+00 + -1.929502763824010e+00 + -1.933052869645757e+00 + -1.936561562187136e+00 + -1.940029290000000e+00 + -1.943456497684402e+00 + -1.946843625242175e+00 + -1.950191109032846e+00 + -1.953499381145042e+00 + -1.956768869000000e+00 + -1.959999996258605e+00 + -1.963193182622893e+00 + -1.966348843806269e+00 + -1.969467390558660e+00 + -1.972549230000000e+00 + -1.975594766012145e+00 + -1.978604397775412e+00 + -1.981578520793180e+00 + -1.984517526364655e+00 + -1.987421802000000e+00 + -1.990291731861874e+00 + -1.993127695559656e+00 + -1.995930069364364e+00 + -1.998699225767272e+00 + -2.001435533000000e+00 + -2.004139355858378e+00 + -2.006811055538481e+00 + -2.009450989763890e+00 + -2.012059511961222e+00 + -2.014636972000000e+00 + -2.017183716742461e+00 + -2.019700089231340e+00 + -2.022186429001382e+00 + -2.024643071393187e+00 + -2.027070349000000e+00 + -2.029468591629637e+00 + -2.031838124095820e+00 + -2.034179268100215e+00 + -2.036492342201070e+00 + -2.038777662000000e+00 + -2.041035540156417e+00 + -2.043266284490031e+00 + -2.045470200083453e+00 + -2.047647589335177e+00 + -2.049798751000000e+00 + -2.051923980684690e+00 + -2.054023570214561e+00 + -2.056097808968247e+00 + -2.058146983649728e+00 + -2.060171377000000e+00 + -2.062171268551674e+00 + -2.064146934640815e+00 + -2.066098649159696e+00 + -2.068026683134947e+00 + -2.069931304000000e+00 + -2.071812776276246e+00 + -2.073671361471072e+00 + -2.075507318519566e+00 + -2.077320903301032e+00 + -2.079112369000000e+00 + -2.080881966139512e+00 + -2.082629940879466e+00 + -2.084356537486872e+00 + -2.086061998635378e+00 + -2.087746563000000e+00 + -2.089410466229110e+00 + -2.091053941828707e+00 + -2.092677220771137e+00 + -2.094280530723314e+00 + -2.095864097000000e+00 + -2.097428142794787e+00 + -2.098972888139794e+00 + -2.100498550462023e+00 + -2.102005344216306e+00 + -2.103493482000000e+00 + -2.104963174393948e+00 + -2.106414628045970e+00 + -2.107848047779690e+00 + -2.109263636820768e+00 + -2.110661595000000e+00 + -2.112042119471926e+00 + -2.113405405101743e+00 + -2.114751645122076e+00 + -2.116081030815763e+00 + -2.117393750000000e+00 + -2.118689987937912e+00 + -2.119969928191390e+00 + -2.121233752504371e+00 + -2.122481640223532e+00 + -2.123713768000000e+00 + -2.124930310308645e+00 + -2.126131439345980e+00 + -2.127317325650607e+00 + -2.128488137848567e+00 + -2.129644042000000e+00 + -2.130785202020548e+00 + -2.131911779449632e+00 + -2.133023934312029e+00 + -2.134121824962966e+00 + -2.135205607000000e+00 + -2.136275433910872e+00 + -2.137331457489147e+00 + -2.138373827747866e+00 + -2.139402692469194e+00 + -2.140418197999999e+00 + -2.141420489142081e+00 + -2.142409707722756e+00 + -2.143385994030927e+00 + -2.144349486926974e+00 + -2.145300323000000e+00 + -2.146238637076898e+00 + -2.147164562479522e+00 + -2.148078231051947e+00 + -2.148979772720135e+00 + -2.149869315000000e+00 + -2.150746983835218e+00 + -2.151612904662093e+00 + -2.152467200947815e+00 + -2.153309993218032e+00 + -2.154141401000000e+00 + -2.154961542883784e+00 + -2.155770535619624e+00 + -2.156568494253713e+00 + -2.157355531798206e+00 + -2.158131760000000e+00 + -2.158897289576013e+00 + -2.159652229917454e+00 + -2.160396688379106e+00 + -2.161130769707277e+00 + -2.161854579000000e+00 + -2.162568220747540e+00 + -2.163271795520776e+00 + -2.163965403104795e+00 + -2.164649143391419e+00 + -2.165323114000000e+00 + -2.165987410839904e+00 + -2.166642128874188e+00 + -2.167287362019562e+00 + -2.167923202786971e+00 + -2.168549742000000e+00 + -2.169167069289456e+00 + -2.169775273557456e+00 + -2.170374442741976e+00 + -2.170964663390078e+00 + -2.171546020000000e+00 + -2.172118595836383e+00 + -2.172682474202094e+00 + -2.173237737093693e+00 + -2.173784464448991e+00 + -2.174322736000000e+00 + -2.174852630794634e+00 + -2.175374225221640e+00 + -2.175887595177988e+00 + -2.176392816678372e+00 + -2.176889964000000e+00 + -2.177379110090651e+00 + -2.177860327096639e+00 + -2.178333686466943e+00 + -2.178799258747679e+00 + -2.179257113000000e+00 + -2.179707317299531e+00 + -2.180149939409128e+00 + -2.180585046165010e+00 + -2.181012703042297e+00 + -2.181432975000000e+00 + -2.181845926509731e+00 + -2.182251621108717e+00 + -2.182650121032542e+00 + -2.183041486916871e+00 + -2.183425780000000e+00 + -2.183803061402426e+00 + -2.184173389857596e+00 + -2.184536823542827e+00 + -2.184893420600431e+00 + -2.185243238000000e+00 + -2.185586332004080e+00 + -2.185922759056265e+00 + -2.186252574500474e+00 + -2.186575831916728e+00 + -2.186892585000000e+00 + -2.187202887393400e+00 + -2.187506791832633e+00 + -2.187804350256584e+00 + -2.188095613705712e+00 + -2.188380633000000e+00 + -2.188659458643119e+00 + -2.188932140238827e+00 + -2.189198726845117e+00 + -2.189459267002233e+00 + -2.189713809000000e+00 + -2.189962400732430e+00 + -2.190205089044370e+00 + -2.190441920581333e+00 + -2.190672942002169e+00 + -2.190898199000000e+00 + -2.191117736662499e+00 + -2.191331600149922e+00 + -2.191539834109547e+00 + -2.191742482380937e+00 + -2.191939589000000e+00 + -2.192131197889608e+00 + -2.192317351831708e+00 + -2.192498093290244e+00 + -2.192673464653123e+00 + -2.192843508000000e+00 + -2.193008265149897e+00 + -2.193167777657277e+00 + -2.193322086821774e+00 + -2.193471233640668e+00 + -2.193615259000000e+00 + -2.193754203483761e+00 + -2.193888106693908e+00 + -2.194017008313546e+00 + -2.194140948496115e+00 + -2.194259967000000e+00 + -2.194374103230534e+00 + -2.194483396307522e+00 + -2.194587885225862e+00 + -2.194687608881331e+00 + -2.194782606000000e+00 + -2.194872915200472e+00 + -2.194958575082935e+00 + -2.195039623893670e+00 + -2.195116099394696e+00 + -2.195188040000000e+00 + -2.195255484352944e+00 + -2.195318470219860e+00 + -2.195377035223315e+00 + -2.195431217130485e+00 + -2.195481054000000e+00 + -2.195526583926446e+00 + -2.195567844428364e+00 + -2.195604873029958e+00 + -2.195637707516086e+00 + -2.195666386000000e+00 + -2.195690946583878e+00 + -2.195711426545218e+00 + -2.195727863658885e+00 + -2.195740296673043e+00 + -2.195748763000000e+00 + -2.195753299607926e+00 + -2.195753945356280e+00 + -2.195750739132009e+00 + -2.195743719093693e+00 + -2.195732923000000e+00 + -2.195718388771941e+00 + -2.195700155840199e+00 + -2.195678263244660e+00 + -2.195652748934943e+00 + -2.195623652000000e+00 + -2.195591012099144e+00 + -2.195554867993407e+00 + -2.195515258985693e+00 + -2.195472225489285e+00 + -2.195425807000000e+00 + -2.195376042760528e+00 + -2.195322973614639e+00 + -2.195266640267298e+00 + -2.195207082686241e+00 + -2.195144342000000e+00 + -2.195078459809379e+00 + -2.195009476453377e+00 + -2.194937433066797e+00 + -2.194862372466152e+00 + -2.194784337000000e+00 + -2.194703368615792e+00 + -2.194619509257747e+00 + -2.194532801771278e+00 + -2.194443290305841e+00 + -2.194351018000000e+00 + -2.194256027737437e+00 + -2.194158364279782e+00 + -2.194058072250755e+00 + -2.193955195446114e+00 + -2.193849779000000e+00 + -2.193741868885436e+00 + -2.193631510785602e+00 + -2.193518750557477e+00 + -2.193403634464252e+00 + -2.193286209000000e+00 + -2.193166521091908e+00 + -2.193044618884907e+00 + -2.192920550384463e+00 + -2.192794362988794e+00 + -2.192666105000000e+00 + -2.192535825071375e+00 + -2.192403570783236e+00 + -2.192269389306454e+00 + -2.192133327558461e+00 + -2.191995432000000e+00 + -2.191855748732582e+00 + -2.191714323538803e+00 + -2.191571201647935e+00 + -2.191426427571225e+00 + -2.191280046000000e+00 + -2.191132101498374e+00 + -2.190982637495000e+00 + -2.190831697104933e+00 + -2.190679323369808e+00 + -2.190525559000000e+00 + -2.190370446239990e+00 + -2.190214026313205e+00 + -2.190056340263544e+00 + -2.189897429222788e+00 + -2.189737334000000e+00 + -2.189576094894156e+00 + -2.189413751026334e+00 + -2.189250341662785e+00 + -2.189085906605225e+00 + -2.188920484000000e+00 + -2.188754111167896e+00 + -2.188586826517446e+00 + -2.188418667831656e+00 + -2.188249671505532e+00 + -2.188079874000000e+00 + -2.187909311655257e+00 + -2.187738019850499e+00 + -2.187566033827133e+00 + -2.187393388939138e+00 + -2.187220120000000e+00 + -2.187046261198758e+00 + -2.186871845656608e+00 + -2.186696906557450e+00 + -2.186521477499284e+00 + -2.186345591000000e+00 + -2.186169278929348e+00 + -2.185992573469870e+00 + -2.185815507881449e+00 + -2.185638116930091e+00 + -2.185460435000000e+00 + -2.185282496723147e+00 + -2.185104339046128e+00 + -2.184925999471230e+00 + -2.184747515559985e+00 + -2.184568926000000e+00 + -2.184390270419004e+00 + -2.184211589307456e+00 + -2.184032923638580e+00 + -2.183854314918697e+00 + -2.183675806000000e+00 + -2.183497440664645e+00 + -2.183319263062145e+00 + -2.183141318226586e+00 + -2.182963652427783e+00 + -2.182786312000000e+00 + -2.182609343803448e+00 + -2.182432796909005e+00 + -2.182256720850984e+00 + -2.182081165100584e+00 + -2.181906180000000e+00 + -2.181731816772078e+00 + -2.181558127901617e+00 + -2.181385166690271e+00 + -2.181212987245827e+00 + -2.181041644000000e+00 + -2.180871191888493e+00 + -2.180701687214938e+00 + -2.180533187286951e+00 + -2.180365750436408e+00 + -2.180199435000000e+00 + -2.180034299706266e+00 + -2.179870405073628e+00 + -2.179707812476292e+00 + -2.179546583956951e+00 + -2.179386782000000e+00 + -2.179228469569652e+00 + -2.179071710572415e+00 + -2.178916570209873e+00 + -2.178763115274171e+00 + -2.178611412000000e+00 + -2.178461526719817e+00 + -2.178313527960817e+00 + -2.178167485170301e+00 + -2.178023468394170e+00 + -2.177881548000000e+00 + -2.177741794911888e+00 + -2.177604281585152e+00 + -2.177469081134519e+00 + -2.177336267165003e+00 + -2.177205914000000e+00 + -2.177078096781969e+00 + -2.176952892179310e+00 + -2.176830377266735e+00 + -2.176710629289710e+00 + -2.176593727000000e+00 + -2.176479750089725e+00 + -2.176368778170715e+00 + -2.176260892035270e+00 + -2.176156174290115e+00 + -2.176054707000000e+00 + -2.175956572480315e+00 + -2.175861855847520e+00 + -2.175770642395214e+00 + -2.175683016800304e+00 + -2.175599066000000e+00 + -2.175518878125867e+00 + -2.175442540126260e+00 + -2.175370140365612e+00 + -2.175301769822078e+00 + -2.175237519000000e+00 + -2.175177478422601e+00 + -2.175121740498959e+00 + -2.175070398200953e+00 + -2.175023544771637e+00 + -2.174981275000000e+00 + -2.174943684570890e+00 + -2.174910868831887e+00 + -2.174882924320826e+00 + -2.174859949548785e+00 + -2.174842043000000e+00 + -2.174829303331104e+00 + -2.174821830410927e+00 + -2.174819725228089e+00 + -2.174823090027552e+00 + -2.174832027000000e+00 + -2.174846638699704e+00 + -2.174867029559900e+00 + -2.174893304738549e+00 + -2.174925569834485e+00 + -2.174963931000000e+00 + -2.175008495084293e+00 + -2.175059370394289e+00 + -2.175116665887380e+00 + -2.175180491027200e+00 + -2.175250956000000e+00 + -2.175328171805043e+00 + -2.175412250937077e+00 + -2.175503306357684e+00 + -2.175601451271103e+00 + -2.175706800000000e+00 + -2.175819467765500e+00 + -2.175939570549688e+00 + -2.176067225041086e+00 + -2.176202548787411e+00 + -2.176345660000000e+00 + -2.176496677644949e+00 + -2.176655722183164e+00 + -2.176822914601843e+00 + -2.176998376191533e+00 + -2.177182229000000e+00 + -2.177374595921169e+00 + -2.177575601341799e+00 + -2.177785370087074e+00 + -2.178004027196875e+00 + -2.178231699000000e+00 + -2.178468512862275e+00 + -2.178714597011325e+00 + -2.178970080062299e+00 + -2.179235091002944e+00 + -2.179509760000000e+00 + -2.179794218272507e+00 + -2.180088598277346e+00 + -2.180393032665419e+00 + -2.180707654074028e+00 + -2.181032597000000e+00 + -2.181367997183408e+00 + -2.181713990526695e+00 + -2.182070713375897e+00 + -2.182438302803596e+00 + -2.182816897000000e+00 + -2.183206635150288e+00 + -2.183607657686531e+00 + -2.184020105275964e+00 + -2.184444118610974e+00 + -2.184879840000000e+00 + -2.185327412889597e+00 + -2.185786981109685e+00 + -2.186258689137973e+00 + -2.186742682377725e+00 + -2.187239107000000e+00 + -2.187748109862624e+00 + -2.188269838801850e+00 + -2.188804442677439e+00 + -2.189352071523296e+00 + -2.189912875000000e+00 + -2.190487003093955e+00 + -2.191074608322116e+00 + -2.191675843787203e+00 + -2.192290862610885e+00 + -2.192919819000000e+00 + -2.193562867915730e+00 + -2.194220164552147e+00 + -2.194891865404003e+00 + -2.195578128862665e+00 + -2.196279113000000e+00 + -2.196994976008069e+00 + -2.197725877814248e+00 + -2.198471979076637e+00 + -2.199233441010543e+00 + -2.200010426000000e+00 + -2.200803097224683e+00 + -2.201611618154960e+00 + -2.202436152922896e+00 + -2.203276866629103e+00 + -2.204133924999999e+00 + -2.205007494076377e+00 + -2.205897739759803e+00 + -2.206804828380775e+00 + -2.207728927012060e+00 + -2.208670203000000e+00 + -2.209628824019662e+00 + -2.210604958519273e+00 + -2.211598775212799e+00 + -2.212610442959744e+00 + -2.213640131000000e+00 + -2.214688008997179e+00 + -2.215754247358006e+00 + -2.216839016713686e+00 + -2.217942487809252e+00 + -2.219064832000000e+00 + -2.220206221175054e+00 + -2.221366827800435e+00 + -2.222546824259503e+00 + -2.223746382727115e+00 + -2.224965677000000e+00 + -2.226204881636015e+00 + -2.227464170011377e+00 + -2.228743716360184e+00 + -2.230043696636770e+00 + -2.231364286000000e+00 + -2.232705659360148e+00 + -2.234067993079961e+00 + -2.235451464138931e+00 + -2.236856249856844e+00 + -2.238282527000000e+00 + -2.239730472524265e+00 + -2.241200265638654e+00 + -2.242692085287406e+00 + -2.244206109271373e+00 + -2.245742517000000e+00 + -2.247301488882056e+00 + -2.248883204910052e+00 + -2.250487845315945e+00 + -2.252115590911750e+00 + -2.253766623000000e+00 + -2.255441123360801e+00 + -2.257139274542631e+00 + -2.258861259219423e+00 + -2.260607260005651e+00 + -2.262377460000000e+00 + -2.264172042724291e+00 + -2.265991192137124e+00 + -2.267835092970031e+00 + -2.269703930922391e+00 + -2.271597891000000e+00 + -2.273517158096056e+00 + -2.275461918662951e+00 + -2.277432359442962e+00 + -2.279428667085979e+00 + -2.281451029000000e+00 + -2.283499633104788e+00 + -2.285574667389452e+00 + -2.287676320195930e+00 + -2.289804780398682e+00 + -2.291960237000000e+00 + -2.294142879454985e+00 + -2.296352898805551e+00 + -2.298590485768695e+00 + -2.300855830078550e+00 + -2.303149122999999e+00 + -2.305470556795997e+00 + -2.307820323545887e+00 + -2.310198615647936e+00 + -2.312605626084681e+00 + -2.315041548000000e+00 + -2.317506574745174e+00 + -2.320000900224320e+00 + -2.322524719122656e+00 + -2.325078227061991e+00 + -2.327661619000000e+00 + -2.330275089774933e+00 + -2.332918835677479e+00 + -2.335593053485966e+00 + -2.338297940187764e+00 + -2.341033693000000e+00 + -2.343800509406319e+00 + -2.346598587410773e+00 + -2.349428125303093e+00 + -2.352289321686241e+00 + -2.355182376000000e+00 + -2.358107488272472e+00 + -2.361064858786305e+00 + -2.364054687769198e+00 + -2.367077175375671e+00 + -2.370132523000000e+00 + -2.373220932737141e+00 + -2.376342606269360e+00 + -2.379497745810629e+00 + -2.382686554578887e+00 + -2.385909236000000e+00 + -2.389165993580208e+00 + -2.392457030846979e+00 + -2.395782552198058e+00 + -2.399142763298191e+00 + -2.402537869000000e+00 + -2.405968074028050e+00 + -2.409433584975734e+00 + -2.412934608682329e+00 + -2.416471351683178e+00 + -2.420044020999999e+00 + -2.423652824137149e+00 + -2.427297969206864e+00 + -2.430979664728804e+00 + -2.434698119632543e+00 + -2.438453543000000e+00 + -2.442246144168148e+00 + -2.446076133195298e+00 + -2.449943720571852e+00 + -2.453849117184546e+00 + -2.457792534000000e+00 + -2.461774182279110e+00 + -2.465794274341647e+00 + -2.469853022790889e+00 + -2.473950640299041e+00 + -2.478087340000000e+00 + -2.482263335430129e+00 + -2.486478840559030e+00 + -2.490734069661931e+00 + -2.495029237377275e+00 + -2.499364559000000e+00 + -2.503740250274207e+00 + -2.508156527124766e+00 + -2.512613605692700e+00 + -2.517111702474631e+00 + -2.521651035000000e+00 + -2.526231821279088e+00 + -2.530854278665131e+00 + -2.535518625205626e+00 + -2.540225080261870e+00 + -2.544973863000000e+00 + -2.549765192573217e+00 + -2.554599288888442e+00 + -2.559476372494764e+00 + -2.564396664604125e+00 + -2.569360386000000e+00 + -2.574367757513646e+00 + -2.579419001423863e+00 + -2.584514340338642e+00 + -2.589653996851589e+00 + -2.594838193999999e+00 + -2.600067155253186e+00 + -2.605341104662645e+00 + -2.610660266783872e+00 + -2.616024866689215e+00 + -2.621435129000000e+00 + -2.626891278432673e+00 + -2.632393541366548e+00 + -2.637942144571998e+00 + -2.643537314797249e+00 + -2.649179279000000e+00 + -2.654868264471787e+00 + -2.660604499292826e+00 + -2.666388211776204e+00 + -2.672219630341834e+00 + -2.678098984000000e+00 + -2.684026502306392e+00 + -2.690002415494505e+00 + -2.696026954017429e+00 + -2.702100348331385e+00 + -2.708222828000000e+00 + -2.714394620651795e+00 + -2.720615948464916e+00 + -2.726887030965477e+00 + -2.733208085290139e+00 + -2.739579324000000e+00 + -2.746000955816228e+00 + -2.752473185385865e+00 + -2.758996214641690e+00 + -2.765570242469367e+00 + -2.772195463000000e+00 + -2.778872066776275e+00 + -2.785600241574056e+00 + -2.792380171842594e+00 + -2.799212037606767e+00 + -2.806096015000000e+00 + -2.813032277129447e+00 + -2.820020994035244e+00 + -2.827062331911353e+00 + -2.834156451970738e+00 + -2.841303512999999e+00 + -2.848503671252426e+00 + -2.855757077985244e+00 + -2.863063881095023e+00 + -2.870424224890850e+00 + -2.877838249999999e+00 + -2.885306093928394e+00 + -2.892827890388094e+00 + -2.900403769651381e+00 + -2.908033857857018e+00 + -2.915718277999999e+00 + -2.923457150039019e+00 + -2.931250588977881e+00 + -2.939098706889091e+00 + -2.947001612872150e+00 + -2.954959411999998e+00 + -2.962972205810575e+00 + -2.971040091434060e+00 + -2.979163163083816e+00 + -2.987341511831155e+00 + -2.995575224999998e+00 + -3.003864386499012e+00 + -3.012209075546996e+00 + -3.020609368336417e+00 + -3.029065337835770e+00 + -3.037577052999999e+00 + -3.046144579520789e+00 + -3.054767979666142e+00 + -3.063447312238444e+00 + -3.072182631692820e+00 + -3.080973988999999e+00 + -3.089821432099915e+00 + -3.098725004829577e+00 + -3.107684748048404e+00 + -3.116700699185534e+00 + -3.125772890999999e+00 + -3.134901352715559e+00 + -3.144086110699870e+00 + -3.153327188100508e+00 + -3.162624603843408e+00 + -3.171978372999998e+00 + -3.181388507546933e+00 + -3.190855016073102e+00 + -3.200377903469057e+00 + -3.209957169965227e+00 + -3.219592812999998e+00 + -3.229284827363896e+00 + -3.239033203402719e+00 + -3.248837927724427e+00 + -3.258698982652799e+00 + -3.268616347999998e+00 + -3.278590000914418e+00 + -3.288619913401403e+00 + -3.298706054004673e+00 + -3.308848387625167e+00 + -3.319046875999998e+00 + -3.329301477934322e+00 + -3.339612147844228e+00 + -3.349978836592516e+00 + -3.360401490990458e+00 + -3.370880054999998e+00 + -3.381414469596773e+00 + -3.392004670266620e+00 + -3.402650589976440e+00 + -3.413352159446950e+00 + -3.424109303999998e+00 + -3.434921945057392e+00 + -3.445790001867386e+00 + -3.456713390096515e+00 + -3.467692020464062e+00 + -3.478725800999998e+00 + -3.489814636890736e+00 + -3.500958427789171e+00 + -3.512157070764700e+00 + -3.523410460589800e+00 + -3.534718486999998e+00 + -3.546081035880271e+00 + -3.557497990199610e+00 + -3.568969229634993e+00 + -3.580494629586285e+00 + -3.592074061999998e+00 + -3.603707395759104e+00 + -3.615394495446970e+00 + -3.627135222559569e+00 + -3.638929435115233e+00 + -3.650776987000000e+00 + -3.662677728742245e+00 + -3.674631507325805e+00 + -3.686638166579447e+00 + -3.698697546469576e+00 + -3.710809483000000e+00 + -3.722973808828209e+00 + -3.735190352595157e+00 + -3.747458940125227e+00 + -3.759779394010754e+00 + -3.772151532000000e+00 + -3.784575168342916e+00 + -3.797050115062026e+00 + -3.809576180292756e+00 + -3.822153166803174e+00 + -3.834780875000000e+00 + -3.847459102820189e+00 + -3.860187643203489e+00 + -3.872966285667494e+00 + -3.885794816080159e+00 + -3.898673017000000e+00 + -3.911600667911320e+00 + -3.924577543724543e+00 + -3.937603416269590e+00 + -3.950678053996523e+00 + -3.963801221000000e+00 + -3.976972678103186e+00 + -3.990192183512261e+00 + -4.003459491521890e+00 + -4.016774351164608e+00 + -4.030136509000000e+00 + -4.043545709147031e+00 + -4.057001691149858e+00 + -4.070504190840202e+00 + -4.084052939835804e+00 + -4.097647667000000e+00 + -4.111288098419088e+00 + -4.124973955231095e+00 + -4.138704955371648e+00 + -4.152480813397504e+00 + -4.166301240000000e+00 + -4.180165942572286e+00 + -4.194074624537170e+00 + -4.208026986370140e+00 + -4.222022725101974e+00 + -4.236061533000000e+00 + -4.250143098846027e+00 + -4.264267108996074e+00 + -4.278433246261039e+00 + -4.292641188627767e+00 + -4.306890611000000e+00 + -4.321181185386233e+00 + -4.335512579090955e+00 + -4.349884456391911e+00 + -4.364296478350024e+00 + -4.378748302000000e+00 + -4.393239580901771e+00 + -4.407769964394226e+00 + -4.422339098999587e+00 + -4.436946628138934e+00 + -4.451592191000000e+00 + -4.466275423153566e+00 + -4.480995955945297e+00 + -4.495753418053649e+00 + -4.510547435209278e+00 + -4.525377628000000e+00 + -4.540243613376725e+00 + -4.555145006346341e+00 + -4.570081418154680e+00 + -4.585052454714511e+00 + -4.600057719000000e+00 + -4.615096811360685e+00 + -4.630169328028963e+00 + -4.645274861878902e+00 + -4.660413001844071e+00 + -4.675583333000000e+00 + -4.690785437259527e+00 + -4.706018892955582e+00 + -4.721283275134317e+00 + -4.736578154793736e+00 + -4.751903099000000e+00 + -4.767257671719769e+00 + -4.782641433752779e+00 + -4.798053942304498e+00 + -4.813494749953581e+00 + -4.828963406000000e+00 + -4.844459456896403e+00 + -4.859982445123970e+00 + -4.875531909767719e+00 + -4.891107385884028e+00 + -4.906708405000000e+00 + -4.922334495555203e+00 + -4.937985181840751e+00 + -4.953659984814029e+00 + -4.969358421588884e+00 + -4.985080006000000e+00 + -5.000824248840914e+00 + -5.016590656340991e+00 + -5.032378731598668e+00 + -5.048187974277996e+00 + -5.064017880000000e+00 + -5.079867940973151e+00 + -5.095737645459726e+00 + -5.111626480823102e+00 + -5.127533934923204e+00 + -5.143459499999997e+00 + -5.159402672483111e+00 + -5.175362954369220e+00 + -5.191339851303958e+00 + -5.207332872891304e+00 + -5.223341533999998e+00 + -5.239365353688981e+00 + -5.255403855076862e+00 + -5.271456565155025e+00 + -5.287523015792912e+00 + -5.303602743999997e+00 + -5.319695290809813e+00 + -5.335800201177900e+00 + -5.351917023850607e+00 + -5.368045312413205e+00 + -5.384184625999997e+00 + -5.400334527792659e+00 + -5.416494583936292e+00 + -5.432664365069262e+00 + -5.448843447960337e+00 + -5.465031412999998e+00 + -5.481227843963791e+00 + -5.497432330198948e+00 + -5.513644465003656e+00 + -5.529863846001168e+00 + -5.546090074999998e+00 + -5.562322757794160e+00 + -5.578561506278419e+00 + -5.594805935740466e+00 + -5.611055664858841e+00 + -5.627310317999997e+00 + -5.643569523867812e+00 + -5.659832914603254e+00 + -5.676100126691991e+00 + -5.692370802379017e+00 + -5.708644587999997e+00 + -5.724921133351554e+00 + -5.741200092816319e+00 + -5.757481124931912e+00 + -5.773763893288205e+00 + -5.790048065999996e+00 + -5.806333315035348e+00 + -5.822619317227983e+00 + -5.838905753168464e+00 + -5.855192307810226e+00 + -5.871478670999998e+00 + -5.887764536642293e+00 + -5.904049603224381e+00 + -5.920333573291820e+00 + -5.936616154283948e+00 + -5.952897057999997e+00 + -5.969175999997380e+00 + -5.985452700712029e+00 + -6.001726884408804e+00 + -6.017998279848357e+00 + -6.034266620999998e+00 + -6.050531645917840e+00 + -6.066793096565948e+00 + -6.083050719154321e+00 + -6.099304265265417e+00 + -6.115553489999997e+00 + -6.131798152082848e+00 + -6.148038016850530e+00 + -6.164272852974367e+00 + -6.180502432221914e+00 + -6.196726531999996e+00 + -6.212944934149927e+00 + -6.229157424398349e+00 + -6.245363792906498e+00 + -6.261563835474968e+00 + -6.277757350999996e+00 + -6.293944141712862e+00 + -6.310124016459128e+00 + -6.326296787703612e+00 + -6.342462271423720e+00 + -6.358620288999997e+00 + -6.374770666045148e+00 + -6.390913231950829e+00 + -6.407047820441565e+00 + -6.423174270811367e+00 + -6.439292425999997e+00 + -6.455402132459349e+00 + -6.471503242529590e+00 + -6.487595612302881e+00 + -6.503679101824962e+00 + -6.519753575999998e+00 + -6.535818903908587e+00 + -6.551874959678351e+00 + -6.567921621040533e+00 + -6.583958769810407e+00 + -6.599986292999997e+00 + -6.616004081814032e+00 + -6.632012031732002e+00 + -6.648010042213294e+00 + -6.663998017633650e+00 + -6.679975866999997e+00 + -6.695943503153600e+00 + -6.711900843425355e+00 + -6.727847809100131e+00 + -6.743784326279565e+00 + -6.759710325999998e+00 + -6.775625743222601e+00 + -6.791530516957345e+00 + -6.807424590371784e+00 + -6.823307911868753e+00 + -6.839180434000000e+00 + -6.855042113095231e+00 + -6.870892911056689e+00 + -6.886732793343159e+00 + -6.902561729209182e+00 + -6.918379693000000e+00 + -6.934186663377454e+00 + -6.949982623974822e+00 + -6.965767562024019e+00 + -6.981541468861679e+00 + -6.997304341000000e+00 + -7.013056179149240e+00 + -7.028796988372566e+00 + -7.044526777567921e+00 + -7.060245560311449e+00 + -7.075953355000000e+00 + -7.091650184045992e+00 + -7.107336074531737e+00 + -7.123011057484204e+00 + -7.138675168614964e+00 + -7.154328448000000e+00 + -7.169970939523060e+00 + -7.185602692089082e+00 + -7.201223758621499e+00 + -7.216834196750369e+00 + -7.232434069000000e+00 + -7.248023441650831e+00 + -7.263602384451917e+00 + -7.279170972021925e+00 + -7.294729285389790e+00 + -7.310277408000000e+00 + -7.325815426345584e+00 + -7.341343434315181e+00 + -7.356861529294433e+00 + -7.372369811668458e+00 + -7.387868387000000e+00 + -7.403357365322682e+00 + -7.418836861854650e+00 + -7.434306995371617e+00 + -7.449767888578398e+00 + -7.465219669000000e+00 + -7.480662468362842e+00 + -7.496096423635060e+00 + -7.511521675509539e+00 + -7.526938368822738e+00 + -7.542346653000000e+00 + -7.557746681513402e+00 + -7.573138613106206e+00 + -7.588522610348394e+00 + -7.603898840111169e+00 + -7.619267474000000e+00 + -7.634628687556104e+00 + -7.649982660818226e+00 + -7.665329578157865e+00 + -7.680669629212444e+00 + -7.696003007000000e+00 + -7.711329908041110e+00 + -7.726650535322031e+00 + -7.741965095546557e+00 + -7.757273799081691e+00 + -7.772576861000000e+00 + -7.787874500482072e+00 + -7.803166941820082e+00 + -7.818454413357075e+00 + -7.833737148083599e+00 + -7.849015383000000e+00 + -7.864289358883056e+00 + -7.879559322305754e+00 + -7.894825523493435e+00 + -7.910088216491122e+00 + -7.925347660000000e+00 + -7.940604117030949e+00 + -7.955857856656654e+00 + -7.971109151088187e+00 + -7.986358275553978e+00 + -8.001605510999999e+00 + -8.016851143003514e+00 + -8.032095461580266e+00 + -8.047338760488806e+00 + -8.062581337981985e+00 + -8.077823497000001e+00 + -8.093065544394179e+00 + -8.108307791577566e+00 + -8.123550554205780e+00 + -8.138794153086785e+00 + -8.154038912999997e+00 + -8.169285162395173e+00 + -8.184533235306290e+00 + -8.199783469410583e+00 + -8.215036206329255e+00 + -8.230291792999997e+00 + -8.245550580528674e+00 + -8.260812923863309e+00 + -8.276079182235739e+00 + -8.291349720375887e+00 + -8.306624906999998e+00 + -8.321905114401627e+00 + -8.337190720129563e+00 + -8.352482105562956e+00 + -8.367779656388556e+00 + -8.383083762999997e+00 + -8.398394819954635e+00 + -8.413713227271362e+00 + -8.429039388338369e+00 + -8.444373710123081e+00 + -8.459716604999997e+00 + -8.475068489773328e+00 + -8.490429785807564e+00 + -8.505800918193746e+00 + -8.521182316459129e+00 + -8.536574414999997e+00 + -8.551977652264261e+00 + -8.567392471335332e+00 + -8.582819319266729e+00 + -8.598258647834163e+00 + -8.613710912999998e+00 + -8.629176574613023e+00 + -8.644656098315155e+00 + -8.660149953296175e+00 + -8.675658612460374e+00 + -8.691182553999997e+00 + -8.706722260346437e+00 + -8.722278218076596e+00 + -8.737850917931995e+00 + -8.753440855866357e+00 + -8.769048531999996e+00 + -8.784674450012169e+00 + -8.800319118245726e+00 + -8.815983049456063e+00 + -8.831666761752983e+00 + -8.847370776999997e+00 + -8.863095620572640e+00 + -8.878841823409489e+00 + -8.894609920288579e+00 + -8.910400450202564e+00 + -8.926213956999996e+00 + -8.942050988642462e+00 + -8.957912097948702e+00 + -8.973797841249739e+00 + -8.989708778929735e+00 + -9.005645476999996e+00 + -9.021608505891232e+00 + -9.037598440089864e+00 + -9.053615857732957e+00 + -9.069661341483132e+00 + -9.085735478999997e+00 + -9.101838862125330e+00 + -9.117972087538210e+00 + -9.134135755396374e+00 + -9.150330469883286e+00 + -9.166556840999997e+00 + -9.182815483079517e+00 + -9.199107013675079e+00 + -9.215432054583626e+00 + -9.231791233265957e+00 + -9.248185180999997e+00 + -9.264614532683632e+00 + -9.281079929109534e+00 + -9.297582014702897e+00 + -9.314121437654787e+00 + -9.330698850999996e+00 + -9.347314912003528e+00 + -9.363970283188971e+00 + -9.380665630745685e+00 + -9.397401624940878e+00 + -9.414178940999998e+00 + -9.430998258244742e+00 + -9.447860260503859e+00 + -9.464765635761319e+00 + -9.481715077019253e+00 + -9.498709280999996e+00 + -9.515748948037771e+00 + -9.532834784442173e+00 + -9.549967500513571e+00 + -9.567147810785697e+00 + -9.584376433999996e+00 + -9.601654092679016e+00 + -9.618981514566920e+00 + -9.636359431588877e+00 + -9.653788580476306e+00 + -9.671269701999998e+00 + -9.688803540653204e+00 + -9.706390846469759e+00 + -9.724032373152683e+00 + -9.741728878390186e+00 + -9.759481124999997e+00 + -9.777289880050315e+00 + -9.795155915261526e+00 + -9.813080006095444e+00 + -9.831062932433847e+00 + -9.849105478999997e+00 + -9.867208434586416e+00 + -9.885372592754736e+00 + -9.903598750780114e+00 + -9.921887710289466e+00 + -9.940240277999996e+00 + -9.958657264807959e+00 + -9.977139486156000e+00 + -9.995687760994512e+00 + -1.001430291248964e+01 + -1.003298577000000e+01 + -1.005173716741557e+01 + -1.007055794170109e+01 + -1.008944893313717e+01 + -1.010841098632109e+01 + -1.012744495000000e+01 + -1.014655167843811e+01 + -1.016573203809090e+01 + -1.018498689280856e+01 + -1.020431709913334e+01 + -1.022372353000000e+01 + -1.024320706932260e+01 + -1.026276860085199e+01 + -1.028240900753776e+01 + -1.030212917226508e+01 + -1.032192999000000e+01 + -1.034181236270672e+01 + -1.036177718920571e+01 + -1.038182537682789e+01 + -1.040195784652257e+01 + -1.042217551000000e+01 + -1.044247927800797e+01 + -1.046287008429113e+01 + -1.048334886197221e+01 + -1.050391653528098e+01 + -1.052457404000000e+01 + -1.054532232135862e+01 + -1.056616233045517e+01 + -1.058709501821704e+01 + -1.060812133414642e+01 + -1.062924224000000e+01 + -1.065045870478386e+01 + -1.067177169452023e+01 + -1.069318217826154e+01 + -1.071469113174534e+01 + -1.073629954000000e+01 + -1.075800839260036e+01 + -1.077981867489276e+01 + -1.080173138018267e+01 + -1.082374751517595e+01 + -1.084586808000000e+01 + -1.086809407471729e+01 + -1.089042651905600e+01 + -1.091286643223940e+01 + -1.093541482646511e+01 + -1.095807273000000e+01 + -1.098084118002027e+01 + -1.100372120627546e+01 + -1.102671384288809e+01 + -1.104982013415814e+01 + -1.107304113000000e+01 + -1.109637788366843e+01 + -1.111983144958023e+01 + -1.114340288661973e+01 + -1.116709326043168e+01 + -1.119090364000000e+01 + -1.121483509746141e+01 + -1.123888871051257e+01 + -1.126306556352589e+01 + -1.128736674843474e+01 + -1.131179335000000e+01 + -1.133634645465832e+01 + -1.136102717560643e+01 + -1.138583662099362e+01 + -1.141077588304154e+01 + -1.143584608000000e+01 + -1.146104834278586e+01 + -1.148638378227322e+01 + -1.151185351768562e+01 + -1.153745868860526e+01 + -1.156320043000000e+01 + -1.158907987623694e+01 + -1.161509817570602e+01 + -1.164125647703402e+01 + -1.166755592508495e+01 + -1.169399768000000e+01 + -1.172058291041322e+01 + -1.174731277861368e+01 + -1.177418844974534e+01 + -1.180121109657743e+01 + -1.182838189999999e+01 + -1.185570204579555e+01 + -1.188317272006268e+01 + -1.191079511332836e+01 + -1.193857042312863e+01 + -1.196649984999999e+01 + -1.199458459773775e+01 + -1.202282587700695e+01 + -1.205122490269986e+01 + -1.207978289369385e+01 + -1.210850107000000e+01 + -1.213738065440104e+01 + -1.216642287877588e+01 + -1.219562898067898e+01 + -1.222500020303083e+01 + -1.225453779000000e+01 + -1.228424298812801e+01 + -1.231411705144751e+01 + -1.234416123799292e+01 + -1.237437680934280e+01 + -1.240476502999999e+01 + -1.243532716862228e+01 + -1.246606450371358e+01 + -1.249697831672881e+01 + -1.252806989018327e+01 + -1.255934050999999e+01 + -1.259079146663310e+01 + -1.262242406008652e+01 + -1.265423959346900e+01 + -1.268623937120698e+01 + -1.271842469999999e+01 + -1.275079689157634e+01 + -1.278335727210730e+01 + -1.281610716656313e+01 + -1.284904789393346e+01 + -1.288218078999999e+01 + -1.291550719883721e+01 + -1.294902845305343e+01 + -1.298274589252790e+01 + -1.301666087289315e+01 + -1.305077474999999e+01 + -1.308508888000826e+01 + -1.311960462335401e+01 + -1.315432334734565e+01 + -1.318924642810263e+01 + -1.322437523999999e+01 + -1.325971115825294e+01 + -1.329525556797775e+01 + -1.333100986045822e+01 + -1.336697543256989e+01 + -1.340315367999999e+01 + -1.343954599979837e+01 + -1.347615379884262e+01 + -1.351297848978655e+01 + -1.355002149050285e+01 + -1.358728421999999e+01 + -1.362476809975143e+01 + -1.366247455932581e+01 + -1.370040503112116e+01 + -1.373856094968169e+01 + -1.377694375999999e+01 + -1.381555491216690e+01 + -1.385439585021465e+01 + -1.389346802391279e+01 + -1.393277289476898e+01 + -1.397231192999999e+01 + -1.401208659972945e+01 + -1.405209837359219e+01 + -1.409234872413346e+01 + -1.413283912961124e+01 + -1.417357107999999e+01 + -1.421454607102127e+01 + -1.425576559243092e+01 + -1.429723113890712e+01 + -1.433894421536596e+01 + -1.438090632999999e+01 + -1.442311899332400e+01 + -1.446558371903957e+01 + -1.450830202765583e+01 + -1.455127544879836e+01 + -1.459450550999999e+01 + -1.463799373973040e+01 + -1.468174167767078e+01 + -1.472575086722947e+01 + -1.477002285383887e+01 + -1.481455918999999e+01 + -1.485936143285468e+01 + -1.490443114031971e+01 + -1.494976987417127e+01 + -1.499537920245961e+01 + -1.504126069999999e+01 + -1.508741594534576e+01 + -1.513384651602358e+01 + -1.518055399754435e+01 + -1.522753998778076e+01 + -1.527480607999999e+01 + -1.532235386668537e+01 + -1.537018495189360e+01 + -1.541830094448208e+01 + -1.546670345694024e+01 + -1.551539410999999e+01 + -1.556437452805486e+01 + -1.561364632974880e+01 + -1.566321114034704e+01 + -1.571307059764629e+01 + -1.576322633999999e+01 + -1.581368000886133e+01 + -1.586443325932826e+01 + -1.591548774371123e+01 + -1.596684510633198e+01 + -1.601850700999999e+01 + -1.607047512763592e+01 + -1.612275112310131e+01 + -1.617533666634843e+01 + -1.622823344044972e+01 + -1.628144312999999e+01 + -1.633496741991010e+01 + -1.638880799548294e+01 + -1.644296655089035e+01 + -1.649744479372194e+01 + -1.655224442999999e+01 + -1.660736716535794e+01 + -1.666281471098534e+01 + -1.671858878335293e+01 + -1.677469110504862e+01 + -1.683112339999999e+01 + -1.688788739643611e+01 + -1.694498483759857e+01 + -1.700241746523662e+01 + -1.706018701398879e+01 + -1.711829522999999e+01 + -1.717674386844329e+01 + -1.723553468924849e+01 + -1.729466945458232e+01 + -1.735414992865347e+01 + -1.741397788000000e+01 + -1.747415508099178e+01 + -1.753468330867943e+01 + -1.759556434768274e+01 + -1.765679999203036e+01 + -1.771839203000000e+01 + -1.778034225038387e+01 + -1.784265246094084e+01 + -1.790532446959567e+01 + -1.796836007816754e+01 + -1.803176110000000e+01 + -1.809552935720682e+01 + -1.815966667575656e+01 + -1.822417488078262e+01 + -1.828905579597735e+01 + -1.835431126000000e+01 + -1.841994311958100e+01 + -1.848595321475806e+01 + -1.855234338946741e+01 + -1.861911549709020e+01 + -1.868627140000000e+01 + -1.875381296458075e+01 + -1.882174205218422e+01 + -1.889006052915166e+01 + -1.895877027181235e+01 + -1.902787316000000e+01 + -1.909737107797965e+01 + -1.916726592078253e+01 + -1.923755957952593e+01 + -1.930825393714747e+01 + -1.937935090000000e+01 + -1.945085238647674e+01 + -1.952276030080435e+01 + -1.959507655061933e+01 + -1.966780305499929e+01 + -1.974094174000000e+01 + -1.981449453618971e+01 + -1.988846337667131e+01 + -1.996285019538828e+01 + -2.003765692760933e+01 + -2.011288552000000e+01 + -2.018853792621815e+01 + -2.026461609876727e+01 + -2.034112199510926e+01 + -2.041805758075517e+01 + -2.049542482000000e+01 + -2.057322568006621e+01 + -2.065146214471047e+01 + -2.073013619484967e+01 + -2.080924980240138e+01 + -2.088880495999999e+01 + -2.096880367169853e+01 + -2.104924793164823e+01 + -2.113013973534757e+01 + -2.121148108557274e+01 + -2.129327399999999e+01 + -2.137552050224973e+01 + -2.145822260294134e+01 + -2.154138232011573e+01 + -2.162500168821614e+01 + -2.170908273999999e+01 + -2.179362750931509e+01 + -2.187863804236205e+01 + -2.196411638687395e+01 + -2.205006458870765e+01 + -2.213648469999998e+01 + -2.222337877888525e+01 + -2.231074889087196e+01 + -2.239859710699802e+01 + -2.248692550346576e+01 + -2.257573614999999e+01 + -2.266503111825121e+01 + -2.275481250533621e+01 + -2.284508240538831e+01 + -2.293584289979252e+01 + -2.302709608999998e+01 + -2.311884408797131e+01 + -2.321108899308483e+01 + -2.330383291176923e+01 + -2.339707796636268e+01 + -2.349082627999998e+01 + -2.358507997580010e+01 + -2.367984117841246e+01 + -2.377511201965127e+01 + -2.387089464198777e+01 + -2.396719118999998e+01 + -2.406400380890314e+01 + -2.416133464354705e+01 + -2.425918584868402e+01 + -2.435755959379285e+01 + -2.445645803999998e+01 + -2.455588334675619e+01 + -2.465583769169985e+01 + -2.475632325621871e+01 + -2.485734222078360e+01 + -2.495889676999998e+01 + -2.506098909313465e+01 + -2.516362138717989e+01 + -2.526679585042222e+01 + -2.537051468121954e+01 + -2.547478008999998e+01 + -2.557959429416147e+01 + -2.568495950782481e+01 + -2.579087794894506e+01 + -2.589735184307986e+01 + -2.600438342000000e+01 + -2.611197491414786e+01 + -2.622012856937007e+01 + -2.632884662779481e+01 + -2.643813132670684e+01 + -2.654798492000000e+01 + -2.665840967182968e+01 + -2.676940784333127e+01 + -2.688098169572292e+01 + -2.699313349264397e+01 + -2.710586551000000e+01 + -2.721918002998715e+01 + -2.733307932892397e+01 + -2.744756569229334e+01 + -2.756264142140602e+01 + -2.767830881000000e+01 + -2.779457014947893e+01 + -2.791142774510221e+01 + -2.802888390934165e+01 + -2.814694096000261e+01 + -2.826560121000000e+01 + -2.838486697334714e+01 + -2.850474058224100e+01 + -2.862522436986088e+01 + -2.874632066501767e+01 + -2.886803181000000e+01 + -2.899036015446983e+01 + -2.911330804180568e+01 + -2.923687781925054e+01 + -2.936107184331833e+01 + -2.948589248000000e+01 + -2.961134209910011e+01 + -2.973742306338333e+01 + -2.986413774439124e+01 + -2.999148852946843e+01 + -3.011947780000000e+01 + -3.024810793640462e+01 + -3.037738133418102e+01 + -3.050730039248281e+01 + -3.063786751048302e+01 + -3.076908508999995e+01 + -3.090095553801825e+01 + -3.103348127528379e+01 + -3.116666472242296e+01 + -3.130050829541340e+01 + -3.143501442000000e+01 + -3.157018552904646e+01 + -3.170602405762165e+01 + -3.184253244456109e+01 + -3.197971313442012e+01 + -3.211756857999995e+01 + -3.225610123724459e+01 + -3.239531355479052e+01 + -3.253520799120049e+01 + -3.267578702273956e+01 + -3.281705312000000e+01 + -3.295900875199519e+01 + -3.310165640007095e+01 + -3.324499854981137e+01 + -3.338903768896250e+01 + -3.353377630999994e+01 + -3.367921690900354e+01 + -3.382536198480008e+01 + -3.397221404252836e+01 + -3.411977559608952e+01 + -3.426804916000000e+01 + -3.441703725062087e+01 + -3.456674239208418e+01 + -3.471716711255165e+01 + -3.486831394366235e+01 + -3.502018541999994e+01 + -3.517278408030963e+01 + -3.532611247314257e+01 + -3.548017315006599e+01 + -3.563496866378715e+01 + -3.579050157000000e+01 + -3.594677442911821e+01 + -3.610378981290209e+01 + -3.626155029281482e+01 + -3.642005843692588e+01 + -3.657931682999995e+01 + -3.673932806450152e+01 + -3.690009471994529e+01 + -3.706161938636306e+01 + -3.722390467398576e+01 + -3.738695318000000e+01 + -3.755076749872251e+01 + -3.771535025114206e+01 + -3.788070405790332e+01 + -3.804683152981476e+01 + -3.821373528999994e+01 + -3.838141797088579e+01 + -3.854988220802454e+01 + -3.871913063690455e+01 + -3.888916589260551e+01 + -3.905999062000000e+01 + -3.923160747217621e+01 + -3.940401910958111e+01 + -3.957722819105772e+01 + -3.975123737141870e+01 + -3.992604931999993e+01 + -4.010166671506960e+01 + -4.027809223215413e+01 + -4.045532854769397e+01 + -4.063337834194039e+01 + -4.081224431000000e+01 + -4.099192915380458e+01 + -4.117243556512365e+01 + -4.135376624213356e+01 + -4.153592389680070e+01 + -4.171891123999993e+01 + -4.190273098404973e+01 + -4.208738585293229e+01 + -4.227287857304115e+01 + -4.245921187066173e+01 + -4.264638848000000e+01 + -4.283441114000928e+01 + -4.302328258823673e+01 + -4.321300556988346e+01 + -4.340358284230594e+01 + -4.359501715999993e+01 + -4.378731127802528e+01 + -4.398046796379431e+01 + -4.417448998677844e+01 + -4.436938011574802e+01 + -4.456514113000000e+01 + -4.476177581459606e+01 + -4.495928695037842e+01 + -4.515767732491430e+01 + -4.535694973791337e+01 + -4.555710698999992e+01 + -4.575815188265746e+01 + -4.596008722124420e+01 + -4.616291581815827e+01 + -4.636664049491562e+01 + -4.657126407000000e+01 + -4.677678936329625e+01 + -4.698321921012249e+01 + -4.719055644645270e+01 + -4.739880390447049e+01 + -4.760796442999992e+01 + -4.781804087601476e+01 + -4.802903608806237e+01 + -4.824095291782189e+01 + -4.845379422957746e+01 + -4.866756289000000e+01 + -4.888226176793614e+01 + -4.909789373759357e+01 + -4.931446167624577e+01 + -4.953196846445736e+01 + -4.975041698999991e+01 + -4.996981014490353e+01 + -5.019015082042799e+01 + -5.041144191483806e+01 + -5.063368633767777e+01 + -5.085688700000000e+01 + -5.108104681363623e+01 + -5.130616869222998e+01 + -5.153225555529192e+01 + -5.175931033093434e+01 + -5.198733594999992e+01 + -5.221633534632925e+01 + -5.244631146064819e+01 + -5.267726723814258e+01 + -5.290920562821528e+01 + -5.314212958000000e+01 + -5.337604204603855e+01 + -5.361094599406351e+01 + -5.384684439089011e+01 + -5.408374019738633e+01 + -5.432163638999991e+01 + -5.456053595406303e+01 + -5.480044186892415e+01 + -5.504135712069162e+01 + -5.528328470761400e+01 + -5.552622762000000e+01 + -5.577018884907901e+01 + -5.601517141284037e+01 + -5.626117832498171e+01 + -5.650821258404100e+01 + -5.675627720999991e+01 + -5.700537523542541e+01 + -5.725550968441112e+01 + -5.750668358361924e+01 + -5.775889996777224e+01 + -5.801216188000000e+01 + -5.826647236807500e+01 + -5.852183447842080e+01 + -5.877825126017432e+01 + -5.903572576827786e+01 + -5.929426106999990e+01 + -5.955386023803142e+01 + -5.981452633626228e+01 + -6.007626243395679e+01 + -6.033907161265666e+01 + -6.060295696000000e+01 + -6.086792156620618e+01 + -6.113396851889991e+01 + -6.140110091270849e+01 + -6.166932185413050e+01 + -6.193863444999990e+01 + -6.220904180836374e+01 + -6.248054704389698e+01 + -6.275315327303672e+01 + -6.302686361375484e+01 + -6.330168120000000e+01 + -6.357760917255472e+01 + -6.385465065869710e+01 + -6.413280879313753e+01 + -6.441208672717147e+01 + -6.469248761000000e+01 + -6.497401459207148e+01 + -6.525667083785252e+01 + -6.554045951020694e+01 + -6.582538376572784e+01 + -6.611144677999999e+01 + -6.639865173804421e+01 + -6.668700181225154e+01 + -6.697650018322166e+01 + -6.726715004865791e+01 + -6.755895460000001e+01 + -6.785191702754956e+01 + -6.814604053706709e+01 + -6.844132833746227e+01 + -6.873778363702108e+01 + -6.903540965000001e+01 + -6.933420959649558e+01 + -6.963418670433917e+01 + -6.993534420146086e+01 + -7.023768531400407e+01 + -7.054121327999999e+01 + -7.084593134500037e+01 + -7.115184275353549e+01 + -7.145895075432828e+01 + -7.176725860332733e+01 + -7.207676955999989e+01 + -7.238748688676215e+01 + -7.269941385037480e+01 + -7.301255372170849e+01 + -7.332690977697268e+01 + -7.364248530000000e+01 + -7.395928357816831e+01 + -7.427730789465394e+01 + -7.459656154154558e+01 + -7.491704782593285e+01 + -7.523877004999987e+01 + -7.556173151464178e+01 + -7.588593553169946e+01 + -7.621138541944168e+01 + -7.653808450163682e+01 + -7.686603610000000e+01 + -7.719524353805981e+01 + -7.752571015313293e+01 + -7.785743928572181e+01 + -7.819043427631229e+01 + -7.852469846999988e+01 + -7.886023521608357e+01 + -7.919704786882774e+01 + -7.953513978918475e+01 + -7.987451434652237e+01 + -8.021517491000000e+01 + -8.055712484895521e+01 + -8.090036753598372e+01 + -8.124490635250547e+01 + -8.159074469172391e+01 + -8.193788593999987e+01 + -8.228633348416081e+01 + -8.263609073313938e+01 + -8.298716109541151e+01 + -8.333954797103145e+01 + -8.369325477000000e+01 + -8.404828491028280e+01 + -8.440464181412719e+01 + -8.476232890793086e+01 + -8.512134962287996e+01 + -8.548170738999985e+01 + -8.584340564352934e+01 + -8.620644783199920e+01 + -8.657083740482393e+01 + -8.693657780824788e+01 + -8.730367250000000e+01 + -8.767212494371194e+01 + -8.804193859639375e+01 + -8.841311692593891e+01 + -8.878566341870939e+01 + -8.915958154999986e+01 + -8.953487479168578e+01 + -8.991154663471849e+01 + -9.028960057395298e+01 + -9.066904010318096e+01 + -9.104986872000001e+01 + -9.143208992686800e+01 + -9.181570723560402e+01 + -9.220072416114067e+01 + -9.258714421980083e+01 + -9.297497092999986e+01 + -9.336420781525349e+01 + -9.375485841426487e+01 + -9.414692626313253e+01 + -9.454041488965315e+01 + -9.493532784000000e+01 + -9.533166867059026e+01 + -9.572944092950100e+01 + -9.612864816948191e+01 + -9.652929395407540e+01 + -9.693138184999984e+01 + -9.733491542620831e+01 + -9.773989825487583e+01 + -9.814633391500892e+01 + -9.855422599474231e+01 + -9.896357808000000e+01 + -9.937439375553778e+01 + -9.978667660951464e+01 + -1.002004302498873e+02 + -1.006156583113964e+02 + -1.010323643999998e+02 + -1.014505521033422e+02 + -1.018702250167552e+02 + -1.022913867648504e+02 + -1.027140410123223e+02 + -1.031381914000000e+02 + -1.035638415489902e+02 + -1.039909950728156e+02 + -1.044196556182856e+02 + -1.048498268814360e+02 + -1.052815124999998e+02 + -1.057147160851634e+02 + -1.061494413040264e+02 + -1.065856918490972e+02 + -1.070234714266025e+02 + -1.074627837000000e+02 + -1.079036323073108e+02 + -1.083460208994117e+02 + -1.087899531790557e+02 + -1.092354329168266e+02 + -1.096824637999998e+02 + -1.101310494754678e+02 + -1.105811936593542e+02 + -1.110329000832090e+02 + -1.114861724712110e+02 + -1.119410145000000e+02 + -1.123974298499644e+02 + -1.128554223365672e+02 + -1.133149957276709e+02 + -1.137761536721920e+02 + -1.142388998999998e+02 + -1.147032381951767e+02 + -1.151691723223398e+02 + -1.156367060267053e+02 + -1.161058430340008e+02 + -1.165765871000000e+02 + -1.170489420136372e+02 + -1.175229116129598e+02 + -1.179984996644491e+02 + -1.184757098197581e+02 + -1.189545458999998e+02 + -1.194350118004852e+02 + -1.199171112501401e+02 + -1.204008480035419e+02 + -1.208862259161727e+02 + -1.213732488000000e+02 + -1.218619204419225e+02 + -1.223522446637547e+02 + -1.228442252993658e+02 + -1.233378661832514e+02 + -1.238331710999998e+02 + -1.243301438371392e+02 + -1.248287883218059e+02 + -1.253291084262349e+02 + -1.258311078947497e+02 + -1.263347906000000e+02 + -1.268401604692280e+02 + -1.273472212887372e+02 + -1.278559768940106e+02 + -1.283664312433523e+02 + -1.288785881999998e+02 + -1.293924515790011e+02 + -1.299080252758494e+02 + -1.304253131892150e+02 + -1.309443191941241e+02 + -1.314650472000000e+02 + -1.319875011267033e+02 + -1.325116848422910e+02 + -1.330376022476014e+02 + -1.335652573069860e+02 + -1.340946538999998e+02 + -1.346257958892211e+02 + -1.351586873010075e+02 + -1.356933320835031e+02 + -1.362297340185993e+02 + -1.367678971000000e+02 + -1.373078254089903e+02 + -1.378495227933144e+02 + -1.383929931342302e+02 + -1.389382404506380e+02 + -1.394852686999998e+02 + -1.400340818144768e+02 + -1.405846838138316e+02 + -1.411370786853171e+02 + -1.416912703410207e+02 + -1.422472628000000e+02 + -1.428050601118040e+02 + -1.433646661562381e+02 + -1.439260848884736e+02 + -1.444893204354731e+02 + -1.450543767999998e+02 + -1.456212579173106e+02 + -1.461899678121767e+02 + -1.467605105156440e+02 + -1.473328900351724e+02 + -1.479071104000000e+02 + -1.484831756560318e+02 + -1.490610898537698e+02 + -1.496408570188282e+02 + -1.502224811435522e+02 + -1.508059663000000e+02 + -1.513913165938295e+02 + -1.519785360517816e+02 + -1.525676287092765e+02 + -1.531585986433134e+02 + -1.537514498999998e+02 + -1.543461865304275e+02 + -1.549428126935839e+02 + -1.555413324987964e+02 + -1.561417499433451e+02 + -1.567440691000000e+02 + -1.573482940967384e+02 + -1.579544290618548e+02 + -1.585624781173004e+02 + -1.591724453723425e+02 + -1.597843348999998e+02 + -1.603981507727703e+02 + -1.610138971515657e+02 + -1.616315781879854e+02 + -1.622511979884414e+02 + -1.628727607000000e+02 + -1.634962704824068e+02 + -1.641217314319286e+02 + -1.647491476962519e+02 + -1.653785235159056e+02 + -1.660098629999998e+02 + -1.666431702132707e+02 + -1.672784494025457e+02 + -1.679157047805161e+02 + -1.685549404452619e+02 + -1.691961606000000e+02 + -1.698393694891200e+02 + -1.704845712252935e+02 + -1.711317699673001e+02 + -1.717809699904770e+02 + -1.724321754999998e+02 + -1.730853906636835e+02 + -1.737406197069920e+02 + -1.743978668390772e+02 + -1.750571362267288e+02 + -1.757184321000000e+02 + -1.763817587329360e+02 + -1.770471204015179e+02 + -1.777145213342737e+02 + -1.783839656916257e+02 + -1.790554576999997e+02 + -1.797290016348361e+02 + -1.804046017812607e+02 + -1.810822624074922e+02 + -1.817619877551305e+02 + -1.824437821000000e+02 + -1.831276497291853e+02 + -1.838135948809234e+02 + -1.845016218201234e+02 + -1.851917348679873e+02 + -1.858839382999998e+02 + -1.865782363787204e+02 + -1.872746334451072e+02 + -1.879731338116912e+02 + -1.886737417231232e+02 + -1.893764615000000e+02 + -1.900812975069574e+02 + -1.907882540726982e+02 + -1.914973354967214e+02 + -1.922085460544087e+02 + -1.929218900999998e+02 + -1.936373720194894e+02 + -1.943549961169706e+02 + -1.950747667123967e+02 + -1.957966881820183e+02 + -1.965207649000000e+02 + -1.972470012275908e+02 + -1.979754014928308e+02 + -1.987059700466389e+02 + -1.994387112870401e+02 + -2.001736295999997e+02 + -2.009107293657797e+02 + -2.016500149830146e+02 + -2.023914908231823e+02 + -2.031351612174878e+02 + -2.038810306000000e+02 + -2.046291034502658e+02 + -2.053793841544099e+02 + -2.061318770718700e+02 + -2.068865865659776e+02 + -2.076435170999997e+02 + -2.084026731731890e+02 + -2.091640591716557e+02 + -2.099276794995817e+02 + -2.106935386300690e+02 + -2.114616410000000e+02 + -2.122319910255027e+02 + -2.130045931490896e+02 + -2.137794518362718e+02 + -2.145565715736208e+02 + -2.153359567999997e+02 + -2.161176119510612e+02 + -2.169015415764713e+02 + -2.176877501734888e+02 + -2.184762421224458e+02 + -2.192670219000000e+02 + -2.200600940505141e+02 + -2.208554631128035e+02 + -2.216531335679009e+02 + -2.224531098207831e+02 + -2.232553963999997e+02 + -2.240599978997985e+02 + -2.248669188401317e+02 + -2.256761637092654e+02 + -2.264877369816284e+02 + -2.273016432000000e+02 + -2.281178869415660e+02 + -2.289364727411434e+02 + -2.297574051254879e+02 + -2.305806886273285e+02 + -2.314063277999997e+02 + -2.322343272067604e+02 + -2.330646913987978e+02 + -2.338974249389632e+02 + -2.347325324124834e+02 + -2.355700184000000e+02 + -2.364098874788788e+02 + -2.372521442301691e+02 + -2.380967932316097e+02 + -2.389438390563173e+02 + -2.397932862999997e+02 + -2.406451395699314e+02 + -2.414994034603918e+02 + -2.423560825773335e+02 + -2.432151815494090e+02 + -2.440767050000000e+02 + -2.449406575494712e+02 + -2.458070438246558e+02 + -2.466758684438366e+02 + -2.475471360143982e+02 + -2.484208511999996e+02 + -2.492970186823550e+02 + -2.501756430678903e+02 + -2.510567289807632e+02 + -2.519402811010791e+02 + -2.528263041000000e+02 + -2.537148026392507e+02 + -2.546057813786489e+02 + -2.554992449908618e+02 + -2.563951981686694e+02 + -2.572936455999996e+02 + -2.581945919701986e+02 + -2.590980419711499e+02 + -2.600040002856254e+02 + -2.609124715849065e+02 + -2.618234606000000e+02 + -2.627369720784054e+02 + -2.636530106775519e+02 + -2.645715811006028e+02 + -2.654926881465747e+02 + -2.664163364999997e+02 + -2.673425308135112e+02 + -2.682712759281322e+02 + -2.692025766270102e+02 + -2.701364375396030e+02 + -2.710728634000000e+02 + -2.720118590149941e+02 + -2.729534291763713e+02 + -2.738975786300543e+02 + -2.748443120676544e+02 + -2.757936342999997e+02 + -2.767455501787848e+02 + -2.777000643997414e+02 + -2.786571816989448e+02 + -2.796169069311608e+02 + -2.805792449000000e+02 + -2.815442003833780e+02 + -2.825117782146733e+02 + -2.834819832031981e+02 + -2.844548201023671e+02 + -2.854302936999997e+02 + -2.864084088239409e+02 + -2.873891703596309e+02 + -2.883725831483384e+02 + -2.893586519481404e+02 + -2.903473816000000e+02 + -2.913387769892773e+02 + -2.923328429461174e+02 + -2.933295843055886e+02 + -2.943290059327143e+02 + -2.953311126999996e+02 + -2.963359094688269e+02 + -2.973434010446480e+02 + -2.983535922946355e+02 + -2.993664881931061e+02 + -3.003820936000000e+02 + -3.014004133189056e+02 + -3.024214522481863e+02 + -3.034452153213495e+02 + -3.044717074825456e+02 + -3.055009335999997e+02 + -3.065328985120693e+02 + -3.075676071368449e+02 + -3.086050644279766e+02 + -3.096452753553112e+02 + -3.106882447999996e+02 + -3.117339776182681e+02 + -3.127824787990806e+02 + -3.138337533166528e+02 + -3.148878060732492e+02 + -3.159446419999996e+02 + -3.170042660531784e+02 + -3.180666831989414e+02 + -3.191318984100575e+02 + -3.201999166629245e+02 + -3.212707428999996e+02 + -3.223443820580361e+02 + -3.234208391391681e+02 + -3.245001191331626e+02 + -3.255822269890704e+02 + -3.266671677000000e+02 + -3.277549462905472e+02 + -3.288455677875165e+02 + -3.299390371822755e+02 + -3.310353594166140e+02 + -3.321345394999996e+02 + -3.332365824854958e+02 + -3.343414934139734e+02 + -3.354492773045120e+02 + -3.365599391517927e+02 + -3.376734840000000e+02 + -3.387899169180687e+02 + -3.399092429402671e+02 + -3.410314670886827e+02 + -3.421565943845668e+02 + -3.432846298999996e+02 + -3.444155787253185e+02 + -3.455494458929433e+02 + -3.466862364711880e+02 + -3.478259555971263e+02 + -3.489686083000000e+02 + -3.501141995805723e+02 + -3.512627346186799e+02 + -3.524142185495274e+02 + -3.535686563776662e+02 + -3.547260531999996e+02 + -3.558864141649047e+02 + -3.570497443588797e+02 + -3.582160488781844e+02 + -3.593853328562213e+02 + -3.605576014000000e+02 + -3.617328596203877e+02 + -3.629111127187760e+02 + -3.640923658296630e+02 + -3.652766239636252e+02 + -3.664638922999996e+02 + -3.676541760949210e+02 + -3.688474804479435e+02 + -3.700438104641370e+02 + -3.712431713192604e+02 + -3.724455682000000e+02 + -3.736510062917217e+02 + -3.748594907617601e+02 + -3.760710267615897e+02 + -3.772856194324323e+02 + -3.785032739999995e+02 + -3.797239957155368e+02 + -3.809477897125092e+02 + -3.821746611611994e+02 + -3.834046153278488e+02 + -3.846376574000000e+02 + -3.858737925371631e+02 + -3.871130260109560e+02 + -3.883553630688227e+02 + -3.896008088834330e+02 + -3.908493686999996e+02 + -3.921010477895665e+02 + -3.933558513233559e+02 + -3.946137845264227e+02 + -3.958748527376295e+02 + -3.971390612000000e+02 + -3.984064151109821e+02 + -3.996769197585321e+02 + -4.009505804131749e+02 + -4.022274022910074e+02 + -4.035073906999995e+02 + -4.047905509809469e+02 + -4.060768883569506e+02 + -4.073664080804837e+02 + -4.086591154923789e+02 + -4.099550159000000e+02 + -4.112541145837781e+02 + -4.125564168232350e+02 + -4.138619279264533e+02 + -4.151706532436620e+02 + -4.164825980999995e+02 + -4.177977678009782e+02 + -4.191161676484082e+02 + -4.204378029703449e+02 + -4.217626791355866e+02 + -4.230908015000000e+02 + -4.244221753941545e+02 + -4.257568060915354e+02 + -4.270946989435893e+02 + -4.284358594322071e+02 + -4.297802928999995e+02 + -4.311280046181774e+02 + -4.324789999621266e+02 + -4.338332843406102e+02 + -4.351908631680312e+02 + -4.365517418000000e+02 + -4.379159255766193e+02 + -4.392834199290754e+02 + -4.406542302894571e+02 + -4.420283620541414e+02 + -4.434058205999995e+02 + -4.447866113008026e+02 + -4.461707395611302e+02 + -4.475582108324218e+02 + -4.489490306166049e+02 + -4.503432043000000e+02 + -4.517407372272212e+02 + -4.531416348857516e+02 + -4.545459027469901e+02 + -4.559535462055746e+02 + -4.573645706999994e+02 + -4.587789817092769e+02 + -4.601967847429616e+02 + -4.616179852648949e+02 + -4.630425886631084e+02 + -4.644706004000000e+02 + -4.659020259847504e+02 + -4.673368709042424e+02 + -4.687751406384571e+02 + -4.702168406679589e+02 + -4.716619764999995e+02 + -4.731105536500806e+02 + -4.745625775980045e+02 + -4.760180538297375e+02 + -4.774769878558138e+02 + -4.789393852000000e+02 + -4.804052513950263e+02 + -4.818745919803221e+02 + -4.833474124743740e+02 + -4.848237183656923e+02 + -4.863035151999994e+02 + -4.877868085453354e+02 + -4.892736039052198e+02 + -4.907639068268139e+02 + -4.922577229425711e+02 + -4.937550578000000e+02 + -4.952559169027076e+02 + -4.967603058178435e+02 + -4.982682301416788e+02 + -4.997796954853789e+02 + -5.012947073999994e+02 + -5.028132714194502e+02 + -5.043353931676896e+02 + -5.058610782730644e+02 + -5.073903323321115e+02 + -5.089231609000000e+02 + -5.104595695386481e+02 + -5.119995639358939e+02 + -5.135431497453484e+02 + -5.150903325247945e+02 + -5.166411178999995e+02 + -5.181955115300345e+02 + -5.197535190105982e+02 + -5.213151459679927e+02 + -5.228803980937674e+02 + -5.244492810000000e+02 + -5.260218002787250e+02 + -5.275979616609960e+02 + -5.291777708470047e+02 + -5.307612334400695e+02 + -5.323483550999995e+02 + -5.339391415258532e+02 + -5.355335984055753e+02 + -5.371317314077364e+02 + -5.387335461786030e+02 + -5.403390484000000e+02 + -5.419482437848817e+02 + -5.435611380730882e+02 + -5.451777369746786e+02 + -5.467980461477442e+02 + -5.484220712999994e+02 + -5.500498181706811e+02 + -5.516812924850077e+02 + -5.533164999617454e+02 + -5.549554463178829e+02 + -5.565981372999994e+02 + -5.582445786620732e+02 + -5.598947761113077e+02 + -5.615487353817318e+02 + -5.632064622631164e+02 + -5.648679624999994e+02 + -5.665332418238921e+02 + -5.682023060429622e+02 + -5.698751609401672e+02 + -5.715518122344557e+02 + -5.732322656999993e+02 + -5.749165271427032e+02 + -5.766046023386582e+02 + -5.782964970855437e+02 + -5.799922172195480e+02 + -5.816917684999994e+02 + -5.833951566715182e+02 + -5.851023876259811e+02 + -5.868134672043320e+02 + -5.885284011210334e+02 + -5.902471951999993e+02 + -5.919698553287222e+02 + -5.936963873375214e+02 + -5.954267970459595e+02 + -5.971610902818754e+02 + -5.988992728999993e+02 + -6.006413507613062e+02 + -6.023873296844711e+02 + -6.041372155081436e+02 + -6.058910141182311e+02 + -6.076487313999993e+02 + -6.094103732273231e+02 + -6.111759454411804e+02 + -6.129454539033432e+02 + -6.147189045204622e+02 + -6.164963032000001e+02 + -6.182776558378988e+02 + -6.200629682925465e+02 + -6.218522464515877e+02 + -6.236454962577363e+02 + -6.254427235999993e+02 + -6.272439343587159e+02 + -6.290491345295636e+02 + -6.308583300588987e+02 + -6.326715267788347e+02 + -6.344887306000001e+02 + -6.363099474872770e+02 + -6.381351833941451e+02 + -6.399644442896572e+02 + -6.417977361628546e+02 + -6.436350648999993e+02 + -6.454764363684590e+02 + -6.473218566292518e+02 + -6.491713317125350e+02 + -6.510248675271151e+02 + -6.528824700000000e+02 + -6.547441451008573e+02 + -6.566098988943567e+02 + -6.584797373808955e+02 + -6.603536664383599e+02 + -6.622316920999993e+02 + -6.641138204605409e+02 + -6.660000574329696e+02 + -6.678904089961051e+02 + -6.697848812885560e+02 + -6.716834803000000e+02 + -6.735862119462199e+02 + -6.754930822688247e+02 + -6.774040973560088e+02 + -6.793192633079277e+02 + -6.812385860999992e+02 + -6.831620716800010e+02 + -6.850897262089175e+02 + -6.870215557906422e+02 + -6.889575663679858e+02 + -6.908977640000001e+02 + -6.928421548145254e+02 + -6.947907448778678e+02 + -6.967435402481692e+02 + -6.987005469959722e+02 + -7.006617711999993e+02 + -7.026272189534540e+02 + -7.045968963883171e+02 + -7.065708096021641e+02 + -7.085489646318440e+02 + -7.105313676000000e+02 + -7.125180246666655e+02 + -7.145089419064375e+02 + -7.165041254104643e+02 + -7.185035813280965e+02 + -7.205073157999993e+02 + -7.225153349567163e+02 + -7.245276449240899e+02 + -7.265442518464891e+02 + -7.285651618951177e+02 + -7.305903812000000e+02 + -7.326199158832851e+02 + -7.346537721472801e+02 + -7.366919561576724e+02 + -7.387344740033407e+02 + -7.407813318999993e+02 + -7.428325361060332e+02 + -7.448880927069067e+02 + -7.469480078569493e+02 + -7.490122878703525e+02 + -7.510809389000000e+02 + -7.531539670309830e+02 + -7.552313785295614e+02 + -7.573131796529314e+02 + -7.593993765758833e+02 + -7.614899754999992e+02 + -7.635849826556215e+02 + -7.656844042998581e+02 + -7.677882466626152e+02 + -7.698965159244972e+02 + -7.720092183000000e+02 + -7.741263600422619e+02 + -7.762479474594095e+02 + -7.783739868214470e+02 + -7.805044843222228e+02 + -7.826394461999992e+02 + -7.847788787331309e+02 + -7.869227882276110e+02 + -7.890711809493379e+02 + -7.912240631015031e+02 + -7.933814410000000e+02 + -7.955433210068886e+02 + -7.977097093640062e+02 + -7.998806123337483e+02 + -8.020560362523712e+02 + -8.042359873999992e+02 + -8.064204720446323e+02 + -8.086094965664213e+02 + -8.108030673053524e+02 + -8.130011905039750e+02 + -8.152038725000000e+02 + -8.174111196729889e+02 + -8.196229383028856e+02 + -8.218393347070877e+02 + -8.240603152936806e+02 + -8.262858863999992e+02 + -8.285160543327190e+02 + -8.307508254786889e+02 + -8.329902062108882e+02 + -8.352342028550189e+02 + -8.374828218000000e+02 + -8.397360694583805e+02 + -8.419939521636865e+02 + -8.442564762682848e+02 + -8.465236481828920e+02 + -8.487954742999991e+02 + -8.510719610066617e+02 + -8.533531147279726e+02 + -8.556389418758610e+02 + -8.579294488297842e+02 + -8.602246420000000e+02 + -8.625245278111681e+02 + -8.648291126593311e+02 + -8.671384029773127e+02 + -8.694524052578884e+02 + -8.717711258999991e+02 + -8.740945712693658e+02 + -8.764227478547659e+02 + -8.787556621558366e+02 + -8.810933206360036e+02 + -8.834357297000000e+02 + -8.857828957473700e+02 + -8.881348253000285e+02 + -8.904915248639498e+02 + -8.928530008754832e+02 + -8.952192597999992e+02 + -8.975903081273602e+02 + -8.999661523549036e+02 + -9.023467989915350e+02 + -9.047322545556517e+02 + -9.071225255000001e+02 + -9.095176182642568e+02 + -9.119175394073789e+02 + -9.143222954857910e+02 + -9.167318930073794e+02 + -9.191463384999990e+02 + -9.215656384936522e+02 + -9.239897994653498e+02 + -9.264188279352956e+02 + -9.288527305046009e+02 + -9.312915136999991e+02 + -9.337351840217814e+02 + -9.361837480744202e+02 + -9.386372124507991e+02 + -9.410955836845210e+02 + -9.435588682999991e+02 + -9.460270728408829e+02 + -9.485002039375468e+02 + -9.509782681943145e+02 + -9.534612721474879e+02 + -9.559492223999991e+02 + -9.584421255818636e+02 + -9.609399882443452e+02 + -9.634428169536077e+02 + -9.659506183300578e+02 + -9.684633989999990e+02 + -9.709811655949806e+02 + -9.735039247636544e+02 + -9.760316831188370e+02 + -9.785644472182881e+02 + -9.811022236999991e+02 + -9.836450192407020e+02 + -9.861928404528198e+02 + -9.887456939756197e+02 + -9.913035865101050e+02 + -9.938665246999991e+02 + -9.964345151588749e+02 + -9.990075645416573e+02 + -1.001585679508894e+03 + -1.004168866725833e+03 + -1.006757132999999e+03 + -1.009350485088950e+03 + -1.011948929193798e+03 + -1.014552472017065e+03 + -1.017161121149714e+03 + -1.019774882999999e+03 + -1.022393763477281e+03 + -1.025017769745607e+03 + -1.027646908706880e+03 + -1.030281186449000e+03 + -1.032920609999999e+03 + -1.035565186772393e+03 + -1.038214923106640e+03 + -1.040869825393126e+03 + -1.043529900494233e+03 + -1.046195154999999e+03 + -1.048865595518282e+03 + -1.051541229525064e+03 + -1.054222063914517e+03 + -1.056908104442841e+03 + -1.059599357999999e+03 + -1.062295832060889e+03 + -1.064997533253958e+03 + -1.067704468147946e+03 + -1.070416643533965e+03 + -1.073134066000000e+03 + -1.075856742179453e+03 + -1.078584679453085e+03 + -1.081317884808236e+03 + -1.084056364390976e+03 + -1.086800124999999e+03 + -1.089549173751501e+03 + -1.092303517161580e+03 + -1.095063162007773e+03 + -1.097828115663242e+03 + -1.100598385000000e+03 + -1.103373976558467e+03 + -1.106154896984741e+03 + -1.108941153067517e+03 + -1.111732751768433e+03 + -1.114529699999999e+03 + -1.117332004656191e+03 + -1.120139672719820e+03 + -1.122952711033347e+03 + -1.125771126202256e+03 + -1.128594925000000e+03 + -1.131424114391635e+03 + -1.134258701616623e+03 + -1.137098693726371e+03 + -1.139944097362279e+03 + -1.142794918999999e+03 + -1.145651165298781e+03 + -1.148512843975744e+03 + -1.151379962155369e+03 + -1.154252525751212e+03 + -1.157130542000000e+03 + -1.160014018647746e+03 + -1.162902961796701e+03 + -1.165797378002409e+03 + -1.168697275086654e+03 + -1.171602659999999e+03 + -1.174513539162380e+03 + -1.177429919411751e+03 + -1.180351807863387e+03 + -1.183279211838743e+03 + -1.186212138000000e+03 + -1.189150592852472e+03 + -1.192094584016226e+03 + -1.195044118757429e+03 + -1.197999203418656e+03 + -1.200959844999999e+03 + -1.203926050837031e+03 + -1.206897827704460e+03 + -1.209875182550288e+03 + -1.212858122751144e+03 + -1.215846655000000e+03 + -1.218840785789916e+03 + -1.221840522671824e+03 + -1.224845872987532e+03 + -1.227856843385819e+03 + -1.230873440999999e+03 + -1.233895673111333e+03 + -1.236923546192640e+03 + -1.239957067026041e+03 + -1.242996243170224e+03 + -1.246041082000000e+03 + -1.249091590572761e+03 + -1.252147775334870e+03 + -1.255209643154299e+03 + -1.258277201722480e+03 + -1.261350457999999e+03 + -1.264429418673061e+03 + -1.267514091394071e+03 + -1.270604483350415e+03 + -1.273700600740823e+03 + -1.276802451000000e+03 + -1.279910042060014e+03 + -1.283023380429426e+03 + -1.286142472688095e+03 + -1.289267326090673e+03 + -1.292397947999999e+03 + -1.295534345774196e+03 + -1.298676526619691e+03 + -1.301824497524618e+03 + -1.304978265240164e+03 + -1.308137837000000e+03 + -1.311303220289759e+03 + -1.314474422298625e+03 + -1.317651449999705e+03 + -1.320834310180745e+03 + -1.324023009999999e+03 + -1.327217556910311e+03 + -1.330417958531748e+03 + -1.333624221890711e+03 + -1.336836353161044e+03 + -1.340054360000000e+03 + -1.343278250622039e+03 + -1.346508031450468e+03 + -1.349743709083262e+03 + -1.352985291084887e+03 + -1.356232784999999e+03 + -1.359486198143600e+03 + -1.362745537187869e+03 + -1.366010809195683e+03 + -1.369282022032803e+03 + -1.372559183000000e+03 + -1.375842299046690e+03 + -1.379131377356324e+03 + -1.382426425045151e+03 + -1.385727449081319e+03 + -1.389034456999999e+03 + -1.392347456524483e+03 + -1.395666454635196e+03 + -1.398991458464414e+03 + -1.402322475610563e+03 + -1.405659513000000e+03 + -1.409002577344138e+03 + -1.412351676333446e+03 + -1.415706817605260e+03 + -1.419068008353351e+03 + -1.422435255999999e+03 + -1.425808567964313e+03 + -1.429187950969556e+03 + -1.432573411998037e+03 + -1.435964958719179e+03 + -1.439362599000000e+03 + -1.442766340439504e+03 + -1.446176189225631e+03 + -1.449592152360744e+03 + -1.453014238529275e+03 + -1.456442454999999e+03 + -1.459876808245435e+03 + -1.463317305593544e+03 + -1.466763954627232e+03 + -1.470216762966463e+03 + -1.473675738000000e+03 + -1.477140887005819e+03 + -1.480612217422037e+03 + -1.484089736511155e+03 + -1.487573451238554e+03 + -1.491063368999999e+03 + -1.494559497483912e+03 + -1.498061844364458e+03 + -1.501570417048641e+03 + -1.505085222548716e+03 + -1.508606267999999e+03 + -1.512133560790714e+03 + -1.515667108906144e+03 + -1.519206919999463e+03 + -1.522753000996140e+03 + -1.526305358999999e+03 + -1.529864001422905e+03 + -1.533428936276007e+03 + -1.537000171179230e+03 + -1.540577712963951e+03 + -1.544161568999999e+03 + -1.547751747018138e+03 + -1.551348254621497e+03 + -1.554951099280540e+03 + -1.558560288310355e+03 + -1.562175828999999e+03 + -1.565797728689906e+03 + -1.569425994958102e+03 + -1.573060635400619e+03 + -1.576701657553543e+03 + -1.580349068999999e+03 + -1.584002877314684e+03 + -1.587663089901193e+03 + -1.591329713983250e+03 + -1.595002756657312e+03 + -1.598682225999999e+03 + -1.602368130282464e+03 + -1.606060476003810e+03 + -1.609759270166310e+03 + -1.613464521194204e+03 + -1.617176236999999e+03 + -1.620894424965112e+03 + -1.624619091996676e+03 + -1.628350245392540e+03 + -1.632087893195996e+03 + -1.635832042999999e+03 + -1.639582702219653e+03 + -1.643339878880967e+03 + -1.647103580589531e+03 + -1.650873814125992e+03 + -1.654650586999999e+03 + -1.658433907212865e+03 + -1.662223782656940e+03 + -1.666020220927786e+03 + -1.669823229228366e+03 + -1.673632814999999e+03 + -1.677448985879331e+03 + -1.681271749582079e+03 + -1.685101113981527e+03 + -1.688937087086922e+03 + -1.692779675999999e+03 + -1.696628887589857e+03 + -1.700484730162859e+03 + -1.704347211664743e+03 + -1.708216339003609e+03 + -1.712092119999999e+03 + -1.715974562928047e+03 + -1.719863675285195e+03 + -1.723759464515762e+03 + -1.727661938269359e+03 + -1.731571103999999e+03 + -1.735486969207614e+03 + -1.739409542118944e+03 + -1.743338830603560e+03 + -1.747274841733617e+03 + -1.751217582999999e+03 + -1.755167062315964e+03 + -1.759123288009624e+03 + -1.763086267739076e+03 + -1.767056008086542e+03 + -1.771032516999999e+03 + -1.775015803098183e+03 + -1.779005873908525e+03 + -1.783002736756524e+03 + -1.787006399126180e+03 + -1.791016869000000e+03 + -1.795034154506543e+03 + -1.799058263107071e+03 + -1.803089202321619e+03 + -1.807126980015989e+03 + -1.811171603999998e+03 + -1.815223082025504e+03 + -1.819281421835726e+03 + -1.823346631247284e+03 + -1.827418718154363e+03 + -1.831497690000000e+03 + -1.835583554215247e+03 + -1.839676319355180e+03 + -1.843775993459518e+03 + -1.847882583413063e+03 + -1.851996096999998e+03 + -1.856116542493618e+03 + -1.860243927542824e+03 + -1.864378259929794e+03 + -1.868519547846306e+03 + -1.872667799000000e+03 + -1.876823020804590e+03 + -1.880985220849939e+03 + -1.885154407199114e+03 + -1.889330588466962e+03 + -1.893513771999998e+03 + -1.897703964771626e+03 + -1.901901175652285e+03 + -1.906105412964281e+03 + -1.910316683519642e+03 + -1.914534995000000e+03 + -1.918760355678869e+03 + -1.922992773607248e+03 + -1.927232256642219e+03 + -1.931478812455566e+03 + -1.935732448999998e+03 + -1.939993174351127e+03 + -1.944260996306766e+03 + -1.948535922544343e+03 + -1.952817960710870e+03 + -1.957107119000000e+03 + -1.961403405774432e+03 + -1.965706828656891e+03 + -1.970017395435863e+03 + -1.974335114379128e+03 + -1.978659992999998e+03 + -1.982992038634364e+03 + -1.987331259958815e+03 + -1.991677665406797e+03 + -1.996031262526477e+03 + -2.000392059000000e+03 + -2.004760062662544e+03 + -2.009135281420714e+03 + -2.013517723394651e+03 + -2.017907396963966e+03 + -2.022304309999998e+03 + -2.026708470129302e+03 + -2.031119885371980e+03 + -2.035538563783372e+03 + -2.039964513323241e+03 + -2.044397742000000e+03 + -2.048838257821666e+03 + -2.053286068652342e+03 + -2.057741182613919e+03 + -2.062203608203423e+03 + -2.066673352999998e+03 + -2.071150424296029e+03 + -2.075634830690161e+03 + -2.080126580708826e+03 + -2.084625682242698e+03 + -2.089132143000000e+03 + -2.093645970720743e+03 + -2.098167173603271e+03 + -2.102695759983049e+03 + -2.107231738184183e+03 + -2.111775115999998e+03 + -2.116325901057278e+03 + -2.120884101683680e+03 + -2.125449726108530e+03 + -2.130022782163747e+03 + -2.134603278000000e+03 + -2.139191221924691e+03 + -2.143786621957349e+03 + -2.148389485981318e+03 + -2.152999821836896e+03 + -2.157617637999998e+03 + -2.162242943074696e+03 + -2.166875744534069e+03 + -2.171516050262848e+03 + -2.176163869133622e+03 + -2.180819209000000e+03 + -2.185482077327040e+03 + -2.190152482871076e+03 + -2.194830433950524e+03 + -2.199515937814626e+03 + -2.204209002999998e+03 + -2.208909638535144e+03 + -2.213617851857877e+03 + -2.218333650781778e+03 + -2.223057044225780e+03 + -2.227788040000000e+03 + -2.232526645541351e+03 + -2.237272869895315e+03 + -2.242026721580949e+03 + -2.246788207788726e+03 + -2.251557336999999e+03 + -2.256334118250544e+03 + -2.261118559161676e+03 + -2.265910667352767e+03 + -2.270710451040453e+03 + -2.275517918999998e+03 + -2.280333080014267e+03 + -2.285155941593499e+03 + -2.289986511539142e+03 + -2.294824798563590e+03 + -2.299670810999998e+03 + -2.304524556927484e+03 + -2.309386044592275e+03 + -2.314255282195762e+03 + -2.319132277836808e+03 + -2.324017039999998e+03 + -2.328909577220459e+03 + -2.333809897224200e+03 + -2.338718008145149e+03 + -2.343633918994289e+03 + -2.348557637999998e+03 + -2.353489172912608e+03 + -2.358428531797252e+03 + -2.363375723128402e+03 + -2.368330755820120e+03 + -2.373293637999998e+03 + -2.378264377427878e+03 + -2.383242982526706e+03 + -2.388229461889857e+03 + -2.393223824063392e+03 + -2.398226076999998e+03 + -2.403236228525699e+03 + -2.408254287477974e+03 + -2.413280262494937e+03 + -2.418314161547432e+03 + -2.423355992999998e+03 + -2.428405765403690e+03 + -2.433463486890615e+03 + -2.438529165661329e+03 + -2.443602810190047e+03 + -2.448684428999998e+03 + -2.453774030522361e+03 + -2.458871622761939e+03 + -2.463977214083831e+03 + -2.469090813490053e+03 + -2.474212428999998e+03 + -2.479342068313477e+03 + -2.484479740528551e+03 + -2.489625454539379e+03 + -2.494779218375099e+03 + -2.499941039999998e+03 + -2.505110927624773e+03 + -2.510288890409285e+03 + -2.515474937052364e+03 + -2.520669075212765e+03 + -2.525871312999998e+03 + -2.531081658990217e+03 + -2.536300122220549e+03 + -2.541526711425658e+03 + -2.546761434689014e+03 + -2.552004299999998e+03 + -2.557255315555890e+03 + -2.562514490474060e+03 + -2.567781833501355e+03 + -2.573057352510476e+03 + -2.578341055999998e+03 + -2.583632952807378e+03 + -2.588933051312229e+03 + -2.594241359879810e+03 + -2.599557887014108e+03 + -2.604882640999998e+03 + -2.610215630196411e+03 + -2.615556863841929e+03 + -2.620906350516468e+03 + -2.626264097579904e+03 + -2.631630113999998e+03 + -2.637004409331177e+03 + -2.642386991059378e+03 + -2.647777867311749e+03 + -2.653177047878767e+03 + -2.658584540999998e+03 + -2.664000354190328e+03 + -2.669424496446296e+03 + -2.674856976816473e+03 + -2.680297803844182e+03 + -2.685746985999998e+03 + -2.691204531699670e+03 + -2.696670449221115e+03 + -2.702144747143579e+03 + -2.707627434527642e+03 + -2.713118519999998e+03 + -2.718618011850782e+03 + -2.724125918263655e+03 + -2.729642247949266e+03 + -2.735167010382318e+03 + -2.740700213999998e+03 + -2.746241866699575e+03 + -2.751791977076697e+03 + -2.757350554006517e+03 + -2.762917606479738e+03 + -2.768493143000000e+03 + -2.774077171851313e+03 + -2.779669701724049e+03 + -2.785270741295763e+03 + -2.790880299095185e+03 + -2.796498383999998e+03 + -2.802125004968091e+03 + -2.807760170344632e+03 + -2.813403888501788e+03 + -2.819056168106567e+03 + -2.824717018000000e+03 + -2.830386447079949e+03 + -2.836064464083062e+03 + -2.841751077483182e+03 + -2.847446295475182e+03 + -2.853150126999998e+03 + -2.858862581346520e+03 + -2.864583667200011e+03 + -2.870313392988722e+03 + -2.876051767015894e+03 + -2.881798798000000e+03 + -2.887554494917011e+03 + -2.893318866668325e+03 + -2.899091922031221e+03 + -2.904873669618204e+03 + -2.910664117999998e+03 + -2.916463275808826e+03 + -2.922271151990944e+03 + -2.928087755363771e+03 + -2.933913094445438e+03 + -2.939747178000000e+03 + -2.945590014959967e+03 + -2.951441614226611e+03 + -2.957301984588625e+03 + -2.963171134666888e+03 + -2.969049072999998e+03 + -2.974935808208847e+03 + -2.980831349417174e+03 + -2.986735705755709e+03 + -2.992648886132359e+03 + -2.998570899000000e+03 + -3.004501752659864e+03 + -3.010441455936329e+03 + -3.016390017858416e+03 + -3.022347447554316e+03 + -3.028313753999998e+03 + -3.034288946006182e+03 + -3.040273032141582e+03 + -3.046266021078663e+03 + -3.052267921739149e+03 + -3.058278743000000e+03 + -3.064298493698561e+03 + -3.070327182685398e+03 + -3.076364818827055e+03 + -3.082411411001809e+03 + -3.088466967999998e+03 + -3.094531498652569e+03 + -3.100605012177602e+03 + -3.106687517554494e+03 + -3.112779023262762e+03 + -3.118879538000000e+03 + -3.124989070695736e+03 + -3.131107630527278e+03 + -3.137235226491168e+03 + -3.143371867254447e+03 + -3.149517561999998e+03 + -3.155672319953097e+03 + -3.161836149115856e+03 + -3.168009058152651e+03 + -3.174191057129923e+03 + -3.180382155000000e+03 + -3.186582360055111e+03 + -3.192791681144167e+03 + -3.199010127274887e+03 + -3.205237707493432e+03 + -3.211474430999998e+03 + -3.217720306909038e+03 + -3.223975343611381e+03 + -3.230239549889151e+03 + -3.236512935355519e+03 + -3.242795509000000e+03 + -3.249087279482800e+03 + -3.255388255944051e+03 + -3.261698447489179e+03 + -3.268017862963467e+03 + -3.274346510999998e+03 + -3.280684400401313e+03 + -3.287031541121871e+03 + -3.293387942369432e+03 + -3.299753611888159e+03 + -3.306128559000000e+03 + -3.312512793688797e+03 + -3.318906324190636e+03 + -3.325309159228279e+03 + -3.331721308860119e+03 + -3.338142781999998e+03 + -3.344573586933190e+03 + -3.351013732692941e+03 + -3.357463228591956e+03 + -3.363922084041577e+03 + -3.370390307999997e+03 + -3.376867909193196e+03 + -3.383354896616435e+03 + -3.389851279509897e+03 + -3.396357067337695e+03 + -3.402872268999997e+03 + -3.409396893188545e+03 + -3.415930949268335e+03 + -3.422474446597307e+03 + -3.429027394238212e+03 + -3.435589800999997e+03 + -3.442161675674332e+03 + -3.448743027574159e+03 + -3.455333866086035e+03 + -3.461934200499129e+03 + -3.468544039999997e+03 + -3.475163393634110e+03 + -3.481792270131310e+03 + -3.488430678432703e+03 + -3.495078627927044e+03 + -3.501736127999997e+03 + -3.508403187927853e+03 + -3.515079816661691e+03 + -3.521766023349286e+03 + -3.528461817510494e+03 + -3.535167207999998e+03 + -3.541882203477436e+03 + -3.548606813620608e+03 + -3.555341047964333e+03 + -3.562084915431213e+03 + -3.568838424999997e+03 + -3.575601585773337e+03 + -3.582374407066678e+03 + -3.589156898185297e+03 + -3.595949068354316e+03 + -3.602750926999998e+03 + -3.609562483468940e+03 + -3.616383746264014e+03 + -3.623214724488114e+03 + -3.630055428392298e+03 + -3.636905866999997e+03 + -3.643766048793625e+03 + -3.650635983516456e+03 + -3.657515680680700e+03 + -3.664405149014039e+03 + -3.671304397999997e+03 + -3.678213437374473e+03 + -3.685132275760156e+03 + -3.692060922035369e+03 + -3.698999385901594e+03 + -3.705947676999997e+03 + -3.712905804797643e+03 + -3.719873778403308e+03 + -3.726851606752037e+03 + -3.733839298718909e+03 + -3.740836863999997e+03 + -3.747844312547997e+03 + -3.754861653211674e+03 + -3.761888895074585e+03 + -3.768926047955085e+03 + -3.775973120999997e+03 + -3.783030123074376e+03 + -3.790097063813473e+03 + -3.797173952745452e+03 + -3.804260798968512e+03 + -3.811357611999997e+03 + -3.818464401472386e+03 + -3.825581176303607e+03 + -3.832707945585785e+03 + -3.839844718954238e+03 + -3.846991505999998e+03 + -3.854148316201428e+03 + -3.861315158822798e+03 + -3.868492043129635e+03 + -3.875678978458987e+03 + -3.882875973999997e+03 + -3.890083038997269e+03 + -3.897300183310783e+03 + -3.904527416633095e+03 + -3.911764748154242e+03 + -3.919012186999997e+03 + -3.926269742452437e+03 + -3.933537424464352e+03 + -3.940815242621297e+03 + -3.948103205720294e+03 + -3.955401322999997e+03 + -3.962709604107895e+03 + -3.970028058998795e+03 + -3.977356697163350e+03 + -3.984695527326294e+03 + -3.992044558999997e+03 + -3.999403802144854e+03 + -4.006773266302151e+03 + -4.014152960820400e+03 + -4.021542894928240e+03 + -4.028943077999997e+03 + -4.036353519583599e+03 + -4.043774229513202e+03 + -4.051205217431756e+03 + -4.058646492564995e+03 + -4.066098063999997e+03 + -4.073559940995820e+03 + -4.081032133755226e+03 + -4.088514652104142e+03 + -4.096007504979870e+03 + -4.103510702000000e+03 + -4.111024253097677e+03 + -4.118548167510886e+03 + -4.126082454416708e+03 + -4.133627123225803e+03 + -4.141182184000000e+03 + -4.148747646853033e+03 + -4.156323520486682e+03 + -4.163909814098826e+03 + -4.171506538118098e+03 + -4.179113701999993e+03 + -4.186731314728609e+03 + -4.194359386189681e+03 + -4.201997926116393e+03 + -4.209646943720986e+03 + -4.217306449000000e+03 + -4.224976452122954e+03 + -4.232656961817923e+03 + -4.240347987240753e+03 + -4.248049538718602e+03 + -4.255761626000000e+03 + -4.263484258397591e+03 + -4.271217445246016e+03 + -4.278961196276451e+03 + -4.286715521743729e+03 + -4.294480431000000e+03 + -4.302255933066315e+03 + -4.310042038085233e+03 + -4.317838756007569e+03 + -4.325646096087124e+03 + -4.333464067999995e+03 + -4.341292681602012e+03 + -4.349131946225953e+03 + -4.356981871466000e+03 + -4.364842467485710e+03 + -4.372713744000000e+03 + -4.380595710372718e+03 + -4.388488375861229e+03 + -4.396391750264422e+03 + -4.404305844160006e+03 + -4.412230667000000e+03 + -4.420166227736474e+03 + -4.428112536405837e+03 + -4.436069603088062e+03 + -4.444037437481978e+03 + -4.452016049000000e+03 + -4.460005447074678e+03 + -4.468005641867199e+03 + -4.476016643204103e+03 + -4.484038460200805e+03 + -4.492071102999995e+03 + -4.500114582060231e+03 + -4.508168906299815e+03 + -4.516234085098625e+03 + -4.524310129075975e+03 + -4.532397048000000e+03 + -4.540494851104932e+03 + -4.548603547974418e+03 + -4.556723148595408e+03 + -4.564853663376500e+03 + -4.572995102000000e+03 + -4.581147473755137e+03 + -4.589310788334117e+03 + -4.597485055595826e+03 + -4.605670285497996e+03 + -4.613866488000000e+03 + -4.622073673035890e+03 + -4.630291850451470e+03 + -4.638521029781397e+03 + -4.646761220207367e+03 + -4.655012431999994e+03 + -4.663274675842531e+03 + -4.671547961173437e+03 + -4.679832297267551e+03 + -4.688127693710799e+03 + -4.696434161000000e+03 + -4.704751709752170e+03 + -4.713080348793824e+03 + -4.721420087561993e+03 + -4.729770937031085e+03 + -4.738132907000000e+03 + -4.746506006604411e+03 + -4.754890245715343e+03 + -4.763285634540516e+03 + -4.771692183448750e+03 + -4.780109901999994e+03 + -4.788538799466528e+03 + -4.796978886096308e+03 + -4.805430172162917e+03 + -4.813892667577567e+03 + -4.822366381999994e+03 + -4.830851325088147e+03 + -4.839347507060687e+03 + -4.847854938123849e+03 + -4.856373628221604e+03 + -4.864903587000000e+03 + -4.873444824101478e+03 + -4.881997349859994e+03 + -4.890561174364984e+03 + -4.899136307099045e+03 + -4.907722758000000e+03 + -4.916320537362784e+03 + -4.924929655587317e+03 + -4.933550122528648e+03 + -4.942181947260091e+03 + -4.950825139999994e+03 + -4.959479711457802e+03 + -4.968145671175524e+03 + -4.976823028809056e+03 + -4.985511794647833e+03 + -4.994211978999993e+03 + -5.002923592007461e+03 + -5.011646643245516e+03 + -5.020381142480443e+03 + -5.029127099988444e+03 + -5.037884526000000e+03 + -5.046653430641881e+03 + -5.055433823857634e+03 + -5.064225715527905e+03 + -5.073029115529123e+03 + -5.081844034000000e+03 + -5.090670481166524e+03 + -5.099508466925161e+03 + -5.108358001180135e+03 + -5.117219094017154e+03 + -5.126091755999993e+03 + -5.134975997626908e+03 + -5.143871827953957e+03 + -5.152779256704796e+03 + -5.161698295082459e+03 + -5.170628952999993e+03 + -5.179571239778229e+03 + -5.188525166029071e+03 + -5.197490742130520e+03 + -5.206467977662855e+03 + -5.215456883000000e+03 + -5.224457468752808e+03 + -5.233469744262066e+03 + -5.242493719420175e+03 + -5.251529405361101e+03 + -5.260576812000000e+03 + -5.269635948683524e+03 + -5.278706825911151e+03 + -5.287789454172008e+03 + -5.296883843495917e+03 + -5.305990003999993e+03 + -5.315107945871962e+03 + -5.324237679253967e+03 + -5.333379214299059e+03 + -5.342532561186867e+03 + -5.351697729999993e+03 + -5.360874730862462e+03 + -5.370063574314554e+03 + -5.379264270604972e+03 + -5.388476829418795e+03 + -5.397701261000000e+03 + -5.406937575887649e+03 + -5.416185784228891e+03 + -5.425445896082409e+03 + -5.434717921547790e+03 + -5.444001871000000e+03 + -5.453297754883592e+03 + -5.462605583222671e+03 + -5.471925366229407e+03 + -5.481257114511772e+03 + -5.490600837999993e+03 + -5.499956546429602e+03 + -5.509324250592133e+03 + -5.518703961068116e+03 + -5.528095687743725e+03 + -5.537499440999993e+03 + -5.546915231361128e+03 + -5.556343068498348e+03 + -5.565782962476614e+03 + -5.575234924238452e+03 + -5.584698964000000e+03 + -5.594175091687396e+03 + -5.603663318133420e+03 + -5.613163653827974e+03 + -5.622676108416792e+03 + -5.632200692000000e+03 + -5.641737415094019e+03 + -5.651286288507457e+03 + -5.660847322604374e+03 + -5.670420527008017e+03 + -5.680005911999993e+03 + -5.689603488367688e+03 + -5.699213267038800e+03 + -5.708835258184300e+03 + -5.718469470901945e+03 + -5.728115915999992e+03 + -5.737774605002216e+03 + -5.747445547634946e+03 + -5.757128753652110e+03 + -5.766824233562706e+03 + -5.776531998000000e+03 + -5.786252057564687e+03 + -5.795984422575620e+03 + -5.805729103393980e+03 + -5.815486110517342e+03 + -5.825255454000000e+03 + -5.835037143865786e+03 + -5.844831191184440e+03 + -5.854637606668529e+03 + -5.864456400127173e+03 + -5.874287581999993e+03 + -5.884131163070459e+03 + -5.893987153665426e+03 + -5.903855564090225e+03 + -5.913736404781277e+03 + -5.923629686000000e+03 + -5.933535418061928e+03 + -5.943453611973397e+03 + -5.953384278439786e+03 + -5.963327427438455e+03 + -5.973283069000000e+03 + -5.983251213525049e+03 + -5.993231872573540e+03 + -6.003225056744913e+03 + -6.013230774891098e+03 + -6.023249038000000e+03 + -6.033279857947543e+03 + -6.043323244258373e+03 + -6.053379206808452e+03 + -6.063447756872029e+03 + -6.073528904999992e+03 + -6.083622661322729e+03 + -6.093729036471069e+03 + -6.103848041095609e+03 + -6.113979685699232e+03 + -6.124123981000000e+03 + -6.134280937646101e+03 + -6.144450565430006e+03 + -6.154632874755873e+03 + -6.164827877191976e+03 + -6.175035583000000e+03 + -6.185256001935748e+03 + -6.195489145360168e+03 + -6.205735024246584e+03 + -6.215993648382001e+03 + -6.226265028000000e+03 + -6.236549173839082e+03 + -6.246846097225249e+03 + -6.257155808917487e+03 + -6.267478318647532e+03 + -6.277813636999993e+03 + -6.288161775039973e+03 + -6.298522743318929e+03 + -6.308896552332133e+03 + -6.319283212690151e+03 + -6.329682735000000e+03 + -6.340095129841887e+03 + -6.350520407726451e+03 + -6.360958579315570e+03 + -6.371409655512612e+03 + -6.381873647000000e+03 + -6.392350564293673e+03 + -6.402840417873253e+03 + -6.413343218449801e+03 + -6.423858977046060e+03 + -6.434387704000000e+03 + -6.444929409479182e+03 + -6.455484104800415e+03 + -6.466051800878468e+03 + -6.476632507643834e+03 + -6.487226235999992e+03 + -6.497832997260840e+03 + -6.508452801649206e+03 + -6.519085659445890e+03 + -6.529731581482401e+03 + -6.540390579000000e+03 + -6.551062663116751e+03 + -6.561747843536938e+03 + -6.572446130639311e+03 + -6.583157536279329e+03 + -6.593882071000000e+03 + -6.604619744763902e+03 + -6.615370568925604e+03 + -6.626134554413510e+03 + -6.636911711064461e+03 + -6.647702049999992e+03 + -6.658505582840396e+03 + -6.669322319643610e+03 + -6.680152270785791e+03 + -6.690995447680221e+03 + -6.701851860999993e+03 + -6.712721521095499e+03 + -6.723604439166686e+03 + -6.734500626205399e+03 + -6.745410092579864e+03 + -6.756332849000000e+03 + -6.767268906377159e+03 + -6.778218275408767e+03 + -6.789180966973428e+03 + -6.800156992270870e+03 + -6.811146362000000e+03 + -6.822149086596000e+03 + -6.833165176809022e+03 + -6.844194643583943e+03 + -6.855237498022965e+03 + -6.866293750999992e+03 + -6.877363413246921e+03 + -6.888446495554016e+03 + -6.899543008749155e+03 + -6.910652963678518e+03 + -6.921776370999992e+03 + -6.932913241460104e+03 + -6.944063586638155e+03 + -6.955227417524035e+03 + -6.966404743973952e+03 + -6.977595577000000e+03 + -6.988799928187199e+03 + -7.000017808173458e+03 + -7.011249227729020e+03 + -7.022494198140058e+03 + -7.033752730000000e+03 + -7.045024833685246e+03 + -7.056310520605544e+03 + -7.067609802022496e+03 + -7.078922688575014e+03 + -7.090249190999992e+03 + -7.101589320140114e+03 + -7.112943086874141e+03 + -7.124310502404821e+03 + -7.135691578318693e+03 + -7.147086324999992e+03 + -7.158494752468790e+03 + -7.169916872475182e+03 + -7.181352696510882e+03 + -7.192802235036809e+03 + -7.204265499000000e+03 + -7.215742499578333e+03 + -7.227233247391960e+03 + -7.238737753333357e+03 + -7.250256028887258e+03 + -7.261788085000000e+03 + -7.273333932305611e+03 + -7.284893581718426e+03 + -7.296467044458082e+03 + -7.308054332025791e+03 + -7.319655454999992e+03 + -7.331270423697963e+03 + -7.342899249833889e+03 + -7.354541944862027e+03 + -7.366198519315069e+03 + -7.377868983999992e+03 + -7.389553350032372e+03 + -7.401251628839381e+03 + -7.412963831476617e+03 + -7.424689968390403e+03 + -7.436430051000000e+03 + -7.448184091042194e+03 + -7.459952098877861e+03 + -7.471734085033887e+03 + -7.483530060856024e+03 + -7.495340038000000e+03 + -7.507164027981260e+03 + -7.519002041143363e+03 + -7.530854088088799e+03 + -7.542720180263871e+03 + -7.554600328999991e+03 + -7.566494545476364e+03 + -7.578402840747203e+03 + -7.590325225748156e+03 + -7.602261711334166e+03 + -7.614212308999990e+03 + -7.626177030375796e+03 + -7.638155885970103e+03 + -7.650148886679111e+03 + -7.662156044359298e+03 + -7.674177370000000e+03 + -7.686212874133715e+03 + -7.698262567927072e+03 + -7.710326462858042e+03 + -7.722404570574414e+03 + -7.734496902000000e+03 + -7.746603467726753e+03 + -7.758724278933637e+03 + -7.770859347070250e+03 + -7.783008683730720e+03 + -7.795172299999991e+03 + -7.807350206747154e+03 + -7.819542415323729e+03 + -7.831748936928582e+03 + -7.843969782379609e+03 + -7.856204962999991e+03 + -7.868454490389281e+03 + -7.880718375856472e+03 + -7.892996630459527e+03 + -7.905289265033489e+03 + -7.917596291000000e+03 + -7.929917719977670e+03 + -7.942253562811284e+03 + -7.954603830544003e+03 + -7.966968534806996e+03 + -7.979347687000000e+03 + -7.991741298275563e+03 + -8.004149379532312e+03 + -8.016571941875890e+03 + -8.029008996834256e+03 + -8.041460555999991e+03 + -8.053926630829063e+03 + -8.066407232164392e+03 + -8.078902371093302e+03 + -8.091412059283156e+03 + -8.103936307999991e+03 + -8.116475128348727e+03 + -8.129028531951320e+03 + -8.141596530160446e+03 + -8.154179133753471e+03 + -8.166776354000000e+03 + -8.179388202500601e+03 + -8.192014690774517e+03 + -8.204655830155478e+03 + -8.217311631752342e+03 + -8.229982107000000e+03 + -8.242667267443709e+03 + -8.255367124181290e+03 + -8.268081688506129e+03 + -8.280810972124258e+03 + -8.293554985999990e+03 + -8.306313740930424e+03 + -8.319087249046417e+03 + -8.331875522237864e+03 + -8.344678571514016e+03 + -8.357496408000001e+03 + -8.370329042981793e+03 + -8.383176487893126e+03 + -8.396038754236946e+03 + -8.408915853538949e+03 + -8.421807796999999e+03 + -8.434714595760075e+03 + -8.447636261558551e+03 + -8.460572806112319e+03 + -8.473524240852536e+03 + -8.486490577000000e+03 + -8.499471825734363e+03 + -8.512467998550057e+03 + -8.525479106914012e+03 + -8.538505162160651e+03 + -8.551546175999989e+03 + -8.564602160205039e+03 + -8.577673125806519e+03 + -8.590759084111069e+03 + -8.603860047081474e+03 + -8.616976026000000e+03 + -8.630107031891077e+03 + -8.643253076647045e+03 + -8.656414171854482e+03 + -8.669590328370747e+03 + -8.682781558000001e+03 + -8.695987872906602e+03 + -8.709209284090668e+03 + -8.722445802791402e+03 + -8.735697441007769e+03 + -8.748964210000000e+03 + -8.762246120773598e+03 + -8.775543185390141e+03 + -8.788855415760128e+03 + -8.802182823160678e+03 + -8.815525418999991e+03 + -8.828883214778334e+03 + -8.842256221886048e+03 + -8.855644451817980e+03 + -8.869047916276779e+03 + -8.882466627000000e+03 + -8.895900595656447e+03 + -8.909349833599377e+03 + -8.922814352320849e+03 + -8.936294163622882e+03 + -8.949789279000001e+03 + -8.963299709806102e+03 + -8.976825467697834e+03 + -8.990366564425331e+03 + -9.003923011733321e+03 + -9.017494820999989e+03 + -9.031082003570244e+03 + -9.044684571593196e+03 + -9.058302536801553e+03 + -9.071935910043938e+03 + -9.085584702999991e+03 + -9.099248927805194e+03 + -9.112928596071970e+03 + -9.126623719398038e+03 + -9.140334309541779e+03 + -9.154060378000000e+03 + -9.167801936143134e+03 + -9.181558995552323e+03 + -9.195331568052554e+03 + -9.209119665711642e+03 + -9.222923300000000e+03 + -9.236742482191501e+03 + -9.250577224366738e+03 + -9.264427538348678e+03 + -9.278293435300837e+03 + -9.292174926999989e+03 + -9.306072025514850e+03 + -9.319984742353185e+03 + -9.333913089239737e+03 + -9.347857078372264e+03 + -9.361816720999988e+03 + -9.375792028076301e+03 + -9.389783011923670e+03 + -9.403789684689073e+03 + -9.417812057757259e+03 + -9.431850143000000e+03 + -9.445903952452922e+03 + -9.459973497383473e+03 + -9.474058789280491e+03 + -9.488159840242311e+03 + -9.502276662000000e+03 + -9.516409266099370e+03 + -9.530557664431180e+03 + -9.544721868948835e+03 + -9.558901891554557e+03 + -9.573097743999990e+03 + -9.587309437907974e+03 + -9.601536984771086e+03 + -9.615780396443668e+03 + -9.630039685320193e+03 + -9.644314862999989e+03 + -9.658605940694017e+03 + -9.672912930240365e+03 + -9.687235843732506e+03 + -9.701574693379191e+03 + -9.715929491000001e+03 + -9.730300248146514e+03 + -9.744686976337332e+03 + -9.759089687418043e+03 + -9.773508393705906e+03 + -9.787943106999999e+03 + -9.802393838806964e+03 + -9.816860600913731e+03 + -9.831343405392694e+03 + -9.845842264577648e+03 + -9.860357189999990e+03 + -9.874888192875391e+03 + -9.889435285292540e+03 + -9.903998479577405e+03 + -9.918577788007758e+03 + -9.933173221999989e+03 + -9.947784792760667e+03 + -9.962412512866247e+03 + -9.977056394630243e+03 + -9.991716449514770e+03 + -1.000639269000000e+04 + -1.002108512827329e+04 + -1.003579377260607e+04 + -1.005051863510592e+04 + -1.006525973450424e+04 + -1.008001708000000e+04 + -1.009479067738785e+04 + -1.010958054488143e+04 + -1.012438669671707e+04 + -1.013920913701616e+04 + -1.015404787999999e+04 + -1.016890294387830e+04 + -1.018377433451942e+04 + -1.019866206104612e+04 + -1.021356614193473e+04 + -1.022848658999999e+04 + -1.024342341422066e+04 + -1.025837662498870e+04 + -1.027334623400460e+04 + -1.028833225445739e+04 + -1.030333470000000e+04 + -1.031835358368700e+04 + -1.033338891534757e+04 + -1.034844070623467e+04 + -1.036350897076085e+04 + -1.037859372000000e+04 + -1.039369496373519e+04 + -1.040881271589677e+04 + -1.042394698923176e+04 + -1.043909779325773e+04 + -1.045426513999999e+04 + -1.046944904320307e+04 + -1.048464951626011e+04 + -1.049986657147346e+04 + -1.051510021951631e+04 + -1.053035046999999e+04 + -1.054561733346490e+04 + -1.056090082649770e+04 + -1.057620096382862e+04 + -1.059151775501402e+04 + -1.060685121000000e+04 + -1.062220133982168e+04 + -1.063756815769902e+04 + -1.065295167679853e+04 + -1.066835190951487e+04 + -1.068376887000000e+04 + -1.069920257172756e+04 + -1.071465302083269e+04 + -1.073012022736731e+04 + -1.074560420973240e+04 + -1.076110497999999e+04 + -1.077662254698522e+04 + -1.079215692468130e+04 + -1.080770812602064e+04 + -1.082327616047952e+04 + -1.083886103999999e+04 + -1.085446277832030e+04 + -1.087008138917964e+04 + -1.088571688451933e+04 + -1.090136927381244e+04 + -1.091703857000000e+04 + -1.093272478707855e+04 + -1.094842793381667e+04 + -1.096414802233435e+04 + -1.097988507119364e+04 + -1.099563909000000e+04 + -1.101141008471468e+04 + -1.102719807131307e+04 + -1.104300306633410e+04 + -1.105882508291791e+04 + -1.107466412999999e+04 + -1.109052021511841e+04 + -1.110639335038349e+04 + -1.112228355101184e+04 + -1.113819083451417e+04 + -1.115411520999999e+04 + -1.117005668368749e+04 + -1.118601527242516e+04 + -1.120199099177015e+04 + -1.121798385114509e+04 + -1.123399386000000e+04 + -1.125002102929127e+04 + -1.126606537454743e+04 + -1.128212691149971e+04 + -1.129820565394142e+04 + -1.131430161000000e+04 + -1.133041478676655e+04 + -1.134654520134832e+04 + -1.136269286880464e+04 + -1.137885779750048e+04 + -1.139503999999999e+04 + -1.141123949060806e+04 + -1.142745627830015e+04 + -1.144369037489935e+04 + -1.145994179819328e+04 + -1.147621056000000e+04 + -1.149249666929363e+04 + -1.150880014040390e+04 + -1.152512098586926e+04 + -1.154145921396878e+04 + -1.155781484000000e+04 + -1.157418788116290e+04 + -1.159057834325663e+04 + -1.160698623573862e+04 + -1.162341157750990e+04 + -1.163985437999999e+04 + -1.165631465167572e+04 + -1.167279241049137e+04 + -1.168928767028212e+04 + -1.170580043559586e+04 + -1.172233071999999e+04 + -1.173887854137991e+04 + -1.175544390951204e+04 + -1.177202683447122e+04 + -1.178862732998528e+04 + -1.180524541000000e+04 + -1.182188108744826e+04 + -1.183853437152247e+04 + -1.185520527417456e+04 + -1.187189381268968e+04 + -1.188860000000000e+04 + -1.190532384555031e+04 + -1.192206535739879e+04 + -1.193882454957090e+04 + -1.195560144455150e+04 + -1.197239604999999e+04 + -1.198920836801576e+04 + -1.200603841862060e+04 + -1.202288622005816e+04 + -1.203975178087160e+04 + -1.205663510999999e+04 + -1.207353621858988e+04 + -1.209045512337030e+04 + -1.210739183876488e+04 + -1.212434637372514e+04 + -1.214131874000000e+04 + -1.215830895146994e+04 + -1.217531702204386e+04 + -1.219234296344827e+04 + -1.220938678454126e+04 + -1.222644850000000e+04 + -1.224352812665298e+04 + -1.226062567426862e+04 + -1.227774115320455e+04 + -1.229487457740941e+04 + -1.231202595999999e+04 + -1.232919531361497e+04 + -1.234638265190763e+04 + -1.236358798719477e+04 + -1.238081132944854e+04 + -1.239805268999999e+04 + -1.241531208224867e+04 + -1.243258952373459e+04 + -1.244988502746729e+04 + -1.246719859854622e+04 + -1.248453025000000e+04 + -1.250187999937352e+04 + -1.251924785997314e+04 + -1.253663384322633e+04 + -1.255403795942784e+04 + -1.257146022000000e+04 + -1.258890063824787e+04 + -1.260635923174130e+04 + -1.262383601348307e+04 + -1.264133098848631e+04 + -1.265884416999999e+04 + -1.267637557565735e+04 + -1.269392521755929e+04 + -1.271149310837373e+04 + -1.272907926320349e+04 + -1.274668368999999e+04 + -1.276430639533002e+04 + -1.278194739917555e+04 + -1.279960671868295e+04 + -1.281728436173200e+04 + -1.283498034000000e+04 + -1.285269466774105e+04 + -1.287042735763161e+04 + -1.288817842259290e+04 + -1.290594787633268e+04 + -1.292373573000000e+04 + -1.294154199365508e+04 + -1.295936667981993e+04 + -1.297720980264566e+04 + -1.299507137776778e+04 + -1.301295141999999e+04 + -1.303084994203483e+04 + -1.304876695061188e+04 + -1.306670245535235e+04 + -1.308465647270176e+04 + -1.310262901999999e+04 + -1.312062011245361e+04 + -1.313862975604251e+04 + -1.315665796055223e+04 + -1.317470474462057e+04 + -1.319277012000000e+04 + -1.321085409533674e+04 + -1.322895668661698e+04 + -1.324707790969205e+04 + -1.326521777718879e+04 + -1.328337630000000e+04 + -1.330155348851537e+04 + -1.331974935491385e+04 + -1.333796391362056e+04 + -1.335619718140170e+04 + -1.337444916999999e+04 + -1.339271988874384e+04 + -1.341100935082290e+04 + -1.342931756987852e+04 + -1.344764455872234e+04 + -1.346599032999999e+04 + -1.348435489664480e+04 + -1.350273827293541e+04 + -1.352114047045672e+04 + -1.353956149695641e+04 + -1.355800137000000e+04 + -1.357646010972569e+04 + -1.359493772045508e+04 + -1.361343421245217e+04 + -1.363194960986463e+04 + -1.365048392000000e+04 + -1.366903714425879e+04 + -1.368760930691950e+04 + -1.370620042690972e+04 + -1.372481050688227e+04 + -1.374343955999999e+04 + -1.376208760473736e+04 + -1.378075465005087e+04 + -1.379944070672178e+04 + -1.381814579185646e+04 + -1.383686991999999e+04 + -1.385561310323773e+04 + -1.387437535187516e+04 + -1.389315667691767e+04 + -1.391195709130158e+04 + -1.393077661000000e+04 + -1.394961524865979e+04 + -1.396847302075987e+04 + -1.398734993803961e+04 + -1.400624601054151e+04 + -1.402516125000000e+04 + -1.404409566989268e+04 + -1.406304928589068e+04 + -1.408202211294199e+04 + -1.410101416389908e+04 + -1.412002544999999e+04 + -1.413905598164077e+04 + -1.415810576964489e+04 + -1.417717482985645e+04 + -1.419626318438561e+04 + -1.421537083999999e+04 + -1.423449779870650e+04 + -1.425364408441530e+04 + -1.427280971628262e+04 + -1.429199469812906e+04 + -1.431119904000000e+04 + -1.433042275698022e+04 + -1.434966586475864e+04 + -1.436892837784737e+04 + -1.438821030874347e+04 + -1.440751167000000e+04 + -1.442683247464153e+04 + -1.444617273693482e+04 + -1.446553246845352e+04 + -1.448491167696990e+04 + -1.450431037999999e+04 + -1.452372859766022e+04 + -1.454316633455064e+04 + -1.456262360064903e+04 + -1.458210041914065e+04 + -1.460159679999998e+04 + -1.462111274782618e+04 + -1.464064828275210e+04 + -1.466020342227678e+04 + -1.467977817411577e+04 + -1.469937255000000e+04 + -1.471898656425745e+04 + -1.473862022905967e+04 + -1.475827355783788e+04 + -1.477794656667906e+04 + -1.479763927000000e+04 + -1.481735168022591e+04 + -1.483708380678435e+04 + -1.485683566151257e+04 + -1.487660726097236e+04 + -1.489639861999999e+04 + -1.491620975095499e+04 + -1.493604066191429e+04 + -1.495589136614776e+04 + -1.497576188555746e+04 + -1.499565223000000e+04 + -1.501556240451360e+04 + -1.503549242791256e+04 + -1.505544231896391e+04 + -1.507541209051985e+04 + -1.509540175000000e+04 + -1.511541130465759e+04 + -1.513544077375174e+04 + -1.515549017500820e+04 + -1.517555951933053e+04 + -1.519564881999999e+04 + -1.521575809081157e+04 + -1.523588734003694e+04 + -1.525603658036919e+04 + -1.527620583277191e+04 + -1.529639510999998e+04 + -1.531660442034102e+04 + -1.533683377728779e+04 + -1.535708319443415e+04 + -1.537735268372974e+04 + -1.539764226000000e+04 + -1.541795193921473e+04 + -1.543828173411788e+04 + -1.545863165713528e+04 + -1.547900172144165e+04 + -1.549939194000000e+04 + -1.551980232569260e+04 + -1.554023289178161e+04 + -1.556068365093051e+04 + -1.558115461506463e+04 + -1.560164579999999e+04 + -1.562215722197032e+04 + -1.564268888875843e+04 + -1.566324081295743e+04 + -1.568381301679686e+04 + -1.570440550999999e+04 + -1.572501829745089e+04 + -1.574565139909988e+04 + -1.576630483256334e+04 + -1.578697860613842e+04 + -1.580767273000000e+04 + -1.582838721786272e+04 + -1.584912209052793e+04 + -1.586987736246908e+04 + -1.589065303680525e+04 + -1.591144913000000e+04 + -1.593226566336701e+04 + -1.595310264072689e+04 + -1.597396007256353e+04 + -1.599483798518879e+04 + -1.601573638999999e+04 + -1.603665529105288e+04 + -1.605759470511345e+04 + -1.607855464838671e+04 + -1.609953513161670e+04 + -1.612053616999998e+04 + -1.614155778004294e+04 + -1.616259997081388e+04 + -1.618366275348782e+04 + -1.620474614514243e+04 + -1.622585016000000e+04 + -1.624697481007329e+04 + -1.626812010728422e+04 + -1.628928606592670e+04 + -1.631047270337715e+04 + -1.633168003000000e+04 + -1.635290805454565e+04 + -1.637415679800609e+04 + -1.639542627592296e+04 + -1.641671649164588e+04 + -1.643802745999998e+04 + -1.645935920179283e+04 + -1.648071172935930e+04 + -1.650208505431875e+04 + -1.652347919039644e+04 + -1.654489414999998e+04 + -1.656632994566724e+04 + -1.658778659427021e+04 + -1.660926410974226e+04 + -1.663076250029903e+04 + -1.665228178000000e+04 + -1.667382196574960e+04 + -1.669538306946903e+04 + -1.671696510420377e+04 + -1.673856808645374e+04 + -1.676019203000000e+04 + -1.678183694695306e+04 + -1.680350285053965e+04 + -1.682518975366077e+04 + -1.684689766850574e+04 + -1.686862660999998e+04 + -1.689037659405139e+04 + -1.691214763326874e+04 + -1.693393974023558e+04 + -1.695575292860696e+04 + -1.697758720999999e+04 + -1.699944259677470e+04 + -1.702131910964586e+04 + -1.704321676352935e+04 + -1.706513556209216e+04 + -1.708707552000000e+04 + -1.710903665790321e+04 + -1.713101898938551e+04 + -1.715302252479822e+04 + -1.717504727292061e+04 + -1.719709325000000e+04 + -1.721916047545785e+04 + -1.724124896174670e+04 + -1.726335872072246e+04 + -1.728548976595536e+04 + -1.730764210999998e+04 + -1.732981576548871e+04 + -1.735201074834581e+04 + -1.737422707347338e+04 + -1.739646475299950e+04 + -1.741872379999999e+04 + -1.744100422776899e+04 + -1.746330604747148e+04 + -1.748562927386205e+04 + -1.750797392735042e+04 + -1.753034002000000e+04 + -1.755272756012299e+04 + -1.757513656385630e+04 + -1.759756704698003e+04 + -1.762001902165879e+04 + -1.764249250000000e+04 + -1.766498749503509e+04 + -1.768750402279969e+04 + -1.771004209797560e+04 + -1.773260173223158e+04 + -1.775518293999998e+04 + -1.777778573725219e+04 + -1.780041013823985e+04 + -1.782305615380817e+04 + -1.784572379125812e+04 + -1.786841306999998e+04 + -1.789112401297027e+04 + -1.791385662520795e+04 + -1.793661091551944e+04 + -1.795938690506571e+04 + -1.798218461000000e+04 + -1.800500404157250e+04 + -1.802784520726593e+04 + -1.805070812126097e+04 + -1.807359280836476e+04 + -1.809649928000000e+04 + -1.811942754077938e+04 + -1.814237760512274e+04 + -1.816534949154946e+04 + -1.818834322012563e+04 + -1.821135879999998e+04 + -1.823439623670011e+04 + -1.825745554979138e+04 + -1.828053675712656e+04 + -1.830363986886144e+04 + -1.832676489999998e+04 + -1.834991186721417e+04 + -1.837308077964562e+04 + -1.839627164834952e+04 + -1.841948449011883e+04 + -1.844271932000000e+04 + -1.846597615248214e+04 + -1.848925500551537e+04 + -1.851255589141731e+04 + -1.853587881367952e+04 + -1.855922379000000e+04 + -1.858259084403384e+04 + -1.860597998459446e+04 + -1.862939122130964e+04 + -1.865282457086503e+04 + -1.867628004999998e+04 + -1.869975767367864e+04 + -1.872325745125265e+04 + -1.874677939400045e+04 + -1.877032351829797e+04 + -1.879388983999998e+04 + -1.881747837395618e+04 + -1.884108913346766e+04 + -1.886472213066709e+04 + -1.888837737699991e+04 + -1.891205489000000e+04 + -1.893575468867888e+04 + -1.895947678210836e+04 + -1.898322118083019e+04 + -1.900698790146692e+04 + -1.903077696000000e+04 + -1.905458837122149e+04 + -1.907842214817702e+04 + -1.910227830328199e+04 + -1.912615684887913e+04 + -1.915005779999998e+04 + -1.917398117252360e+04 + -1.919792697876641e+04 + -1.922189523162585e+04 + -1.924588594641727e+04 + -1.926989913999998e+04 + -1.929393482787026e+04 + -1.931799301676958e+04 + -1.934207372057390e+04 + -1.936617696590223e+04 + -1.939030276000000e+04 + -1.941445110358184e+04 + -1.943862202401437e+04 + -1.946281554213054e+04 + -1.948703165938676e+04 + -1.951127038999998e+04 + -1.953553175531375e+04 + -1.955981576784185e+04 + -1.958412243887696e+04 + -1.960845178142548e+04 + -1.963280380999998e+04 + -1.965717854010317e+04 + -1.968157598764983e+04 + -1.970599616682758e+04 + -1.973043908928781e+04 + -1.975490477000000e+04 + -1.977939322508017e+04 + -1.980390446612710e+04 + -1.982843850672890e+04 + -1.985299536464853e+04 + -1.987757505000000e+04 + -1.990217757132781e+04 + -1.992680295149982e+04 + -1.995145120910892e+04 + -1.997612235106852e+04 + -2.000081638999998e+04 + -2.002553334291399e+04 + -2.005027322692212e+04 + -2.007503605456619e+04 + -2.009982183242756e+04 + -2.012463057999998e+04 + -2.014946232050470e+04 + -2.017431705750974e+04 + -2.019919480128925e+04 + -2.022409557869796e+04 + -2.024901940000000e+04 + -2.027396626876084e+04 + -2.029893620825063e+04 + -2.032392923791251e+04 + -2.034894536415857e+04 + -2.037398460000000e+04 + -2.039904696273123e+04 + -2.042413246681671e+04 + -2.044924112749899e+04 + -2.047437296177629e+04 + -2.049952797999998e+04 + -2.052470619105544e+04 + -2.054990761544010e+04 + -2.057513226917888e+04 + -2.060038015795331e+04 + -2.062565129999998e+04 + -2.065094571854568e+04 + -2.067626342205840e+04 + -2.070160442063917e+04 + -2.072696873225083e+04 + -2.075235637000000e+04 + -2.077776734528838e+04 + -2.080320167696219e+04 + -2.082865938214449e+04 + -2.085414047244347e+04 + -2.087964496000000e+04 + -2.090517285805777e+04 + -2.093072418166662e+04 + -2.095629894648643e+04 + -2.098189716820891e+04 + -2.100751885999998e+04 + -2.103316403410750e+04 + -2.105883270565323e+04 + -2.108452489048020e+04 + -2.111024060421226e+04 + -2.113597985999998e+04 + -2.116174267011207e+04 + -2.118752904964539e+04 + -2.121333901446366e+04 + -2.123917258027344e+04 + -2.126502976000000e+04 + -2.129091056582378e+04 + -2.131681501406569e+04 + -2.134274311932949e+04 + -2.136869489244911e+04 + -2.139467035000000e+04 + -2.142066951097057e+04 + -2.144669238829770e+04 + -2.147273899344209e+04 + -2.149880933837191e+04 + -2.152490343999998e+04 + -2.155102131698669e+04 + -2.157716298229367e+04 + -2.160332844744082e+04 + -2.162951772437019e+04 + -2.165573082999998e+04 + -2.168196778300156e+04 + -2.170822859627157e+04 + -2.173451328139519e+04 + -2.176082185053812e+04 + -2.178715432000000e+04 + -2.181351070821111e+04 + -2.183989103150391e+04 + -2.186629530288904e+04 + -2.189272353140778e+04 + -2.191917573000000e+04 + -2.194565191497987e+04 + -2.197215210512397e+04 + -2.199867631637375e+04 + -2.202522455969623e+04 + -2.205179684999998e+04 + -2.207839320363563e+04 + -2.210501363148952e+04 + -2.213165814783550e+04 + -2.215832677350680e+04 + -2.218501951999998e+04 + -2.221173639532642e+04 + -2.223847741910094e+04 + -2.226524260850044e+04 + -2.229203197306979e+04 + -2.231884553000000e+04 + -2.234568329890952e+04 + -2.237254528774074e+04 + -2.239943150806158e+04 + -2.242634198094407e+04 + -2.245327672000000e+04 + -2.248023573585785e+04 + -2.250721904853940e+04 + -2.253422667413636e+04 + -2.256125861967753e+04 + -2.258831489999998e+04 + -2.261539553480252e+04 + -2.264250054093014e+04 + -2.266962993052693e+04 + -2.269678371062339e+04 + -2.272396189999998e+04 + -2.275116452153000e+04 + -2.277839158301363e+04 + -2.280564309534358e+04 + -2.283291907950545e+04 + -2.286021955000000e+04 + -2.288754451771620e+04 + -2.291489399812011e+04 + -2.294226800704672e+04 + -2.296966655917324e+04 + -2.299708967000000e+04 + -2.302453735490683e+04 + -2.305200962641978e+04 + -2.307950649734620e+04 + -2.310702798236498e+04 + -2.313457409999998e+04 + -2.316214486856879e+04 + -2.318974029613145e+04 + -2.321736039447026e+04 + -2.324500518448656e+04 + -2.327267467999998e+04 + -2.330036889175501e+04 + -2.332808783850466e+04 + -2.335583153764027e+04 + -2.338360000121357e+04 + -2.341139324000000e+04 + -2.343921126672951e+04 + -2.346705410394532e+04 + -2.349492176951476e+04 + -2.352281427072812e+04 + -2.355073162000000e+04 + -2.357867383414989e+04 + -2.360664093193761e+04 + -2.363463292919098e+04 + -2.366264983684289e+04 + -2.369069166999998e+04 + -2.371875844507638e+04 + -2.374685017193912e+04 + -2.377496686591387e+04 + -2.380310855200050e+04 + -2.383127523999998e+04 + -2.385946693403709e+04 + -2.388768365690810e+04 + -2.391592542840760e+04 + -2.394419225713039e+04 + -2.397248416000000e+04 + -2.400080115691363e+04 + -2.402914325571854e+04 + -2.405751046800998e+04 + -2.408590281513924e+04 + -2.411432031000000e+04 + -2.414276296293371e+04 + -2.417123079796694e+04 + -2.419972383203242e+04 + -2.422824206706797e+04 + -2.425678551999998e+04 + -2.428535421485334e+04 + -2.431394816200073e+04 + -2.434256737430352e+04 + -2.437121187300517e+04 + -2.439988166999998e+04 + -2.442857677325164e+04 + -2.445729720103540e+04 + -2.448604297194451e+04 + -2.451481410082247e+04 + -2.454361060000000e+04 + -2.457243248182022e+04 + -2.460127976431730e+04 + -2.463015246522100e+04 + -2.465905059942621e+04 + -2.468797417999998e+04 + -2.471692321945616e+04 + -2.474589773222362e+04 + -2.477489773480251e+04 + -2.480392324581322e+04 + -2.483297427999998e+04 + -2.486205084917968e+04 + -2.489115296405446e+04 + -2.492028064024748e+04 + -2.494943390068509e+04 + -2.497861275999998e+04 + -2.500781722863925e+04 + -2.503704732334474e+04 + -2.506630305872901e+04 + -2.509558444448151e+04 + -2.512489150000000e+04 + -2.515422424757818e+04 + -2.518358269497819e+04 + -2.521296685328307e+04 + -2.524237674371633e+04 + -2.527181237999998e+04 + -2.530127377273551e+04 + -2.533076094162915e+04 + -2.536027390311031e+04 + -2.538981266594794e+04 + -2.541937724999998e+04 + -2.544896767801531e+04 + -2.547858395406606e+04 + -2.550822608802068e+04 + -2.553789410507042e+04 + -2.556758802000000e+04 + -2.559730784246711e+04 + -2.562705359189478e+04 + -2.565682528525429e+04 + -2.568662293247029e+04 + -2.571644655000000e+04 + -2.574629615713407e+04 + -2.577617176604459e+04 + -2.580607338891535e+04 + -2.583600104095400e+04 + -2.586595473999998e+04 + -2.589593450400127e+04 + -2.592594034522632e+04 + -2.595597227610649e+04 + -2.598603031176017e+04 + -2.601611446999998e+04 + -2.604622476879967e+04 + -2.607636122042706e+04 + -2.610652383730750e+04 + -2.613671263455693e+04 + -2.616692763000000e+04 + -2.619716884161440e+04 + -2.622743628160562e+04 + -2.625772996246335e+04 + -2.628804989951923e+04 + -2.631839611000000e+04 + -2.634876861165059e+04 + -2.637916741999917e+04 + -2.640959254907723e+04 + -2.644004401148305e+04 + -2.647052181999998e+04 + -2.650102598983489e+04 + -2.653155654491748e+04 + -2.656211350103963e+04 + -2.659269685960091e+04 + -2.662330663999998e+04 + -2.665394286925502e+04 + -2.668460555517195e+04 + -2.671529470758218e+04 + -2.674601034670874e+04 + -2.677675249000000e+04 + -2.680752115154265e+04 + -2.683831634157976e+04 + -2.686913807564717e+04 + -2.689998637786448e+04 + -2.693086126000000e+04 + -2.696176272916689e+04 + -2.699269080763921e+04 + -2.702364551500381e+04 + -2.705462686113689e+04 + -2.708563485999998e+04 + -2.711666952809900e+04 + -2.714773087942122e+04 + -2.717881892991400e+04 + -2.720993369901594e+04 + -2.724107519999998e+04 + -2.727224344400496e+04 + -2.730343845029767e+04 + -2.733466023562860e+04 + -2.736590881019403e+04 + -2.739718419000000e+04 + -2.742848639412166e+04 + -2.745981543741902e+04 + -2.749117133427855e+04 + -2.752255410001475e+04 + -2.755396375000000e+04 + -2.758540029934807e+04 + -2.761686376223195e+04 + -2.764835415476579e+04 + -2.767987149599797e+04 + -2.771141579999997e+04 + -2.774298707825893e+04 + -2.777458534547961e+04 + -2.780621061813955e+04 + -2.783786291408934e+04 + -2.786954224999998e+04 + -2.790124864062599e+04 + -2.793298209650868e+04 + -2.796474263269793e+04 + -2.799653027215609e+04 + -2.802834503000000e+04 + -2.806018691678819e+04 + -2.809205594694776e+04 + -2.812395213748840e+04 + -2.815587550739401e+04 + -2.818782607000000e+04 + -2.821980383662756e+04 + -2.825180882556652e+04 + -2.828384105454396e+04 + -2.831590053775406e+04 + -2.834798728999998e+04 + -2.838010132665882e+04 + -2.841224266303304e+04 + -2.844441131398464e+04 + -2.847660729395342e+04 + -2.850883061999998e+04 + -2.854108131015271e+04 + -2.857335937943549e+04 + -2.860566484233479e+04 + -2.863799771378746e+04 + -2.867035801000000e+04 + -2.870274574643904e+04 + -2.873516093256026e+04 + -2.876760358436649e+04 + -2.880007372909855e+04 + -2.883257138000000e+04 + -2.886509654365616e+04 + -2.889764923870952e+04 + -2.893022948372628e+04 + -2.896283729252707e+04 + -2.899547267999998e+04 + -2.902813566168059e+04 + -2.906082625190441e+04 + -2.909354446650575e+04 + -2.912629032389195e+04 + -2.915906383999998e+04 + -2.919186502919167e+04 + -2.922469390650106e+04 + -2.925755048733507e+04 + -2.929043478725063e+04 + -2.932334682000000e+04 + -2.935628660019129e+04 + -2.938925415043921e+04 + -2.942224948806936e+04 + -2.945527261979538e+04 + -2.948832356000000e+04 + -2.952140232838030e+04 + -2.955450894361826e+04 + -2.958764342089699e+04 + -2.962080577090842e+04 + -2.965399600999997e+04 + -2.968721415715336e+04 + -2.972046022629084e+04 + -2.975373423280592e+04 + -2.978703619592538e+04 + -2.982036612999997e+04 + -2.985372404652352e+04 + -2.988710995909819e+04 + -2.992052388535540e+04 + -2.995396584761164e+04 + -2.998743586000000e+04 + -3.002093393306212e+04 + -3.005446008516229e+04 + -3.008801433415196e+04 + -3.012159669410924e+04 + -3.015520717999998e+04 + -3.018884580728562e+04 + -3.022251259030155e+04 + -3.025620754490144e+04 + -3.028993068950620e+04 + -3.032368203999997e+04 + -3.035746161072607e+04 + -3.039126941699663e+04 + -3.042510547393189e+04 + -3.045896979612689e+04 + -3.049286239999998e+04 + -3.052678330335746e+04 + -3.056073252467363e+04 + -3.059471007984014e+04 + -3.062871598061521e+04 + -3.066275024000000e+04 + -3.069681287356475e+04 + -3.073090390293319e+04 + -3.076502334621974e+04 + -3.079917121404217e+04 + -3.083334751999997e+04 + -3.086755228023463e+04 + -3.090178551167921e+04 + -3.093604723225209e+04 + -3.097033746063074e+04 + -3.100465620999998e+04 + -3.103900349175437e+04 + -3.107337932446759e+04 + -3.110778372555016e+04 + -3.114221670819769e+04 + -3.117667828999997e+04 + -3.121116848957679e+04 + -3.124568731744623e+04 + -3.128023478769976e+04 + -3.131481092268889e+04 + -3.134941574000000e+04 + -3.138404925321087e+04 + -3.141871147408727e+04 + -3.145340241670125e+04 + -3.148812209933996e+04 + -3.152287053999997e+04 + -3.155764775577736e+04 + -3.159245376185456e+04 + -3.162728857279384e+04 + -3.166215220315186e+04 + -3.169704466999997e+04 + -3.173196599132658e+04 + -3.176691618227545e+04 + -3.180189525721778e+04 + -3.183690323068162e+04 + -3.187194012000000e+04 + -3.190700594323912e+04 + -3.194210071422084e+04 + -3.197722444868163e+04 + -3.201237716636405e+04 + -3.204755888000000e+04 + -3.208276960047787e+04 + -3.211800935023939e+04 + -3.215327814773632e+04 + -3.218857600158763e+04 + -3.222390292999997e+04 + -3.225925895535199e+04 + -3.229464408973093e+04 + -3.233005834462857e+04 + -3.236550173499735e+04 + -3.240097427999997e+04 + -3.243647600017841e+04 + -3.247200691143220e+04 + -3.250756702669824e+04 + -3.254315635678932e+04 + -3.257877492000000e+04 + -3.261442273792899e+04 + -3.265009982555360e+04 + -3.268580619650297e+04 + -3.272154186523808e+04 + -3.275730685000000e+04 + -3.279310116911321e+04 + -3.282892483177912e+04 + -3.286477785354587e+04 + -3.290066026202923e+04 + -3.293657206999998e+04 + -3.297251328378459e+04 + -3.300848392497173e+04 + -3.304448401407080e+04 + -3.308051356412061e+04 + -3.311657258999998e+04 + -3.315266110752813e+04 + -3.318877912993135e+04 + -3.322492667413289e+04 + -3.326110376297835e+04 + -3.329731041000000e+04 + -3.333354662539115e+04 + -3.336981243131684e+04 + -3.340610784669931e+04 + -3.344243288124504e+04 + -3.347878755000000e+04 + -3.351517187197135e+04 + -3.355158586607149e+04 + -3.358802954742114e+04 + -3.362450292603845e+04 + -3.366100601999997e+04 + -3.369753885113907e+04 + -3.373410143432544e+04 + -3.377069378324004e+04 + -3.380731591267414e+04 + -3.384396783999997e+04 + -3.388064958319757e+04 + -3.391736115628571e+04 + -3.395410257481684e+04 + -3.399087385786547e+04 + -3.402767502000000e+04 + -3.406450607498593e+04 + -3.410136704569621e+04 + -3.413825794940452e+04 + -3.417517879208190e+04 + -3.421212959000000e+04 + -3.424911036577375e+04 + -3.428612113814977e+04 + -3.432316192150666e+04 + -3.436023272579379e+04 + -3.439733356999998e+04 + -3.443446447621852e+04 + -3.447162545467578e+04 + -3.450881651888796e+04 + -3.454603769159935e+04 + -3.458328898999997e+04 + -3.462057042742187e+04 + -3.465788201822977e+04 + -3.469522377879945e+04 + -3.473259572783916e+04 + -3.476999788000000e+04 + -3.480743024920274e+04 + -3.484489285737491e+04 + -3.488238572274076e+04 + -3.491990885516477e+04 + -3.495746227000000e+04 + -3.499504598627476e+04 + -3.503266002163332e+04 + -3.507030439252002e+04 + -3.510797911399110e+04 + -3.514568419999997e+04 + -3.518341966538968e+04 + -3.522118553105182e+04 + -3.525898181605689e+04 + -3.529680853433678e+04 + -3.533466569999997e+04 + -3.537255332834070e+04 + -3.541047143775194e+04 + -3.544842004484196e+04 + -3.548639916273562e+04 + -3.552440881000000e+04 + -3.556244900769237e+04 + -3.560051977192599e+04 + -3.563862111528000e+04 + -3.567675304773126e+04 + -3.571491559000000e+04 + -3.575310876677514e+04 + -3.579133259025522e+04 + -3.582958707123636e+04 + -3.586787222387752e+04 + -3.590618806999997e+04 + -3.594453463328037e+04 + -3.598291192576281e+04 + -3.602131995866808e+04 + -3.605975874687618e+04 + -3.609822830999997e+04 + -3.613672866872384e+04 + -3.617525983663845e+04 + -3.621382182906358e+04 + -3.625241466608051e+04 + -3.629103836000000e+04 + -3.632969292147818e+04 + -3.636837837559645e+04 + -3.640709474306420e+04 + -3.644584203277781e+04 + -3.648462026000000e+04 + -3.652342944409004e+04 + -3.656226960170399e+04 + -3.660114075023598e+04 + -3.664004290875195e+04 + -3.667897608999997e+04 + -3.671794030512006e+04 + -3.675693557547587e+04 + -3.679596192058374e+04 + -3.683501935339895e+04 + -3.687410788999997e+04 + -3.691322754870546e+04 + -3.695237834723054e+04 + -3.699156030212950e+04 + -3.703077342834224e+04 + -3.707001774000000e+04 + -3.710929325203417e+04 + -3.714859998429457e+04 + -3.718793795692067e+04 + -3.722730718808447e+04 + -3.726670768999997e+04 + -3.730613947399677e+04 + -3.734560256273038e+04 + -3.738509697431393e+04 + -3.742462271654350e+04 + -3.746417980999997e+04 + -3.750376828013417e+04 + -3.754338813660564e+04 + -3.758303939267375e+04 + -3.762272207245150e+04 + -3.766243618999997e+04 + -3.770218175518502e+04 + -3.774195878945678e+04 + -3.778176731257176e+04 + -3.782160733741773e+04 + -3.786147888000000e+04 + -3.790138195864309e+04 + -3.794131659132003e+04 + -3.798128279431332e+04 + -3.802128058166662e+04 + -3.806130996999997e+04 + -3.810137097737330e+04 + -3.814146362046005e+04 + -3.818158791697343e+04 + -3.822174388615621e+04 + -3.826193153999997e+04 + -3.830215088938532e+04 + -3.834240195971708e+04 + -3.838268477130731e+04 + -3.842299933188884e+04 + -3.846334565999997e+04 + -3.850372377842891e+04 + -3.854413369633216e+04 + -3.858457542816681e+04 + -3.862504900090371e+04 + -3.866555443000000e+04 + -3.870609172486966e+04 + -3.874666090332934e+04 + -3.878726198409722e+04 + -3.882789498399841e+04 + -3.886855991999997e+04 + -3.890925680946781e+04 + -3.894998567054054e+04 + -3.899074651957592e+04 + -3.903153937021765e+04 + -3.907236423999997e+04 + -3.911322114736439e+04 + -3.915411010384250e+04 + -3.919503112747372e+04 + -3.923598424716172e+04 + -3.927696946999997e+04 + -3.931798680038228e+04 + -3.935903628964794e+04 + -3.940011793259103e+04 + -3.944123164809230e+04 + -3.948237770000000e+04 + -3.952355620675491e+04 + -3.956476584662273e+04 + -3.960600801403134e+04 + -3.964728735507886e+04 + -3.968859104999997e+04 + 1.819886919039960e+01 + 1.816405130921966e+01 + 1.812918280091599e+01 + 1.809426424653407e+01 + 1.805929622711938e+01 + 1.802427932371738e+01 + 1.798921411737356e+01 + 1.795410118913339e+01 + 1.791894112004234e+01 + 1.788373449114589e+01 + 1.784848188348950e+01 + 1.781318387811866e+01 + 1.777784105607885e+01 + 1.774245399841551e+01 + 1.770702328617416e+01 + 1.767154950040025e+01 + 1.763603322213927e+01 + 1.760047503243666e+01 + 1.756487551233793e+01 + 1.752923524288855e+01 + 1.749355480513398e+01 + 1.745783478011969e+01 + 1.742207574889119e+01 + 1.738627829249391e+01 + 1.735044299197335e+01 + 1.731457042837498e+01 + 1.727866118274428e+01 + 1.724271583612671e+01 + 1.720673496956776e+01 + 1.717071916411290e+01 + 1.713466900080759e+01 + 1.709858506069732e+01 + 1.706246792482757e+01 + 1.702631817424381e+01 + 1.699013638999150e+01 + 1.695392315311613e+01 + 1.691767904466317e+01 + 1.688140464567809e+01 + 1.684510053720637e+01 + 1.680876730029348e+01 + 1.677240551598491e+01 + 1.673601576532611e+01 + 1.669959862936258e+01 + 1.666315468913977e+01 + 1.662668452570318e+01 + 1.659018872009826e+01 + 1.655366785337049e+01 + 1.651712250656536e+01 + 1.648055326072833e+01 + 1.644396069690488e+01 + 1.640734539614049e+01 + 1.637070793948063e+01 + 1.633404890797076e+01 + 1.629736888265638e+01 + 1.626066844458294e+01 + 1.622394817479593e+01 + 1.618720865434082e+01 + 1.615045046426309e+01 + 1.611367418560821e+01 + 1.607688039942165e+01 + 1.604006968674890e+01 + 1.600324262863542e+01 + 1.596639980612668e+01 + 1.592954180026817e+01 + 1.589266919210536e+01 + 1.585578256268371e+01 + 1.581888249304872e+01 + 1.578196956424585e+01 + 1.574504435732057e+01 + 1.570810745331836e+01 + 1.567115943328470e+01 + 1.563420087826506e+01 + 1.559723236930490e+01 + 1.556025448744972e+01 + 1.552326781374499e+01 + 1.548627292923617e+01 + 1.544927041496875e+01 + 1.541226085198818e+01 + 1.537524482133997e+01 + 1.533822290406956e+01 + 1.530119568122245e+01 + 1.526416373384410e+01 + 1.522712764298000e+01 + 1.519008798967561e+01 + 1.515304535497640e+01 + 1.511600031992786e+01 + 1.507895346557546e+01 + 1.504190537296467e+01 + 1.500485662314096e+01 + 1.496780779714983e+01 + 1.493075947603672e+01 + 1.489371224084713e+01 + 1.485666667262652e+01 + 1.481962335242037e+01 + 1.478258286127416e+01 + 1.474554578023335e+01 + 1.470851269034343e+01 + 1.467148417264987e+01 + 1.463446080819813e+01 + 1.459744317803371e+01 + 1.456043186320207e+01 + 1.452342744474869e+01 + 1.448643050371903e+01 + 1.444944162115858e+01 + 1.441246137811282e+01 + 1.437549035562720e+01 + 1.433852913474722e+01 + 1.430157829651834e+01 + 1.426463842198604e+01 + 1.422771009219579e+01 + 1.419079388819307e+01 + 1.415389039102334e+01 + 1.411700018173210e+01 + 1.408012384136481e+01 + 1.404326195096694e+01 + 1.400641509158398e+01 + 1.396958384426139e+01 + 1.393276879004464e+01 + 1.389597050997923e+01 + 1.385918958511061e+01 + 1.382242659648426e+01 + 1.378568212514566e+01 + 1.374895675214029e+01 + 1.371225105851361e+01 + 1.367556562531110e+01 + 1.363890103357824e+01 + 1.360225786436050e+01 + 1.356563669870335e+01 + 1.352903811765228e+01 + 1.349246270225274e+01 + 1.345591103355023e+01 + 1.341938369259021e+01 + 1.338288126041816e+01 + 1.334640431807955e+01 + 1.330995344661985e+01 + 1.327352922708455e+01 + 1.323713224051912e+01 + 1.320076306796902e+01 + 1.316442229047974e+01 + 1.312811048909675e+01 + 1.309182824486553e+01 + 1.305557613883154e+01 + 1.301935475204027e+01 + 1.298316466553718e+01 + 1.294700646036776e+01 + 1.291088071757748e+01 + 1.287478801821181e+01 + 1.283872894331622e+01 + 1.280270407393619e+01 + 1.276671399111720e+01 + 1.273075927590472e+01 + 1.269484050934423e+01 + 1.265895827248119e+01 + 1.262311314636109e+01 + 1.258730571202939e+01 + 1.255153655053158e+01 + 1.251580624291313e+01 + 1.248011537021950e+01 + 1.244446451349619e+01 + 1.240885425378865e+01 + 1.237328517214237e+01 + 1.233775784960283e+01 + 1.230227286721548e+01 + 1.226683080602582e+01 + 1.223143224707931e+01 + 1.219607777142142e+01 + 1.216076796009764e+01 + 1.212550339415344e+01 + 1.209028465463429e+01 + 1.205511232258567e+01 + 1.201998697905305e+01 + 1.198490920508190e+01 + 1.194987958171770e+01 + 1.191489869000593e+01 + 1.187996711099206e+01 + 1.184508542572157e+01 + 1.181025421523992e+01 + 1.177547406059259e+01 + 1.174074554282507e+01 + 1.170606924298281e+01 + 1.167144574211130e+01 + 1.163687562125602e+01 + 1.160235946146243e+01 + 1.156789784377601e+01 + 1.153349134924223e+01 + 1.149914055890657e+01 + 1.146484605381452e+01 + 1.143060841501152e+01 + 1.139642822354308e+01 + 1.136230606045464e+01 + 1.132824250679170e+01 + 1.129423814359974e+01 + 1.126029355192421e+01 + 1.122640931281059e+01 + 1.119258600730437e+01 + 1.115882421645102e+01 + 1.112512452129600e+01 + 1.109148750288479e+01 + 1.105791374226288e+01 + 1.102440382047573e+01 + 1.099095831856882e+01 + 1.095757781758762e+01 + 1.092426289857761e+01 + 1.089101414258426e+01 + 1.085783213065305e+01 + 1.082471744382945e+01 + 1.079167066315892e+01 + 1.075869236968697e+01 + 1.072578314445905e+01 + 1.069294356852064e+01 + 1.066017422291721e+01 + 1.062747568869424e+01 + 1.059484854689720e+01 + 1.056229337857157e+01 + 1.052981076476282e+01 + 1.049740128651643e+01 + 1.046506552487787e+01 + 1.043280406089261e+01 + 1.040061747560614e+01 + 1.036850635006392e+01 + 1.033647126531143e+01 + 1.030451280239414e+01 + 1.027263154235754e+01 + 1.024082806624708e+01 + 1.020910295510825e+01 + 1.017745678998653e+01 + 1.014589015192738e+01 + 1.011440362197629e+01 + 1.008299778117872e+01 + 1.005167321058014e+01 + 1.002043049122605e+01 + 9.989270204161905e+00 + 9.958192930433185e+00 + 9.927199251085362e+00 + 9.896289747163912e+00 + 9.865464999714312e+00 + 9.834725589782032e+00 + 9.804072098412551e+00 + 9.773505106651340e+00 + 9.743025195543877e+00 + 9.712632946135633e+00 + 9.682328939472086e+00 + 9.652113756598707e+00 + 9.621987978560973e+00 + 9.591952186404358e+00 + 9.562006961174337e+00 + 9.532152883916382e+00 + 9.502390535675973e+00 + 9.472720497498580e+00 + 9.443143350429677e+00 + 9.413659675514742e+00 + 9.384270053799249e+00 + 9.354975066328668e+00 + 9.325775294148480e+00 + 9.296671318304156e+00 + 9.267663719841170e+00 + 9.238753079804999e+00 + 9.209939979241115e+00 + 9.181224999194994e+00 + 9.152608720712111e+00 + 9.124091724837941e+00 + 9.095674592617954e+00 + 9.067357905097630e+00 + 9.039142243322441e+00 + 9.011028188337864e+00 + 8.983016292445305e+00 + 8.955106689123129e+00 + 8.927299195109086e+00 + 8.899593619314148e+00 + 8.871989770649284e+00 + 8.844487458025464e+00 + 8.817086490353660e+00 + 8.789786676544839e+00 + 8.762587825509975e+00 + 8.735489746160036e+00 + 8.708492247405990e+00 + 8.681595138158814e+00 + 8.654798227329470e+00 + 8.628101323828931e+00 + 8.601504236568170e+00 + 8.575006774458155e+00 + 8.548608746409855e+00 + 8.522309961334242e+00 + 8.496110228142285e+00 + 8.470009355744955e+00 + 8.444007153053219e+00 + 8.418103428978053e+00 + 8.392297992430423e+00 + 8.366590652321301e+00 + 8.340981217561657e+00 + 8.315469497062459e+00 + 8.290055299734679e+00 + 8.264738434489287e+00 + 8.239518710237252e+00 + 8.214395935889545e+00 + 8.189369920357137e+00 + 8.164440472550996e+00 + 8.139607401382095e+00 + 8.114870515761401e+00 + 8.090229624599887e+00 + 8.065684536808520e+00 + 8.041235061298275e+00 + 8.016881006980118e+00 + 7.992622182765018e+00 + 7.968458397563950e+00 + 7.944389460287880e+00 + 7.920415179847782e+00 + 7.896535365154623e+00 + 7.872749825119373e+00 + 7.849058368653004e+00 + 7.825460804666484e+00 + 7.801956942070785e+00 + 7.778546589776878e+00 + 7.755229556695729e+00 + 7.732005651738313e+00 + 7.708874683815599e+00 + 7.685836461838553e+00 + 7.662890794718152e+00 + 7.640037491365358e+00 + 7.617276360691148e+00 + 7.594607211606489e+00 + 7.572029853022352e+00 + 7.549544093849708e+00 + 7.527149742999527e+00 + 7.504846609382777e+00 + 7.482634501910429e+00 + 7.460513229493455e+00 + 7.438482601042820e+00 + 7.416542425469502e+00 + 7.394692511684465e+00 + 7.372932668598682e+00 + 7.351262705123121e+00 + 7.329682430168755e+00 + 7.308191652646553e+00 + 7.286790181467484e+00 + 7.265477825542517e+00 + 7.244254393782626e+00 + 7.223119695098779e+00 + 7.202073538401945e+00 + 7.181115732603097e+00 + 7.160246086613204e+00 + 7.139464409343233e+00 + 7.118770509704159e+00 + 7.098164196606950e+00 + 7.077645278962576e+00 + 7.057213565682006e+00 + 7.036868865676214e+00 + 7.016610987856166e+00 + 6.996439741132834e+00 + 6.976354934417186e+00 + 6.956356376620197e+00 + 6.936443876652832e+00 + 6.916617243426063e+00 + 6.896876285850862e+00 + 6.877220812838198e+00 + 6.857650633299039e+00 + 6.838165556144358e+00 + 6.818765390285123e+00 + 6.799449944632306e+00 + 6.780219028096876e+00 + 6.761072449589803e+00 + 6.742010018022058e+00 + 6.723031542304612e+00 + 6.704136831348434e+00 + 6.685325694064492e+00 + 6.666597939363759e+00 + 6.647953376157203e+00 + 6.629391813355797e+00 + 6.610913059870509e+00 + 6.592516924612310e+00 + 6.574203216492170e+00 + 6.555971744421058e+00 + 6.537822317309946e+00 + 6.519754744069803e+00 + 6.501768833611600e+00 + 6.483864394846306e+00 + 6.466041236684891e+00 + 6.448299168038327e+00 + 6.430637997817582e+00 + 6.413057534933630e+00 + 6.395557588297437e+00 + 6.378137966819973e+00 + 6.360798479412210e+00 + 6.343538934985117e+00 + 6.326359142449666e+00 + 6.309258910716826e+00 + 6.292238048697567e+00 + 6.275296365302859e+00 + 6.258433669443673e+00 + 6.241649770030978e+00 + 6.224944475975745e+00 + 6.208317596188945e+00 + 6.191768939581546e+00 + 6.175298315064517e+00 + 6.158905531548833e+00 + 6.142590397945462e+00 + 6.126352723165372e+00 + 6.110192316119535e+00 + 6.094108985718921e+00 + 6.078102540874499e+00 + 6.062172790497243e+00 + 6.046319543498117e+00 + 6.030542608788095e+00 + 6.014841795278149e+00 + 5.999216911879244e+00 + 5.983667767502356e+00 + 5.968194171058448e+00 + 5.952795931458497e+00 + 5.937472857613470e+00 + 5.922224758434336e+00 + 5.907051442832068e+00 + 5.891952719717633e+00 + 5.876928398002004e+00 + 5.861978286596150e+00 + 5.847102194411042e+00 + 5.832299930357648e+00 + 5.817571303346941e+00 + 5.802916122289888e+00 + 5.788334196097460e+00 + 5.773825333680629e+00 + 5.759389343950365e+00 + 5.745026035817635e+00 + 5.730735218193414e+00 + 5.716516699988668e+00 + 5.702370290114368e+00 + 5.688295797481486e+00 + 5.674293031000990e+00 + 5.660361799583851e+00 + 5.646501912141039e+00 + 5.632713177583526e+00 + 5.618995404822279e+00 + 5.605348402768269e+00 + 5.591771980332467e+00 + 5.578265946425844e+00 + 5.564830109959368e+00 + 5.551464279844009e+00 + 5.538168264990741e+00 + 5.524941874310529e+00 + 5.511784916714348e+00 + 5.498697201113163e+00 + 5.485678536417949e+00 + 5.472728731539672e+00 + 5.459847595389306e+00 + 5.447031950668924e+00 + 5.434258417587328e+00 + 5.421509378931341e+00 + 5.408790202589383e+00 + 5.396101725502927e+00 + 5.383442552149385e+00 + 5.370812984345576e+00 + 5.358213038980171e+00 + 5.345642546622406e+00 + 5.333101463094469e+00 + 5.320589727671893e+00 + 5.308107264570896e+00 + 5.295654006874094e+00 + 5.283229887334054e+00 + 5.270834837715376e+00 + 5.258468789944041e+00 + 5.246131676000384e+00 + 5.233823428164784e+00 + 5.221543979540164e+00 + 5.209293263422158e+00 + 5.197071212820756e+00 + 5.184877759775030e+00 + 5.172712837552656e+00 + 5.160576380473933e+00 + 5.148468322098453e+00 + 5.136388595958436e+00 + 5.124337135726472e+00 + 5.112313875048248e+00 + 5.100318748362747e+00 + 5.088351690727316e+00 + 5.076412636057685e+00 + 5.064501518839616e+00 + 5.052618274474592e+00 + 5.040762837425790e+00 + 5.028935142799786e+00 + 5.017135126690616e+00 + 5.005362723409764e+00 + 4.993617868159293e+00 + 4.981900498164465e+00 + 4.970210548621045e+00 + 4.958547954855285e+00 + 4.946912653797631e+00 + 4.935304581602165e+00 + 4.923723674430540e+00 + 4.912169869219803e+00 + 4.900643102722201e+00 + 4.889143311682699e+00 + 4.877670433140317e+00 + 4.866224404407401e+00 + 4.854805162838305e+00 + 4.843412645620055e+00 + 4.832046790470948e+00 + 4.820707535433044e+00 + 4.809394818449872e+00 + 4.798108577205163e+00 + 4.786848749644187e+00 + 4.775615274885523e+00 + 4.764408091170769e+00 + 4.753227136475494e+00 + 4.742072350593207e+00 + 4.730943672320340e+00 + 4.719841039761676e+00 + 4.708764392793848e+00 + 4.697713670947889e+00 + 4.686688813308958e+00 + 4.675689760077749e+00 + 4.664716450776882e+00 + 4.653768824355144e+00 + 4.642846821849820e+00 + 4.631950383864691e+00 + 4.621079449911801e+00 + 4.610233960271208e+00 + 4.599413855814052e+00 + 4.588619077726644e+00 + 4.577849566594633e+00 + 4.567105263324451e+00 + 4.556386109499332e+00 + 4.545692045920192e+00 + 4.535023013830533e+00 + 4.524378955558838e+00 + 4.513759812087895e+00 + 4.503165524982657e+00 + 4.492596037765430e+00 + 4.482051291649467e+00 + 4.471531227881555e+00 + 4.461035790191556e+00 + 4.450564920902143e+00 + 4.440118561957067e+00 + 4.429696656605228e+00 + 4.419299148008472e+00 + 4.408925979093191e+00 + 4.398577092681390e+00 + 4.388252432408600e+00 + 4.377951941955149e+00 + 4.367675563647617e+00 + 4.357423241911437e+00 + 4.347194921954801e+00 + 4.336990545878941e+00 + 4.326810057907454e+00 + 4.316653403787664e+00 + 4.306520526395858e+00 + 4.296411370095723e+00 + 4.286325880845187e+00 + 4.276264002847886e+00 + 4.266225680614548e+00 + 4.256210859410026e+00 + 4.246219484255977e+00 + 4.236251500111860e+00 + 4.226306852214246e+00 + 4.216385487075638e+00 + 4.206487350454588e+00 + 4.196612386873075e+00 + 4.186760542346591e+00 + 4.176931763412669e+00 + 4.167125996517556e+00 + 4.157343187931778e+00 + 4.147583283659849e+00 + 4.137846229598449e+00 + 4.128131973158434e+00 + 4.118440461620333e+00 + 4.108771640965568e+00 + 4.099125458381889e+00 + 4.089501861348100e+00 + 4.079900796648449e+00 + 4.070322211621146e+00 + 4.060766054068578e+00 + 4.051232271987449e+00 + 4.041720812903752e+00 + 4.032231624389123e+00 + 4.022764654839604e+00 + 4.013319852102482e+00 + 4.003897163974346e+00 + 3.994496539375360e+00 + 3.985117927005318e+00 + 3.975761275162846e+00 + 3.966426531955261e+00 + 3.957113646635731e+00 + 3.947822569087183e+00 + 3.938553247905866e+00 + 3.929305631685672e+00 + 3.920079669524097e+00 + 3.910875311427285e+00 + 3.901692507178291e+00 + 3.892531206204140e+00 + 3.883391358396833e+00 + 3.874272913657344e+00 + 3.865175821793835e+00 + 3.856100032879644e+00 + 3.847045497225832e+00 + 3.838012165320722e+00 + 3.828999987558789e+00 + 3.820008914413926e+00 + 3.811038896595444e+00 + 3.802089885254187e+00 + 3.793161831237620e+00 + 3.784254684890362e+00 + 3.775368398252875e+00 + 3.766502923100440e+00 + 3.757658209579520e+00 + 3.748834209295840e+00 + 3.740030874433959e+00 + 3.731248156753105e+00 + 3.722486008117639e+00 + 3.713744380387517e+00 + 3.705023225280349e+00 + 3.696322495070079e+00 + 3.687642142318910e+00 + 3.678982119432947e+00 + 3.670342378801804e+00 + 3.661722872867589e+00 + 3.653123554235752e+00 + 3.644544376023881e+00 + 3.635985291634798e+00 + 3.627446254192749e+00 + 3.618927216218316e+00 + 3.610428130335504e+00 + 3.601948951051534e+00 + 3.593489632231203e+00 + 3.585050126838500e+00 + 3.576630388153148e+00 + 3.568230370457688e+00 + 3.559850028500160e+00 + 3.551489315016799e+00 + 3.543148183918080e+00 + 3.534826590807976e+00 + 3.526524489935861e+00 + 3.518241835011243e+00 + 3.509978579991082e+00 + 3.501734680809670e+00 + 3.493510092697708e+00 + 3.485304769188242e+00 + 3.477118665289576e+00 + 3.468951736765370e+00 + 3.460803939385451e+00 + 3.452675227557384e+00 + 3.444565556219660e+00 + 3.436474882078371e+00 + 3.428403160505571e+00 + 3.420350346590145e+00 + 3.412316396304707e+00 + 3.404301265792477e+00 + 3.396304911176187e+00 + 3.388327288491305e+00 + 3.380368354160161e+00 + 3.372428064623584e+00 + 3.364506375833607e+00 + 3.356603244211646e+00 + 3.348718626640740e+00 + 3.340852480236101e+00 + 3.333004761693424e+00 + 3.325175427644580e+00 + 3.317364435527293e+00 + 3.309571742493616e+00 + 3.301797305408617e+00 + 3.294041081414062e+00 + 3.286303028194412e+00 + 3.278583103726798e+00 + 3.270881265443555e+00 + 3.263197471011956e+00 + 3.255531678390248e+00 + 3.247883845209004e+00 + 3.240253929642225e+00 + 3.232641890347850e+00 + 3.225047684908113e+00 + 3.217471271425511e+00 + 3.209912608923088e+00 + 3.202371655667555e+00 + 3.194848370181972e+00 + 3.187342711658790e+00 + 3.179854638881827e+00 + 3.172384110380890e+00 + 3.164931084707051e+00 + 3.157495521384956e+00 + 3.150077380058776e+00 + 3.142676619877814e+00 + 3.135293200073600e+00 + 3.127927079946843e+00 + 3.120578218905895e+00 + 3.113246577218404e+00 + 3.105932114937815e+00 + 3.098634790846833e+00 + 3.091354565058342e+00 + 3.084091398271724e+00 + 3.076845250485478e+00 + 3.069616081755534e+00 + 3.062403852244847e+00 + 3.055208522175611e+00 + 3.048030052549899e+00 + 3.040868404439169e+00 + 3.033723537524932e+00 + 3.026595412527113e+00 + 3.019483991059011e+00 + 3.012389234177664e+00 + 3.005311102428361e+00 + 2.998249556405217e+00 + 2.991204558170310e+00 + 2.984176069441633e+00 + 2.977164051262350e+00 + 2.970168464749357e+00 + 2.963189271462946e+00 + 2.956226433370632e+00 + 2.949279912203107e+00 + 2.942349669912605e+00 + 2.935435668735732e+00 + 2.928537870478263e+00 + 2.921656236941307e+00 + 2.914790730227295e+00 + 2.907941313056430e+00 + 2.901107948114753e+00 + 2.894290597705669e+00 + 2.887489224186201e+00 + 2.880703790004707e+00 + 2.873934257778331e+00 + 2.867180590721052e+00 + 2.860442752157053e+00 + 2.853720705060128e+00 + 2.847014412004529e+00 + 2.840323835735203e+00 + 2.833648939802342e+00 + 2.826989688360529e+00 + 2.820346045051714e+00 + 2.813717971838249e+00 + 2.807105432809830e+00 + 2.800508392739872e+00 + 2.793926814285955e+00 + 2.787360661385112e+00 + 2.780809898806732e+00 + 2.774274490318202e+00 + 2.767754399418414e+00 + 2.761249589980850e+00 + 2.754760027276096e+00 + 2.748285675987487e+00 + 2.741826499952791e+00 + 2.735382462817888e+00 + 2.728953529693038e+00 + 2.722539666591989e+00 + 2.716140837199994e+00 + 2.709757005928732e+00 + 2.703388138477217e+00 + 2.697034199719776e+00 + 2.690695154612969e+00 + 2.684370968516638e+00 + 2.678061606722120e+00 + 2.671767034388748e+00 + 2.665487216663756e+00 + 2.659222119420964e+00 + 2.652971708466898e+00 + 2.646735949170681e+00 + 2.640514807328682e+00 + 2.634308248813400e+00 + 2.628116239351709e+00 + 2.621938745026414e+00 + 2.615775731978081e+00 + 2.609627166192403e+00 + 2.603493014125890e+00 + 2.597373242133807e+00 + 2.591267815972944e+00 + 2.585176702619338e+00 + 2.579099869106871e+00 + 2.573037281022092e+00 + 2.566988905295047e+00 + 2.560954709471109e+00 + 2.554934660124372e+00 + 2.548928723658388e+00 + 2.542936866838776e+00 + 2.536959057454498e+00 + 2.530995262744329e+00 + 2.525045449466102e+00 + 2.519109584724339e+00 + 2.513187636248425e+00 + 2.507279571988880e+00 + 2.501385358975255e+00 + 2.495504964649074e+00 + 2.489638357011514e+00 + 2.483785503877440e+00 + 2.477946372993181e+00 + 2.472120932153270e+00 + 2.466309149445497e+00 + 2.460510993028623e+00 + 2.454726431004756e+00 + 2.448955431279121e+00 + 2.443197962222979e+00 + 2.437453992749473e+00 + 2.431723490873920e+00 + 2.426006424702946e+00 + 2.420302763016215e+00 + 2.414612474849150e+00 + 2.408935529060812e+00 + 2.403271894158963e+00 + 2.397621539164549e+00 + 2.391984432987164e+00 + 2.386360544006294e+00 + 2.380749841635097e+00 + 2.375152295549876e+00 + 2.369567874852914e+00 + 2.363996548806182e+00 + 2.358438286639424e+00 + 2.352893057314327e+00 + 2.347360830870365e+00 + 2.341841577523073e+00 + 2.336335266205249e+00 + 2.330841866367291e+00 + 2.325361348038680e+00 + 2.319893681430557e+00 + 2.314438836350213e+00 + 2.308996782525400e+00 + 2.303567490482044e+00 + 2.298150930506590e+00 + 2.292747072570897e+00 + 2.287355886755380e+00 + 2.281977343203661e+00 + 2.276611412298537e+00 + 2.271258065216148e+00 + 2.265917272602903e+00 + 2.260589004572198e+00 + 2.255273232290290e+00 + 2.249969926489583e+00 + 2.244679057245269e+00 + 2.239400595715412e+00 + 2.234134513404378e+00 + 2.228880781619142e+00 + 2.223639370658599e+00 + 2.218410251587366e+00 + 2.213193396784469e+00 + 2.207988776668006e+00 + 2.202796362022329e+00 + 2.197616125439223e+00 + 2.192448038696543e+00 + 2.187292072961057e+00 + 2.182148199232985e+00 + 2.177016389630072e+00 + 2.171896616273634e+00 + 2.166788850281173e+00 + 2.161693063906081e+00 + 2.156609229682959e+00 + 2.151537319229026e+00 + 2.146477304323076e+00 + 2.141429157241363e+00 + 2.136392850965788e+00 + 2.131368357474257e+00 + 2.126355648357996e+00 + 2.121354696293322e+00 + 2.116365474412895e+00 + 2.111387955644702e+00 + 2.106422111701483e+00 + 2.101467915161587e+00 + 2.096525339256323e+00 + 2.091594356269759e+00 + 2.086674939419586e+00 + 2.081767062556874e+00 + 2.076870697792070e+00 + 2.071985817585338e+00 + 2.067112395385072e+00 + 2.062250405333861e+00 + 2.057399820685575e+00 + 2.052560613690088e+00 + 2.047732757766644e+00 + 2.042916226752948e+00 + 2.038110994473384e+00 + 2.033317034456518e+00 + 2.028534320143143e+00 + 2.023762825026766e+00 + 2.019002522786289e+00 + 2.014253387371010e+00 + 2.009515392980107e+00 + 2.004788513366984e+00 + 2.000072722331970e+00 + 1.995367994084281e+00 + 1.990674302528751e+00 + 1.985991621875054e+00 + 1.981319927135009e+00 + 1.976659192144684e+00 + 1.972009390577169e+00 + 1.967370497271926e+00 + 1.962742487042732e+00 + 1.958125334423246e+00 + 1.953519013586483e+00 + 1.948923499307053e+00 + 1.944338766657414e+00 + 1.939764790288013e+00 + 1.935201544608222e+00 + 1.930649004305472e+00 + 1.926107145248571e+00 + 1.921575942429786e+00 + 1.917055370205972e+00 + 1.912545403920843e+00 + 1.908046018818900e+00 + 1.903557190023903e+00 + 1.899078893306255e+00 + 1.894611103753838e+00 + 1.890153795996532e+00 + 1.885706946418488e+00 + 1.881270530851602e+00 + 1.876844524133795e+00 + 1.872428902057808e+00 + 1.868023640555410e+00 + 1.863628715350075e+00 + 1.859244102235784e+00 + 1.854869776995271e+00 + 1.850505715382929e+00 + 1.846151893336109e+00 + 1.841808287113597e+00 + 1.837474873250078e+00 + 1.833151627500553e+00 + 1.828838525627810e+00 + 1.824535544058799e+00 + 1.820242659570220e+00 + 1.815959848693184e+00 + 1.811687087278560e+00 + 1.807424351697017e+00 + 1.803171618832717e+00 + 1.798928865872756e+00 + 1.794696068965575e+00 + 1.790473204047284e+00 + 1.786260248130819e+00 + 1.782057178544553e+00 + 1.777863972462110e+00 + 1.773680606381003e+00 + 1.769507057021776e+00 + 1.765343301422396e+00 + 1.761189316819223e+00 + 1.757045080431908e+00 + 1.752910569457795e+00 + 1.748785761158555e+00 + 1.744670632638435e+00 + 1.740565161064786e+00 + 1.736469324420783e+00 + 1.732383100261040e+00 + 1.728306465595185e+00 + 1.724239397824929e+00 + 1.720181874717039e+00 + 1.716133874165168e+00 + 1.712095373335323e+00 + 1.708066350172305e+00 + 1.704046783495335e+00 + 1.700036650108550e+00 + 1.696035927584721e+00 + 1.692044595228792e+00 + 1.688062630443701e+00 + 1.684090010658294e+00 + 1.680126714513943e+00 + 1.676172720475745e+00 + 1.672228006742128e+00 + 1.668292551293985e+00 + 1.664366332353613e+00 + 1.660449328407859e+00 + 1.656541518177628e+00 + 1.652642880144113e+00 + 1.648753392536404e+00 + 1.644873033459887e+00 + 1.641001782122826e+00 + 1.637139617814172e+00 + 1.633286518510287e+00 + 1.629442462986198e+00 + 1.625607430332010e+00 + 1.621781398816242e+00 + 1.617964347766720e+00 + 1.614156256896447e+00 + 1.610357104501046e+00 + 1.606566869528149e+00 + 1.602785531410984e+00 + 1.599013068753329e+00 + 1.595249460964023e+00 + 1.591494688099418e+00 + 1.587748729316499e+00 + 1.584011563661260e+00 + 1.580283170366302e+00 + 1.576563528988570e+00 + 1.572852618951484e+00 + 1.569150419635379e+00 + 1.565456911345500e+00 + 1.561772073862832e+00 + 1.558095886092752e+00 + 1.554428327545355e+00 + 1.550769378486889e+00 + 1.547119019605569e+00 + 1.543477229774924e+00 + 1.539843988467248e+00 + 1.536219276909035e+00 + 1.532603074921508e+00 + 1.528995362108325e+00 + 1.525396118748853e+00 + 1.521805324624735e+00 + 1.518222960029189e+00 + 1.514649006474386e+00 + 1.511083443428444e+00 + 1.507526250258141e+00 + 1.503977408416516e+00 + 1.500436898413773e+00 + 1.496904700394939e+00 + 1.493380795245864e+00 + 1.489865163193006e+00 + 1.486357784560608e+00 + 1.482858641108495e+00 + 1.479367713055452e+00 + 1.475884980161266e+00 + 1.472410424507970e+00 + 1.468944026943859e+00 + 1.465485767394345e+00 + 1.462035627142284e+00 + 1.458593587684006e+00 + 1.455159630175685e+00 + 1.451733734815932e+00 + 1.448315882934573e+00 + 1.444906056735074e+00 + 1.441504236540235e+00 + 1.438110403156542e+00 + 1.434724538391589e+00 + 1.431346623689521e+00 + 1.427976640530080e+00 + 1.424614570482120e+00 + 1.421260394583763e+00 + 1.417914094214416e+00 + 1.414575651360096e+00 + 1.411245047472556e+00 + 1.407922264020345e+00 + 1.404607282835409e+00 + 1.401300085863839e+00 + 1.398000654669480e+00 + 1.394708970355802e+00 + 1.391425015768138e+00 + 1.388148773421020e+00 + 1.384880223867722e+00 + 1.381619349200565e+00 + 1.378366132070768e+00 + 1.375120554406545e+00 + 1.371882598139520e+00 + 1.368652245409923e+00 + 1.365429478679455e+00 + 1.362214280063096e+00 + 1.359006631544336e+00 + 1.355806515436745e+00 + 1.352613914349998e+00 + 1.349428810943258e+00 + 1.346251187518860e+00 + 1.343081026346772e+00 + 1.339918309785753e+00 + 1.336763020411036e+00 + 1.333615141050152e+00 + 1.330474654669161e+00 + 1.327341544046787e+00 + 1.324215791384803e+00 + 1.321097378844990e+00 + 1.317986290481943e+00 + 1.314882509034005e+00 + 1.311786015892630e+00 + 1.308696795174431e+00 + 1.305614830505543e+00 + 1.302540104058677e+00 + 1.299472598550161e+00 + 1.296412297281467e+00 + 1.293359183920592e+00 + 1.290313241355424e+00 + 1.287274452437336e+00 + 1.284242800425524e+00 + 1.281218268758570e+00 + 1.278200840971845e+00 + 1.275190500607243e+00 + 1.272187230813499e+00 + 1.269191014801793e+00 + 1.266201836184906e+00 + 1.263219678244913e+00 + 1.260244524601725e+00 + 1.257276359774731e+00 + 1.254315166659556e+00 + 1.251360928048363e+00 + 1.248413628680403e+00 + 1.245473252809453e+00 + 1.242539783866816e+00 + 1.239613204552696e+00 + 1.236693499248504e+00 + 1.233780652886690e+00 + 1.230874648458874e+00 + 1.227975469917747e+00 + 1.225083101914005e+00 + 1.222197528080320e+00 + 1.219318732110022e+00 + 1.216446698153004e+00 + 1.213581411119631e+00 + 1.210722855169846e+00 + 1.207871013783067e+00 + 1.205025871252235e+00 + 1.202187412198626e+00 + 1.199355621285526e+00 + 1.196530482820713e+00 + 1.193711981112556e+00 + 1.190900100540550e+00 + 1.188094825240966e+00 + 1.185296139829988e+00 + 1.182504029475636e+00 + 1.179718478336080e+00 + 1.176939470878376e+00 + 1.174166992368249e+00 + 1.171401026812090e+00 + 1.168641558834780e+00 + 1.165888574598542e+00 + 1.163142057955574e+00 + 1.160401992993886e+00 + 1.157668365957760e+00 + 1.154941161055026e+00 + 1.152220362585152e+00 + 1.149505957086653e+00 + 1.146797929075238e+00 + 1.144096262593592e+00 + 1.141400943461258e+00 + 1.138711957183691e+00 + 1.136029288721929e+00 + 1.133352922646698e+00 + 1.130682844729473e+00 + 1.128019040871752e+00 + 1.125361495026299e+00 + 1.122710192813904e+00 + 1.120065120718956e+00 + 1.117426263156838e+00 + 1.114793605478939e+00 + 1.112167133798269e+00 + 1.109546832911627e+00 + 1.106932688501257e+00 + 1.104324686965964e+00 + 1.101722813149718e+00 + 1.099127052563592e+00 + 1.096537391606932e+00 + 1.093953815681381e+00 + 1.091376310184498e+00 + 1.088804860939953e+00 + 1.086239454210304e+00 + 1.083680075729654e+00 + 1.081126710640132e+00 + 1.078579345517792e+00 + 1.076037966489376e+00 + 1.073502558540405e+00 + 1.070973108232868e+00 + 1.068449602380304e+00 + 1.065932027008114e+00 + 1.063420367175204e+00 + 1.060914608734753e+00 + 1.058414739347230e+00 + 1.055920744292059e+00 + 1.053432609053344e+00 + 1.050950321750202e+00 + 1.048473868029229e+00 + 1.046003233045439e+00 + 1.043538404185668e+00 + 1.041079368203120e+00 + 1.038626111099421e+00 + 1.036178618557632e+00 + 1.033736877638818e+00 + 1.031300875738098e+00 + 1.028870598541307e+00 + 1.026446032279889e+00 + 1.024027163804816e+00 + 1.021613979927458e+00 + 1.019206467749757e+00 + 1.016804614201984e+00 + 1.014408404816817e+00 + 1.012017826403038e+00 + 1.009632866898133e+00 + 1.007253512894913e+00 + 1.004879750675442e+00 + 1.002511566831996e+00 + 1.000148949144661e+00 + 9.977918847145494e-01 + 9.954403597206510e-01 + 9.930943614528360e-01 + 9.907538771638876e-01 + 9.884188936740620e-01 + 9.860893982148410e-01 + 9.837653780132122e-01 + 9.814468200970017e-01 + 9.791337116433633e-01 + 9.768260399168340e-01 + 9.745237922184445e-01 + 9.722269558574113e-01 + 9.699355181124353e-01 + 9.676494662336389e-01 + 9.653687876978898e-01 + 9.630934699744002e-01 + 9.608235003289306e-01 + 9.585588662885656e-01 + 9.562995554410040e-01 + 9.540455551859868e-01 + 9.517968530931009e-01 + 9.495534368089285e-01 + 9.473152938858177e-01 + 9.450824119343849e-01 + 9.428547786515334e-01 + 9.406323818265262e-01 + 9.384152091378831e-01 + 9.362032482272097e-01 + 9.339964869112888e-01 + 9.317949130473703e-01 + 9.295985145023118e-01 + 9.274072791539573e-01 + 9.252211948367504e-01 + 9.230402494034374e-01 + 9.208644309329761e-01 + 9.186937274260252e-01 + 9.165281267878318e-01 + 9.143676170947915e-01 + 9.122121864230248e-01 + 9.100618228160294e-01 + 9.079165144228375e-01 + 9.057762494101416e-01 + 9.036410159434681e-01 + 9.015108022852656e-01 + 8.993855966397851e-01 + 8.972653871281971e-01 + 8.951501621417270e-01 + 8.930399100640233e-01 + 8.909346191274636e-01 + 8.888342777366450e-01 + 8.867388743114113e-01 + 8.846483971822027e-01 + 8.825628348501290e-01 + 8.804821758633001e-01 + 8.784064086961259e-01 + 8.763355218399448e-01 + 8.742695038377348e-01 + 8.722083433110044e-01 + 8.701520288632569e-01 + 8.681005490981059e-01 + 8.660538926659707e-01 + 8.640120483003465e-01 + 8.619750047630993e-01 + 8.599427507590320e-01 + 8.579152750196485e-01 + 8.558925663402844e-01 + 8.538746136199826e-01 + 8.518614056598554e-01 + 8.498529312415619e-01 + 8.478491793952468e-01 + 8.458501390477595e-01 + 8.438557990450950e-01 + 8.418661484572341e-01 + 8.398811762963594e-01 + 8.379008715007146e-01 + 8.359252231704962e-01 + 8.339542204095658e-01 + 8.319878522912720e-01 + 8.300261079597043e-01 + 8.280689765769632e-01 + 8.261164473121154e-01 + 8.241685093974929e-01 + 8.222251520716921e-01 + 8.202863645582569e-01 + 8.183521361157176e-01 + 8.164224560764958e-01 + 8.144973138478017e-01 + 8.125766987264839e-01 + 8.106606000443977e-01 + 8.087490072689457e-01 + 8.068419098143643e-01 + 8.049392971238847e-01 + 8.030411587461223e-01 + 8.011474841608780e-01 + 7.992582628576694e-01 + 7.973734844327619e-01 + 7.954931384582111e-01 + 7.936172145055983e-01 + 7.917457022017643e-01 + 7.898785912282000e-01 + 7.880158712799969e-01 + 7.861575320154112e-01 + 7.843035631939634e-01 + 7.824539546080472e-01 + 7.806086959262468e-01 + 7.787677769423338e-01 + 7.769311875560446e-01 + 7.750989176225404e-01 + 7.732709569908990e-01 + 7.714472955211897e-01 + 7.696279231052234e-01 + 7.678128297211314e-01 + 7.660020053954494e-01 + 7.641954400346342e-01 + 7.623931236302374e-01 + 7.605950462809293e-01 + 7.588011979868697e-01 + 7.570115688113509e-01 + 7.552261489288551e-01 + 7.534449284973839e-01 + 7.516678976340316e-01 + 7.498950464317325e-01 + 7.481263651057625e-01 + 7.463618439154999e-01 + 7.446014731170009e-01 + 7.428452429690515e-01 + 7.410931437272748e-01 + 7.393451656574506e-01 + 7.376012991597445e-01 + 7.358615346418724e-01 + 7.341258624174279e-01 + 7.323942728636854e-01 + 7.306667564033948e-01 + 7.289433034811010e-01 + 7.272239046205010e-01 + 7.255085503342084e-01 + 7.237972310231134e-01 + 7.220899372662432e-01 + 7.203866596915919e-01 + 7.186873887451811e-01 + 7.169921150530395e-01 + 7.153008293485266e-01 + 7.136135222426403e-01 + 7.119301843519804e-01 + 7.102508063464710e-01 + 7.085753789925818e-01 + 7.069038930034782e-01 + 7.052363390742027e-01 + 7.035727080638011e-01 + 7.019129907750439e-01 + 7.002571779481277e-01 + 6.986052604437083e-01 + 6.969572291357313e-01 + 6.953130748988353e-01 + 6.936727887020229e-01 + 6.920363614522840e-01 + 6.904037839913317e-01 + 6.887750473773369e-01 + 6.871501426160253e-01 + 6.855290605871678e-01 + 6.839117924040222e-01 + 6.822983292013320e-01 + 6.806886620123715e-01 + 6.790827818618205e-01 + 6.774806798407976e-01 + 6.758823471448717e-01 + 6.742877749381793e-01 + 6.726969543916550e-01 + 6.711098767181217e-01 + 6.695265330883416e-01 + 6.679469147191758e-01 + 6.663710129607012e-01 + 6.647988190816884e-01 + 6.632303243195694e-01 + 6.616655199754741e-01 + 6.601043974442724e-01 + 6.585469481315596e-01 + 6.569931633409668e-01 + 6.554430344698831e-01 + 6.538965529680326e-01 + 6.523537102216029e-01 + 6.508144977390269e-01 + 6.492789070902356e-01 + 6.477469297088367e-01 + 6.462185570471877e-01 + 6.446937806333431e-01 + 6.431725921186934e-01 + 6.416549831028872e-01 + 6.401409451291091e-01 + 6.386304698205402e-01 + 6.371235488250266e-01 + 6.356201737964903e-01 + 6.341203364004929e-01 + 6.326240283504814e-01 + 6.311312414023209e-01 + 6.296419672600960e-01 + 6.281561976359686e-01 + 6.266739242956670e-01 + 6.251951391194286e-01 + 6.237198339322255e-01 + 6.222480004492102e-01 + 6.207796305731890e-01 + 6.193147162378443e-01 + 6.178532493027741e-01 + 6.163952216645050e-01 + 6.149406252489860e-01 + 6.134894519984210e-01 + 6.120416938491114e-01 + 6.105973427932988e-01 + 6.091563909278429e-01 + 6.077188302116784e-01 + 6.062846526061169e-01 + 6.048538502614210e-01 + 6.034264152733694e-01 + 6.020023397068369e-01 + 6.005816156757759e-01 + 5.991642353027361e-01 + 5.977501907311656e-01 + 5.963394741611191e-01 + 5.949320777981851e-01 + 5.935279938331314e-01 + 5.921272144271874e-01 + 5.907297318514260e-01 + 5.893355384452119e-01 + 5.879446264415406e-01 + 5.865569881004723e-01 + 5.851726157460215e-01 + 5.837915017610071e-01 + 5.824136384878618e-01 + 5.810390182425506e-01 + 5.796676334763571e-01 + 5.782994765837256e-01 + 5.769345398883866e-01 + 5.755728159164172e-01 + 5.742142971757567e-01 + 5.728589760824728e-01 + 5.715068451452269e-01 + 5.701578968814059e-01 + 5.688121237820786e-01 + 5.674695184134964e-01 + 5.661300733449661e-01 + 5.647937811172017e-01 + 5.634606343850478e-01 + 5.621306258016221e-01 + 5.608037479323033e-01 + 5.594799934047830e-01 + 5.581593548937616e-01 + 5.568418251014721e-01 + 5.555273967809645e-01 + 5.542160626584934e-01 + 5.529078153599613e-01 + 5.516026476869607e-01 + 5.503005524879265e-01 + 5.490015224489507e-01 + 5.477055503706971e-01 + 5.464126291394721e-01 + 5.451227516109934e-01 + 5.438359105997491e-01 + 5.425520989229732e-01 + 5.412713094911021e-01 + 5.399935352390902e-01 + 5.387187691034094e-01 + 5.374470040087604e-01 + 5.361782329113639e-01 + 5.349124487848638e-01 + 5.336496445514358e-01 + 5.323898132502169e-01 + 5.311329480044003e-01 + 5.298790417206248e-01 + 5.286280874266170e-01 + 5.273800783238198e-01 + 5.261350074387794e-01 + 5.248928678300107e-01 + 5.236536526638225e-01 + 5.224173550711992e-01 + 5.211839681868924e-01 + 5.199534851768893e-01 + 5.187258992380968e-01 + 5.175012035574781e-01 + 5.162793913049719e-01 + 5.150604557657682e-01 + 5.138443902158373e-01 + 5.126311878387027e-01 + 5.114208419243966e-01 + 5.102133457996210e-01 + 5.090086927559088e-01 + 5.078068761246483e-01 + 5.066078892445277e-01 + 5.054117254264059e-01 + 5.042183780707232e-01 + 5.030278406156905e-01 + 5.018401064512819e-01 + 5.006551689399448e-01 + 4.994730214827681e-01 + 4.982936576173500e-01 + 4.971170708196209e-01 + 4.959432545161117e-01 + 4.947722021784954e-01 + 4.936039073158257e-01 + 4.924383634713059e-01 + 4.912755642215030e-01 + 4.901155031029966e-01 + 4.889581736357913e-01 + 4.878035694426762e-01 + 4.866516841617155e-01 + 4.855025114095978e-01 + 4.843560447562852e-01 + 4.832122778219742e-01 + 4.820712043014287e-01 + 4.809328179063641e-01 + 4.797971123302187e-01 + 4.786640812406847e-01 + 4.775337183184862e-01 + 4.764060173050261e-01 + 4.752809720122764e-01 + 4.741585762111217e-01 + 4.730388236459319e-01 + 4.719217080626099e-01 + 4.708072233097221e-01 + 4.696953632391141e-01 + 4.685861216425942e-01 + 4.674794923970907e-01 + 4.663754693817960e-01 + 4.652740464091041e-01 + 4.641752174102053e-01 + 4.630789763262008e-01 + 4.619853169869301e-01 + 4.608942333544824e-01 + 4.598057194461572e-01 + 4.587197692038988e-01 + 4.576363766028796e-01 + 4.565555356433393e-01 + 4.554772403132010e-01 + 4.544014845983438e-01 + 4.533282625263808e-01 + 4.522575682421708e-01 + 4.511893957834002e-01 + 4.501237391381080e-01 + 4.490605924783677e-01 + 4.479999499003729e-01 + 4.469418054464263e-01 + 4.458861533467765e-01 + 4.448329877503518e-01 + 4.437823027182840e-01 + 4.427340924804984e-01 + 4.416883512661691e-01 + 4.406450732493647e-01 + 4.396042525949710e-01 + 4.385658835318161e-01 + 4.375299603591805e-01 + 4.364964773180230e-01 + 4.354654286529535e-01 + 4.344368086449737e-01 + 4.334106115882546e-01 + 4.323868317973252e-01 + 4.313654636098987e-01 + 4.303465013528135e-01 + 4.293299393608014e-01 + 4.283157719943348e-01 + 4.273039936389254e-01 + 4.262945986862648e-01 + 4.252875815207442e-01 + 4.242829365388910e-01 + 4.232806581737543e-01 + 4.222807409163533e-01 + 4.212831792180283e-01 + 4.202879675160039e-01 + 4.192951002821390e-01 + 4.183045720190468e-01 + 4.173163772308877e-01 + 4.163305103895081e-01 + 4.153469660738844e-01 + 4.143657388899517e-01 + 4.133868232899575e-01 + 4.124102138463511e-01 + 4.114359052122226e-01 + 4.104638919136763e-01 + 4.094941685453110e-01 + 4.085267297797091e-01 + 4.075615702608746e-01 + 4.065986846247451e-01 + 4.056380675149961e-01 + 4.046797136100241e-01 + 4.037236175791848e-01 + 4.027697740867438e-01 + 4.018181778668817e-01 + 4.008688236537185e-01 + 3.999217061664900e-01 + 3.989768201804763e-01 + 3.980341604347806e-01 + 3.970937216243767e-01 + 3.961554986257766e-01 + 3.952194862561716e-01 + 3.942856791749246e-01 + 3.933540722797152e-01 + 3.924246604937682e-01 + 3.914974386057962e-01 + 3.905724013991230e-01 + 3.896495437210211e-01 + 3.887288605266558e-01 + 3.878103467384126e-01 + 3.868939972364765e-01 + 3.859798068708922e-01 + 3.850677705778728e-01 + 3.841578833358520e-01 + 3.832501400994028e-01 + 3.823445358203205e-01 + 3.814410654593080e-01 + 3.805397240004402e-01 + 3.796405064500310e-01 + 3.787434078281850e-01 + 3.778484231550217e-01 + 3.769555474293507e-01 + 3.760647756653435e-01 + 3.751761029803938e-01 + 3.742895244689736e-01 + 3.734050351897463e-01 + 3.725226302131253e-01 + 3.716423046174657e-01 + 3.707640535037849e-01 + 3.698878720450124e-01 + 3.690137553825144e-01 + 3.681416986254492e-01 + 3.672716969834960e-01 + 3.664037456169829e-01 + 3.655378396196581e-01 + 3.646739742353085e-01 + 3.638121447254376e-01 + 3.629523462980791e-01 + 3.620945741078652e-01 + 3.612388233782960e-01 + 3.603850894480597e-01 + 3.595333675652969e-01 + 3.586836529581271e-01 + 3.578359408972814e-01 + 3.569902267099870e-01 + 3.561465057284826e-01 + 3.553047732480240e-01 + 3.544650245489661e-01 + 3.536272549462773e-01 + 3.527914598373741e-01 + 3.519576346281267e-01 + 3.511257746893044e-01 + 3.502958753213151e-01 + 3.494679319027064e-01 + 3.486419398734283e-01 + 3.478178946841421e-01 + 3.469957917542031e-01 + 3.461756264906874e-01 + 3.453573943334770e-01 + 3.445410907466465e-01 + 3.437267112053919e-01 + 3.429142511753236e-01 + 3.421037061357636e-01 + 3.412950715914754e-01 + 3.404883430871680e-01 + 3.396835161220511e-01 + 3.388805861778083e-01 + 3.380795488742585e-01 + 3.372803997559251e-01 + 3.364831342820857e-01 + 3.356877480644493e-01 + 3.348942367473924e-01 + 3.341025959445759e-01 + 3.333128211902062e-01 + 3.325249080644813e-01 + 3.317388522349806e-01 + 3.309546493345364e-01 + 3.301722950123494e-01 + 3.293917849546400e-01 + 3.286131147929292e-01 + 3.278362801760134e-01 + 3.270612768173198e-01 + 3.262881004205997e-01 + 3.255167466776044e-01 + 3.247472112772871e-01 + 3.239794899591130e-01 + 3.232135784952971e-01 + 3.224494726613195e-01 + 3.216871681318617e-01 + 3.209266606147330e-01 + 3.201679460077643e-01 + 3.194110200806869e-01 + 3.186558785671545e-01 + 3.179025173467112e-01 + 3.171509322207308e-01 + 3.164011189537259e-01 + 3.156530734030461e-01 + 3.149067914068442e-01 + 3.141622688005587e-01 + 3.134195014964591e-01 + 3.126784853846941e-01 + 3.119392163234083e-01 + 3.112016901727622e-01 + 3.104659028247113e-01 + 3.097318502149898e-01 + 3.089995283319454e-01 + 3.082689330721005e-01 + 3.075400602663161e-01 + 3.068129059444455e-01 + 3.060874661234649e-01 + 3.053637367362504e-01 + 3.046417137079979e-01 + 3.039213930418579e-01 + 3.032027708160949e-01 + 3.024858429572295e-01 + 3.017706054435225e-01 + 3.010570543910775e-01 + 3.003451858060657e-01 + 2.996349956970866e-01 + 2.989264801519126e-01 + 2.982196352437723e-01 + 2.975144570351090e-01 + 2.968109415899733e-01 + 2.961090850010212e-01 + 2.954088833712375e-01 + 2.947103327960572e-01 + 2.940134293733013e-01 + 2.933181692489545e-01 + 2.926245486550200e-01 + 2.919325636534309e-01 + 2.912422103009782e-01 + 2.905534849067459e-01 + 2.898663836341845e-01 + 2.891809025716850e-01 + 2.884970379499296e-01 + 2.878147859946116e-01 + 2.871341429080666e-01 + 2.864551048929091e-01 + 2.857776681836698e-01 + 2.851018290276999e-01 + 2.844275836247271e-01 + 2.837549281975342e-01 + 2.830838590248334e-01 + 2.824143724756855e-01 + 2.817464648036653e-01 + 2.810801321714598e-01 + 2.804153709678348e-01 + 2.797521775256636e-01 + 2.790905480675208e-01 + 2.784304789557904e-01 + 2.777719665515425e-01 + 2.771150071618110e-01 + 2.764595971293084e-01 + 2.758057328280932e-01 + 2.751534106502780e-01 + 2.745026269461392e-01 + 2.738533780697315e-01 + 2.732056604090851e-01 + 2.725594703611444e-01 + 2.719148043516682e-01 + 2.712716588414341e-01 + 2.706300302048149e-01 + 2.699899148252890e-01 + 2.693513091897057e-01 + 2.687142097803046e-01 + 2.680786130525787e-01 + 2.674445154290628e-01 + 2.668119133864937e-01 + 2.661808034370396e-01 + 2.655511820876730e-01 + 2.649230457933240e-01 + 2.642963910212357e-01 + 2.636712143601993e-01 + 2.630475123173707e-01 + 2.624252813696634e-01 + 2.618045181320832e-01 + 2.611852191534089e-01 + 2.605673809346502e-01 + 2.599510000768786e-01 + 2.593360731482656e-01 + 2.587225966857610e-01 + 2.581105673020023e-01 + 2.574999816218594e-01 + 2.568908362541323e-01 + 2.562831277681717e-01 + 2.556768527608984e-01 + 2.550720078808991e-01 + 2.544685898158210e-01 + 2.538665952029344e-01 + 2.532660206235198e-01 + 2.526668627769018e-01 + 2.520691183671673e-01 + 2.514727840477828e-01 + 2.508778564832650e-01 + 2.502843323450471e-01 + 2.496922083184503e-01 + 2.491014811741685e-01 + 2.485121476502302e-01 + 2.479242043768022e-01 + 2.473376480750936e-01 + 2.467524755330193e-01 + 2.461686835577825e-01 + 2.455862688342147e-01 + 2.450052280577099e-01 + 2.444255580815592e-01 + 2.438472556710649e-01 + 2.432703175621061e-01 + 2.426947405763887e-01 + 2.421205215374063e-01 + 2.415476572469543e-01 + 2.409761444715131e-01 + 2.404059800179439e-01 + 2.398371607406130e-01 + 2.392696835287547e-01 + 2.387035451998669e-01 + 2.381387425460481e-01 + 2.375752724876774e-01 + 2.370131319015487e-01 + 2.364523176042576e-01 + 2.358928264341899e-01 + 2.353346553264918e-01 + 2.347778012643444e-01 + 2.342222610470114e-01 + 2.336680315481888e-01 + 2.331151097630285e-01 + 2.325634926002070e-01 + 2.320131769932555e-01 + 2.314641599195968e-01 + 2.309164382391606e-01 + 2.303700088852327e-01 + 2.298248689205450e-01 + 2.292810152383314e-01 + 2.287384447851956e-01 + 2.281971546639490e-01 + 2.276571417710518e-01 + 2.271184030444375e-01 + 2.265809356255705e-01 + 2.260447364485637e-01 + 2.255098024678978e-01 + 2.249761308577043e-01 + 2.244437185775247e-01 + 2.239125625576918e-01 + 2.233826599324974e-01 + 2.228540078061592e-01 + 2.223266032159731e-01 + 2.218004431318401e-01 + 2.212755246546400e-01 + 2.207518449320089e-01 + 2.202294009962271e-01 + 2.197081899542086e-01 + 2.191882089472320e-01 + 2.186694550184439e-01 + 2.181519252528444e-01 + 2.176356167922184e-01 + 2.171205268020538e-01 + 2.166066524055817e-01 + 2.160939907132310e-01 + 2.155825389406823e-01 + 2.150722942170351e-01 + 2.145632535999474e-01 + 2.140554143128364e-01 + 2.135487735713488e-01 + 2.130433285376680e-01 + 2.125390763961598e-01 + 2.120360143225986e-01 + 2.115341394866721e-01 + 2.110334491301104e-01 + 2.105339404911263e-01 + 2.100356107692428e-01 + 2.095384571593845e-01 + 2.090424768875718e-01 + 2.085476672286868e-01 + 2.080540254649787e-01 + 2.075615488420417e-01 + 2.070702345483775e-01 + 2.065800798220710e-01 + 2.060910819584345e-01 + 2.056032383022174e-01 + 2.051165461242101e-01 + 2.046310026800104e-01 + 2.041466052775261e-01 + 2.036633511978056e-01 + 2.031812377402102e-01 + 2.027002622847228e-01 + 2.022204221329609e-01 + 2.017417145657832e-01 + 2.012641369577989e-01 + 2.007876866366623e-01 + 2.003123609141739e-01 + 1.998381571847571e-01 + 1.993650728223044e-01 + 1.988931051658790e-01 + 1.984222515391579e-01 + 1.979525093566432e-01 + 1.974838760798574e-01 + 1.970163490394448e-01 + 1.965499255924087e-01 + 1.960846031634822e-01 + 1.956203792216227e-01 + 1.951572511456189e-01 + 1.946952162469976e-01 + 1.942342720345011e-01 + 1.937744160087473e-01 + 1.933156455855021e-01 + 1.928579581617409e-01 + 1.924013511601302e-01 + 1.919458220512063e-01 + 1.914913683490730e-01 + 1.910379875218400e-01 + 1.905856769690048e-01 + 1.901344342343178e-01 + 1.896842568245793e-01 + 1.892351421122624e-01 + 1.887870876905098e-01 + 1.883400911382459e-01 + 1.878941498265747e-01 + 1.874492612709913e-01 + 1.870054230608876e-01 + 1.865626327533365e-01 + 1.861208878413260e-01 + 1.856801858136446e-01 + 1.852405242290795e-01 + 1.848019006561455e-01 + 1.843643126785636e-01 + 1.839277579090688e-01 + 1.834922338318696e-01 + 1.830577379260484e-01 + 1.826242679175190e-01 + 1.821918214082343e-01 + 1.817603959062191e-01 + 1.813299890419420e-01 + 1.809005984096844e-01 + 1.804722215715290e-01 + 1.800448561676805e-01 + 1.796184998471616e-01 + 1.791931502428031e-01 + 1.787688049585256e-01 + 1.783454616010165e-01 + 1.779231178061076e-01 + 1.775017712916434e-01 + 1.770814197185338e-01 + 1.766620606643876e-01 + 1.762436917707559e-01 + 1.758263107475443e-01 + 1.754099153420786e-01 + 1.749945031691721e-01 + 1.745800718793870e-01 + 1.741666192327871e-01 + 1.737541428912765e-01 + 1.733426405128906e-01 + 1.729321098263452e-01 + 1.725225486037993e-01 + 1.721139545737653e-01 + 1.717063253659770e-01 + 1.712996586978482e-01 + 1.708939523524782e-01 + 1.704892041326627e-01 + 1.700854117308264e-01 + 1.696825728315613e-01 + 1.692806852413719e-01 + 1.688797467224855e-01 + 1.684797550119726e-01 + 1.680807078836749e-01 + 1.676826031516292e-01 + 1.672854386236076e-01 + 1.668892120257625e-01 + 1.664939211364053e-01 + 1.660995637803862e-01 + 1.657061377653481e-01 + 1.653136408872399e-01 + 1.649220709518862e-01 + 1.645314258210089e-01 + 1.641417032838678e-01 + 1.637529010922642e-01 + 1.633650171697226e-01 + 1.629780493848234e-01 + 1.625919955117979e-01 + 1.622068533722165e-01 + 1.618226208359533e-01 + 1.614392958070308e-01 + 1.610568761629498e-01 + 1.606753597398519e-01 + 1.602947443525945e-01 + 1.599150279328092e-01 + 1.595362083976419e-01 + 1.591582835876941e-01 + 1.587812513806259e-01 + 1.584051096895935e-01 + 1.580298564486488e-01 + 1.576554895336735e-01 + 1.572820068494865e-01 + 1.569094063844064e-01 + 1.565376859935812e-01 + 1.561668435389086e-01 + 1.557968770322644e-01 + 1.554277844430587e-01 + 1.550595637013555e-01 + 1.546922127252644e-01 + 1.543257294352004e-01 + 1.539601117962810e-01 + 1.535953578647676e-01 + 1.532314655336160e-01 + 1.528684326732572e-01 + 1.525062574192351e-01 + 1.521449377506904e-01 + 1.517844715391133e-01 + 1.514248568043021e-01 + 1.510660915767141e-01 + 1.507081738546517e-01 + 1.503511015850067e-01 + 1.499948728018673e-01 + 1.496394855976185e-01 + 1.492849379312514e-01 + 1.489312277844460e-01 + 1.485783532079666e-01 + 1.482263122946834e-01 + 1.478751030581660e-01 + 1.475247234388987e-01 + 1.471751715323068e-01 + 1.468264454469171e-01 + 1.464785432366154e-01 + 1.461314629165984e-01 + 1.457852025452889e-01 + 1.454397602433564e-01 + 1.450951340257908e-01 + 1.447513219420110e-01 + 1.444083221508569e-01 + 1.440661327109938e-01 + 1.437247516822454e-01 + 1.433841772072484e-01 + 1.430444073901374e-01 + 1.427054403077353e-01 + 1.423672740371242e-01 + 1.420299067307786e-01 + 1.416933365389871e-01 + 1.413575615237460e-01 + 1.410225798197031e-01 + 1.406883895901710e-01 + 1.403549889435149e-01 + 1.400223760032770e-01 + 1.396905489310608e-01 + 1.393595059398338e-01 + 1.390292451224472e-01 + 1.386997645496112e-01 + 1.383710625136722e-01 + 1.380431371824567e-01 + 1.377159866326167e-01 + 1.373896091055998e-01 + 1.370640027752214e-01 + 1.367391657592925e-01 + 1.364150963290927e-01 + 1.360917926783734e-01 + 1.357692529162330e-01 + 1.354474752994454e-01 + 1.351264580487222e-01 + 1.348061993122180e-01 + 1.344866973181728e-01 + 1.341679503229942e-01 + 1.338499565682958e-01 + 1.335327142013654e-01 + 1.332162214320106e-01 + 1.329004765902875e-01 + 1.325854779258548e-01 + 1.322712236439755e-01 + 1.319577119415115e-01 + 1.316449410589970e-01 + 1.313329092858229e-01 + 1.310216149439378e-01 + 1.307110562100912e-01 + 1.304012313067827e-01 + 1.300921386594473e-01 + 1.297837764741680e-01 + 1.294761429396481e-01 + 1.291692364743583e-01 + 1.288630553353233e-01 + 1.285575977268189e-01 + 1.282528620001082e-01 + 1.279488464604472e-01 + 1.276455493846629e-01 + 1.273429690961771e-01 + 1.270411039265308e-01 + 1.267399521912149e-01 + 1.264395121606617e-01 + 1.261397821624623e-01 + 1.258407605612416e-01 + 1.255424456647032e-01 + 1.252448358003494e-01 + 1.249479293256652e-01 + 1.246517245985584e-01 + 1.243562199440227e-01 + 1.240614136699035e-01 + 1.237673041530004e-01 + 1.234738897833225e-01 + 1.231811689369122e-01 + 1.228891399541342e-01 + 1.225978011897444e-01 + 1.223071510285007e-01 + 1.220171878485729e-01 + 1.217279100221359e-01 + 1.214393159216827e-01 + 1.211514039479249e-01 + 1.208641725134118e-01 + 1.205776200272462e-01 + 1.202917448643591e-01 + 1.200065453999493e-01 + 1.197220200388544e-01 + 1.194381672203820e-01 + 1.191549853738593e-01 + 1.188724728867774e-01 + 1.185906281982059e-01 + 1.183094497403516e-01 + 1.180289358802264e-01 + 1.177490851172322e-01 + 1.174698959467804e-01 + 1.171913666738082e-01 + 1.169134957966049e-01 + 1.166362818664215e-01 + 1.163597231976005e-01 + 1.160838182295854e-01 + 1.158085654926154e-01 + 1.155339634301463e-01 + 1.152600105152270e-01 + 1.149867052503320e-01 + 1.147140461081585e-01 + 1.144420315115333e-01 + 1.141706598827868e-01 + 1.138999297876022e-01 + 1.136298397724347e-01 + 1.133603883121320e-01 + 1.130915738073908e-01 + 1.128233947404400e-01 + 1.125558496925181e-01 + 1.122889371862514e-01 + 1.120226557224089e-01 + 1.117570037925019e-01 + 1.114919798574519e-01 + 1.112275824488763e-01 + 1.109638101830490e-01 + 1.107006615250319e-01 + 1.104381349837027e-01 + 1.101762291935612e-01 + 1.099149425817940e-01 + 1.096542736337556e-01 + 1.093942210611990e-01 + 1.091347833750166e-01 + 1.088759590309823e-01 + 1.086177465849946e-01 + 1.083601446468375e-01 + 1.081031518187680e-01 + 1.078467666394599e-01 + 1.075909876365943e-01 + 1.073358133695979e-01 + 1.070812424691808e-01 + 1.068272734935852e-01 + 1.065739049762412e-01 + 1.063211355273724e-01 + 1.060689637753818e-01 + 1.058173883167701e-01 + 1.055664076522099e-01 + 1.053160204177178e-01 + 1.050662253083730e-01 + 1.048170208178578e-01 + 1.045684055544151e-01 + 1.043203782231123e-01 + 1.040729373745031e-01 + 1.038260815947437e-01 + 1.035798095388146e-01 + 1.033341198496129e-01 + 1.030890111353506e-01 + 1.028444819874746e-01 + 1.026005310752415e-01 + 1.023571570589419e-01 + 1.021143585605686e-01 + 1.018721342063349e-01 + 1.016304826397723e-01 + 1.013894025261397e-01 + 1.011488925387934e-01 + 1.009089513225976e-01 + 1.006695774897203e-01 + 1.004307697412456e-01 + 1.001925267601519e-01 + 9.995484714928184e-02 + 9.971772961029543e-02 + 9.948117285600222e-02 + 9.924517553207431e-02 + 9.900973631553432e-02 + 9.877485389836264e-02 + 9.854052696321766e-02 + 9.830675420513860e-02 + 9.807353432053509e-02 + 9.784086599274402e-02 + 9.760874792461041e-02 + 9.737717882679013e-02 + 9.714615739564664e-02 + 9.691568233864475e-02 + 9.668575237168565e-02 + 9.645636620785530e-02 + 9.622752256709627e-02 + 9.599922017186617e-02 + 9.577145773476621e-02 + 9.554423398421420e-02 + 9.531754766036228e-02 + 9.509139748951084e-02 + 9.486578220406576e-02 + 9.464070054491446e-02 + 9.441615125061270e-02 + 9.419213306609350e-02 + 9.396864474309072e-02 + 9.374568502973517e-02 + 9.352325267700241e-02 + 9.330134644110695e-02 + 9.307996508088016e-02 + 9.285910736057421e-02 + 9.263877204899980e-02 + 9.241895790580944e-02 + 9.219966369924239e-02 + 9.198088821288498e-02 + 9.176263021506216e-02 + 9.154488848131032e-02 + 9.132766180778470e-02 + 9.111094897456264e-02 + 9.089474876186458e-02 + 9.067905996494766e-02 + 9.046388137755315e-02 + 9.024921179550818e-02 + 9.003505002157913e-02 + 8.982139485394958e-02 + 8.960824509461397e-02 + 8.939559956013639e-02 + 8.918345705664489e-02 + 8.897181638949078e-02 + 8.876067638185769e-02 + 8.855003585659860e-02 + 8.833989363307079e-02 + 8.813024852794749e-02 + 8.792109937217552e-02 + 8.771244500411417e-02 + 8.750428424959714e-02 + 8.729661594151820e-02 + 8.708943892038122e-02 + 8.688275202450782e-02 + 8.667655409894154e-02 + 8.647084399500883e-02 + 8.626562056133152e-02 + 8.606088264727801e-02 + 8.585662910508592e-02 + 8.565285879275303e-02 + 8.544957056834111e-02 + 8.524676329099723e-02 + 8.504443583456130e-02 + 8.484258706983651e-02 + 8.464121585953427e-02 + 8.444032107793580e-02 + 8.423990160173883e-02 + 8.403995630660985e-02 + 8.384048408000297e-02 + 8.364148380739438e-02 + 8.344295436518863e-02 + 8.324489464319758e-02 + 8.304730353631981e-02 + 8.285017993778075e-02 + 8.265352274723796e-02 + 8.245733086610936e-02 + 8.226160319325253e-02 + 8.206633863307464e-02 + 8.187153609323948e-02 + 8.167719448173898e-02 + 8.148331271120264e-02 + 8.128988969775824e-02 + 8.109692435907066e-02 + 8.090441561287741e-02 + 8.071236238052074e-02 + 8.052076359329230e-02 + 8.032961817777003e-02 + 8.013892505875637e-02 + 7.994868317110082e-02 + 7.975889145271257e-02 + 7.956954884117817e-02 + 7.938065427030368e-02 + 7.919220668490153e-02 + 7.900420503735811e-02 + 7.881664826626886e-02 + 7.862953532075569e-02 + 7.844286516277023e-02 + 7.825663674598571e-02 + 7.807084902471831e-02 + 7.788550095723919e-02 + 7.770059150356423e-02 + 7.751611963202924e-02 + 7.733208431808952e-02 + 7.714848452080215e-02 + 7.696531920714285e-02 + 7.678258736267791e-02 + 7.660028796508990e-02 + 7.641841999039113e-02 + 7.623698241865759e-02 + 7.605597423593993e-02 + 7.587539442823407e-02 + 7.569524197859213e-02 + 7.551551588287067e-02 + 7.533621514035872e-02 + 7.515733874432233e-02 + 7.497888568830134e-02 + 7.480085497230264e-02 + 7.462324560850588e-02 + 7.444605660139321e-02 + 7.426928695221670e-02 + 7.409293566891996e-02 + 7.391700176673981e-02 + 7.374148426439617e-02 + 7.356638217848553e-02 + 7.339169452681224e-02 + 7.321742033027905e-02 + 7.304355861552074e-02 + 7.287010840815517e-02 + 7.269706873346385e-02 + 7.252443862192341e-02 + 7.235221710900047e-02 + 7.218040323296335e-02 + 7.200899602923243e-02 + 7.183799453782064e-02 + 7.166739780328910e-02 + 7.149720486628935e-02 + 7.132741477295296e-02 + 7.115802657678706e-02 + 7.098903933028558e-02 + 7.082045208266741e-02 + 7.065226388294948e-02 + 7.048447379884398e-02 + 7.031708089501212e-02 + 7.015008422428966e-02 + 6.998348285005500e-02 + 6.981727584178653e-02 + 6.965146227138255e-02 + 6.948604121024822e-02 + 6.932101172952453e-02 + 6.915637290177927e-02 + 6.899212381049466e-02 + 6.882826353831664e-02 + 6.866479115768206e-02 + 6.850170575553913e-02 + 6.833900642241270e-02 + 6.817669223943085e-02 + 6.801476230171261e-02 + 6.785321570853013e-02 + 6.769205154760372e-02 + 6.753126891476798e-02 + 6.737086691265436e-02 + 6.721084464422118e-02 + 6.705120121123045e-02 + 6.689193571639933e-02 + 6.673304726820108e-02 + 6.657453497707803e-02 + 6.641639795452120e-02 + 6.625863531334618e-02 + 6.610124617104705e-02 + 6.594422964866814e-02 + 6.578758486543299e-02 + 6.563131094165708e-02 + 6.547540700031643e-02 + 6.531987216906585e-02 + 6.516470557837332e-02 + 6.500990635997374e-02 + 6.485547364331327e-02 + 6.470140656204018e-02 + 6.454770425495959e-02 + 6.439436585565442e-02 + 6.424139050518185e-02 + 6.408877735587415e-02 + 6.393652554954243e-02 + 6.378463422860429e-02 + 6.363310254304622e-02 + 6.348192964216136e-02 + 6.333111467891862e-02 + 6.318065681259971e-02 + 6.303055519506789e-02 + 6.288080898167979e-02 + 6.273141734118348e-02 + 6.258237943694549e-02 + 6.243369443027962e-02 + 6.228536148636779e-02 + 6.213737977420020e-02 + 6.198974846629291e-02 + 6.184246673830131e-02 + 6.169553375986874e-02 + 6.154894870195880e-02 + 6.140271075002260e-02 + 6.125681908485271e-02 + 6.111127288458502e-02 + 6.096607133498923e-02 + 6.082121362330186e-02 + 6.067669893778977e-02 + 6.053252647002724e-02 + 6.038869541057570e-02 + 6.024520494995262e-02 + 6.010205428441878e-02 + 5.995924261312039e-02 + 5.981676913606451e-02 + 5.967463305068377e-02 + 5.953283356000520e-02 + 5.939136987405440e-02 + 5.925024120274393e-02 + 5.910944675075109e-02 + 5.896898572024133e-02 + 5.882885733338872e-02 + 5.868906080796419e-02 + 5.854959534943303e-02 + 5.841046018167366e-02 + 5.827165452667381e-02 + 5.813317759547484e-02 + 5.799502862000658e-02 + 5.785720683145133e-02 + 5.771971144629215e-02 + 5.758254169802959e-02 + 5.744569682168989e-02 + 5.730917604038396e-02 + 5.717297859181381e-02 + 5.703710371771920e-02 + 5.690155065150715e-02 + 5.676631863514842e-02 + 5.663140691176988e-02 + 5.649681471550019e-02 + 5.636254129818796e-02 + 5.622858591623749e-02 + 5.609494780637568e-02 + 5.596162621719655e-02 + 5.582862040612746e-02 + 5.569592962469259e-02 + 5.556355312969728e-02 + 5.543149018219354e-02 + 5.529974004038473e-02 + 5.516830196534387e-02 + 5.503717522021812e-02 + 5.490635906464716e-02 + 5.477585276550550e-02 + 5.464565559606692e-02 + 5.451576682310098e-02 + 5.438618571514530e-02 + 5.425691154653305e-02 + 5.412794360021059e-02 + 5.399928115030668e-02 + 5.387092346220070e-02 + 5.374286982722595e-02 + 5.361511953114147e-02 + 5.348767184376742e-02 + 5.336052605560348e-02 + 5.323368145967647e-02 + 5.310713734132736e-02 + 5.298089299097181e-02 + 5.285494769998652e-02 + 5.272930075794034e-02 + 5.260395145882699e-02 + 5.247889910164562e-02 + 5.235414299012888e-02 + 5.222968242359068e-02 + 5.210551670083303e-02 + 5.198164512479836e-02 + 5.185806699751586e-02 + 5.173478162567831e-02 + 5.161178832737659e-02 + 5.148908640731603e-02 + 5.136667516904735e-02 + 5.124455393713079e-02 + 5.112272202541198e-02 + 5.100117874145336e-02 + 5.087992340415878e-02 + 5.075895533879782e-02 + 5.063827387090040e-02 + 5.051787831631814e-02 + 5.039776799903027e-02 + 5.027794224956293e-02 + 5.015840039018700e-02 + 5.003914174911826e-02 + 4.992016566217095e-02 + 4.980147146451835e-02 + 4.968305848565534e-02 + 4.956492605368701e-02 + 4.944707351707827e-02 + 4.932950021762891e-02 + 4.921220548403189e-02 + 4.909518865563611e-02 + 4.897844908008390e-02 + 4.886198610990230e-02 + 4.874579909103274e-02 + 4.862988736593407e-02 + 4.851425027765261e-02 + 4.839888718215429e-02 + 4.828379743688660e-02 + 4.816898039344981e-02 + 4.805443540666063e-02 + 4.794016183433197e-02 + 4.782615903628512e-02 + 4.771242637003347e-02 + 4.759896319572905e-02 + 4.748576888074151e-02 + 4.737284278529100e-02 + 4.726018427180961e-02 + 4.714779271742187e-02 + 4.703566749191604e-02 + 4.692380796083007e-02 + 4.681221349433169e-02 + 4.670088346648602e-02 + 4.658981725459431e-02 + 4.647901423861556e-02 + 4.636847379587147e-02 + 4.625819530165282e-02 + 4.614817813294777e-02 + 4.603842167462240e-02 + 4.592892531626435e-02 + 4.581968844113481e-02 + 4.571071043517037e-02 + 4.560199068716930e-02 + 4.549352858172214e-02 + 4.538532351139604e-02 + 4.527737487551017e-02 + 4.516968206047602e-02 + 4.506224446147816e-02 + 4.495506148526871e-02 + 4.484813252205796e-02 + 4.474145696749865e-02 + 4.463503423038843e-02 + 4.452886371233129e-02 + 4.442294481444441e-02 + 4.431727694184067e-02 + 4.421185950249874e-02 + 4.410669190344264e-02 + 4.400177354999718e-02 + 4.389710385863231e-02 + 4.379268224561385e-02 + 4.368850811842072e-02 + 4.358458089102907e-02 + 4.348089998166713e-02 + 4.337746480946211e-02 + 4.327427479290449e-02 + 4.317132935104961e-02 + 4.306862790538591e-02 + 4.296616988021409e-02 + 4.286395470044128e-02 + 4.276198178905291e-02 + 4.266025057462805e-02 + 4.255876048879523e-02 + 4.245751096050290e-02 + 4.235650142249979e-02 + 4.225573130868781e-02 + 4.215520004666404e-02 + 4.205490707118502e-02 + 4.195485182346907e-02 + 4.185503374212908e-02 + 4.175545226547943e-02 + 4.165610683241490e-02 + 4.155699688308426e-02 + 4.145812186261490e-02 + 4.135948121993713e-02 + 4.126107439911159e-02 + 4.116290084444992e-02 + 4.106496000347922e-02 + 4.096725133001662e-02 + 4.086977427671171e-02 + 4.077252829257245e-02 + 4.067551282880393e-02 + 4.057872734137224e-02 + 4.048217129089726e-02 + 4.038584413229004e-02 + 4.028974532214836e-02 + 4.019387432351806e-02 + 4.009823059884492e-02 + 4.000281360871143e-02 + 3.990762281263916e-02 + 3.981265768084040e-02 + 3.971791768404272e-02 + 3.962340228400397e-02 + 3.952911094818634e-02 + 3.943504314911767e-02 + 3.934119836154264e-02 + 3.924757605343093e-02 + 3.915417569553582e-02 + 3.906099677494528e-02 + 3.896803876462888e-02 + 3.887530113159737e-02 + 3.878278335807980e-02 + 3.869048493136602e-02 + 3.859840533540703e-02 + 3.850654403935494e-02 + 3.841490053099362e-02 + 3.832347430867683e-02 + 3.823226484521238e-02 + 3.814127162485314e-02 + 3.805049414568009e-02 + 3.795993189727254e-02 + 3.786958436838804e-02 + 3.777945104950908e-02 + 3.768953143238456e-02 + 3.759982501241896e-02 + 3.751033128919119e-02 + 3.742104976390229e-02 + 3.733197993101921e-02 + 3.724312127956968e-02 + 3.715447331892652e-02 + 3.706603555538170e-02 + 3.697780748276271e-02 + 3.688978860838169e-02 + 3.680197844047203e-02 + 3.671437648083043e-02 + 3.662698223859462e-02 + 3.653979522528233e-02 + 3.645281495050149e-02 + 3.636604092401768e-02 + 3.627947265609178e-02 + 3.619310965841749e-02 + 3.610695145022991e-02 + 3.602099755007137e-02 + 3.593524746722872e-02 + 3.584970072102484e-02 + 3.576435683502647e-02 + 3.567921532660211e-02 + 3.559427571435374e-02 + 3.550953752116312e-02 + 3.542500027693078e-02 + 3.534066350384148e-02 + 3.525652672170999e-02 + 3.517258946309138e-02 + 3.508885125628003e-02 + 3.500531162678291e-02 + 3.492197010940873e-02 + 3.483882623525817e-02 + 3.475587953263527e-02 + 3.467312953919303e-02 + 3.459057578876406e-02 + 3.450821781198862e-02 + 3.442605515251815e-02 + 3.434408734999198e-02 + 3.426231393723483e-02 + 3.418073445581654e-02 + 3.409934844816886e-02 + 3.401815545480084e-02 + 3.393715501912345e-02 + 3.385634668688416e-02 + 3.377573000557660e-02 + 3.369530452244349e-02 + 3.361506978424043e-02 + 3.353502533757853e-02 + 3.345517073173793e-02 + 3.337550551749302e-02 + 3.329602924688495e-02 + 3.321674147756795e-02 + 3.313764176353938e-02 + 3.305872964905672e-02 + 3.298000469915312e-02 + 3.290146647870388e-02 + 3.282311452928582e-02 + 3.274494841443989e-02 + 3.266696770524984e-02 + 3.258917195579586e-02 + 3.251156072835543e-02 + 3.243413358906379e-02 + 3.235689009597382e-02 + 3.227982981604406e-02 + 3.220295232152501e-02 + 3.212625717796667e-02 + 3.204974395335523e-02 + 3.197341221860803e-02 + 3.189726154412311e-02 + 3.182129149811844e-02 + 3.174550164996338e-02 + 3.166989158019418e-02 + 3.159446086552403e-02 + 3.151920907660396e-02 + 3.144413578635114e-02 + 3.136924057603768e-02 + 3.129452303273958e-02 + 3.121998273049806e-02 + 3.114561924523454e-02 + 3.107143216011856e-02 + 3.099742105799270e-02 + 3.092358552474080e-02 + 3.084992514925186e-02 + 3.077643951122557e-02 + 3.070312819305190e-02 + 3.062999078663620e-02 + 3.055702688363699e-02 + 3.048423607160382e-02 + 3.041161793396379e-02 + 3.033917207015753e-02 + 3.026689807846873e-02 + 3.019479554128978e-02 + 3.012286404982057e-02 + 3.005110320361238e-02 + 2.997951260681279e-02 + 2.990809185255966e-02 + 2.983684053226568e-02 + 2.976575824919644e-02 + 2.969484460522792e-02 + 2.962409920087884e-02 + 2.955352163824170e-02 + 2.948311151647172e-02 + 2.941286843696999e-02 + 2.934279201383451e-02 + 2.927288185012002e-02 + 2.920313754405053e-02 + 2.913355871301240e-02 + 2.906414496442064e-02 + 2.899489589842786e-02 + 2.892581113440367e-02 + 2.885689028447655e-02 + 2.878813295173805e-02 + 2.871953875071393e-02 + 2.865110730129668e-02 + 2.858283822298987e-02 + 2.851473112218837e-02 + 2.844678561226344e-02 + 2.837900131834860e-02 + 2.831137785774055e-02 + 2.824391484910513e-02 + 2.817661191561155e-02 + 2.810946867322990e-02 + 2.804248474261467e-02 + 2.797565975421473e-02 + 2.790899332866750e-02 + 2.784248508684310e-02 + 2.777613465729838e-02 + 2.770994166752542e-02 + 2.764390574354870e-02 + 2.757802651096095e-02 + 2.751230360164656e-02 + 2.744673664902718e-02 + 2.738132528259552e-02 + 2.731606912935157e-02 + 2.725096782054994e-02 + 2.718602099881922e-02 + 2.712122829441404e-02 + 2.705658933533107e-02 + 2.699210376654040e-02 + 2.692777122629619e-02 + 2.686359134766074e-02 + 2.679956376855050e-02 + 2.673568813034148e-02 + 2.667196407449100e-02 + 2.660839123653040e-02 + 2.654496925932325e-02 + 2.648169779209303e-02 + 2.641857647953893e-02 + 2.635560496230991e-02 + 2.629278288047702e-02 + 2.623010988365326e-02 + 2.616758562366618e-02 + 2.610520975031933e-02 + 2.604298190560967e-02 + 2.598090173551063e-02 + 2.591896889422694e-02 + 2.585718303968204e-02 + 2.579554382343284e-02 + 2.573405088890056e-02 + 2.567270388987125e-02 + 2.561150248485746e-02 + 2.555044633271165e-02 + 2.548953508798725e-02 + 2.542876840653832e-02 + 2.536814594843316e-02 + 2.530766736730480e-02 + 2.524733231969847e-02 + 2.518714047250964e-02 + 2.512709148651639e-02 + 2.506718502244662e-02 + 2.500742074738894e-02 + 2.494779831751132e-02 + 2.488831739145854e-02 + 2.482897764751662e-02 + 2.476977875068464e-02 + 2.471072035836960e-02 + 2.465180213557270e-02 + 2.459302375909131e-02 + 2.453438490494015e-02 + 2.447588522684439e-02 + 2.441752439632618e-02 + 2.435930209686822e-02 + 2.430121799563831e-02 + 2.424327176191257e-02 + 2.418546306971884e-02 + 2.412779159320392e-02 + 2.407025701015483e-02 + 2.401285899981317e-02 + 2.395559723318545e-02 + 2.389847138352894e-02 + 2.384148113010493e-02 + 2.378462615809818e-02 + 2.372790614977233e-02 + 2.367132078182184e-02 + 2.361486973012364e-02 + 2.355855267628842e-02 + 2.350236930912776e-02 + 2.344631931433045e-02 + 2.339040237260871e-02 + 2.333461816188979e-02 + 2.327896637661412e-02 + 2.322344670789084e-02 + 2.316805883183489e-02 + 2.311280243515378e-02 + 2.305767721100430e-02 + 2.300268285332459e-02 + 2.294781904779433e-02 + 2.289308548135576e-02 + 2.283848185163481e-02 + 2.278400785322520e-02 + 2.272966317657758e-02 + 2.267544750906289e-02 + 2.262136054576819e-02 + 2.256740198705395e-02 + 2.251357153295969e-02 + 2.245986887460843e-02 + 2.240629370372433e-02 + 2.235284572997816e-02 + 2.229952464835474e-02 + 2.224633014749699e-02 + 2.219326193897085e-02 + 2.214031972536749e-02 + 2.208750320104361e-02 + 2.203481207216736e-02 + 2.198224604054712e-02 + 2.192980480490446e-02 + 2.187748807809197e-02 + 2.182529556587840e-02 + 2.177322696546572e-02 + 2.172128198614426e-02 + 2.166946034002218e-02 + 2.161776173674180e-02 + 2.156618587797776e-02 + 2.151473246970182e-02 + 2.146340122648612e-02 + 2.141219186176581e-02 + 2.136110408595206e-02 + 2.131013760699968e-02 + 2.125929213929720e-02 + 2.120856739943827e-02 + 2.115796310282077e-02 + 2.110747896148574e-02 + 2.105711468773744e-02 + 2.100686999708853e-02 + 2.095674460640470e-02 + 2.090673823470377e-02 + 2.085685060390323e-02 + 2.080708143284909e-02 + 2.075743043865024e-02 + 2.070789733896461e-02 + 2.065848185338519e-02 + 2.060918370358626e-02 + 2.056000261352416e-02 + 2.051093830731156e-02 + 2.046199050841276e-02 + 2.041315893918832e-02 + 2.036444332670061e-02 + 2.031584339811113e-02 + 2.026735887003067e-02 + 2.021898947028043e-02 + 2.017073493461869e-02 + 2.012259498752183e-02 + 2.007456935661649e-02 + 2.002665777297167e-02 + 1.997885996051357e-02 + 1.993117564943755e-02 + 1.988360457741823e-02 + 1.983614647798445e-02 + 1.978880107994216e-02 + 1.974156811085027e-02 + 1.969444731166336e-02 + 1.964743841723802e-02 + 1.960054115227784e-02 + 1.955375525837354e-02 + 1.950708047707909e-02 + 1.946051654129042e-02 + 1.941406318497287e-02 + 1.936772014474174e-02 + 1.932148716119990e-02 + 1.927536398140690e-02 + 1.922935034564512e-02 + 1.918344598004621e-02 + 1.913765063001423e-02 + 1.909196404425976e-02 + 1.904638595925336e-02 + 1.900091612254435e-02 + 1.895555428061772e-02 + 1.891030016469744e-02 + 1.886515352637163e-02 + 1.882011411999479e-02 + 1.877518167521970e-02 + 1.873035594232305e-02 + 1.868563668110157e-02 + 1.864102363246424e-02 + 1.859651653826993e-02 + 1.855211514697215e-02 + 1.850781921692858e-02 + 1.846362849811174e-02 + 1.841954273425713e-02 + 1.837556167722399e-02 + 1.833168508274460e-02 + 1.828791270638760e-02 + 1.824424429587613e-02 + 1.820067960186819e-02 + 1.815721838079814e-02 + 1.811386039242822e-02 + 1.807060539155448e-02 + 1.802745312809132e-02 + 1.798440336027523e-02 + 1.794145584667328e-02 + 1.789861034396235e-02 + 1.785586661445831e-02 + 1.781322441713358e-02 + 1.777068350457842e-02 + 1.772824363606736e-02 + 1.768590457480606e-02 + 1.764366608520215e-02 + 1.760152792777148e-02 + 1.755948986314423e-02 + 1.751755165530868e-02 + 1.747571306959304e-02 + 1.743397386795286e-02 + 1.739233380686453e-02 + 1.735079266012027e-02 + 1.730935019969591e-02 + 1.726800617515195e-02 + 1.722676035861175e-02 + 1.718561252878033e-02 + 1.714456244423200e-02 + 1.710360987245781e-02 + 1.706275458478830e-02 + 1.702199634263978e-02 + 1.698133492251816e-02 + 1.694077010510486e-02 + 1.690030164730331e-02 + 1.685992932068962e-02 + 1.681965290804600e-02 + 1.677947217681714e-02 + 1.673938689521877e-02 + 1.669939683666028e-02 + 1.665950177991986e-02 + 1.661970150205939e-02 + 1.657999577711447e-02 + 1.654038437906895e-02 + 1.650086708478969e-02 + 1.646144367321789e-02 + 1.642211391798679e-02 + 1.638287759543946e-02 + 1.634373448754158e-02 + 1.630468437732129e-02 + 1.626572704186934e-02 + 1.622686225251448e-02 + 1.618808979718675e-02 + 1.614940946170872e-02 + 1.611082101996680e-02 + 1.607232425440962e-02 + 1.603391895041710e-02 + 1.599560489122978e-02 + 1.595738185903658e-02 + 1.591924963852954e-02 + 1.588120801872957e-02 + 1.584325677844977e-02 + 1.580539569909259e-02 + 1.576762457813194e-02 + 1.572994319938502e-02 + 1.569235134492561e-02 + 1.565484881311209e-02 + 1.561743538476679e-02 + 1.558011083839993e-02 + 1.554287497958433e-02 + 1.550572759639263e-02 + 1.546866846769255e-02 + 1.543169739218311e-02 + 1.539481416304957e-02 + 1.535801856691179e-02 + 1.532131039389481e-02 + 1.528468944222785e-02 + 1.524815551153465e-02 + 1.521170838270258e-02 + 1.517534784896265e-02 + 1.513907371629201e-02 + 1.510288577422132e-02 + 1.506678381600347e-02 + 1.503076764230081e-02 + 1.499483704594779e-02 + 1.495899182406707e-02 + 1.492323178010325e-02 + 1.488755670734373e-02 + 1.485196640180961e-02 + 1.481646066682016e-02 + 1.478103929846549e-02 + 1.474570209596410e-02 + 1.471044886603309e-02 + 1.467527940456368e-02 + 1.464019351058384e-02 + 1.460519099481427e-02 + 1.457027165550810e-02 + 1.453543528903122e-02 + 1.450068170024789e-02 + 1.446601069487559e-02 + 1.443142207965322e-02 + 1.439691566271563e-02 + 1.436249124410753e-02 + 1.432814862414793e-02 + 1.429388761459182e-02 + 1.425970802314729e-02 + 1.422560965483165e-02 + 1.419159231701311e-02 + 1.415765581387266e-02 + 1.412379995194592e-02 + 1.409002455116132e-02 + 1.405632941792580e-02 + 1.402271435207008e-02 + 1.398917917303820e-02 + 1.395572369124169e-02 + 1.392234770932106e-02 + 1.388905104392205e-02 + 1.385583350997071e-02 + 1.382269491788422e-02 + 1.378963507987753e-02 + 1.375665380791922e-02 + 1.372375091366015e-02 + 1.369092621121959e-02 + 1.365817951715469e-02 + 1.362551064978645e-02 + 1.359291942581090e-02 + 1.356040565686591e-02 + 1.352796915179558e-02 + 1.349560973812908e-02 + 1.346332723540702e-02 + 1.343112144483463e-02 + 1.339899219360082e-02 + 1.336693930891053e-02 + 1.333496260043193e-02 + 1.330306188734133e-02 + 1.327123699224855e-02 + 1.323948773476910e-02 + 1.320781393642416e-02 + 1.317621541705235e-02 + 1.314469199166208e-02 + 1.311324348463446e-02 + 1.308186972415766e-02 + 1.305057053367403e-02 + 1.301934573289238e-02 + 1.298819514247371e-02 + 1.295711859027024e-02 + 1.292611590142988e-02 + 1.289518689817567e-02 + 1.286433140253433e-02 + 1.283354924175516e-02 + 1.280284024626999e-02 + 1.277220424320462e-02 + 1.274164105512193e-02 + 1.271115050454398e-02 + 1.268073242580564e-02 + 1.265038664885281e-02 + 1.262011299771964e-02 + 1.258991130045622e-02 + 1.255978138928737e-02 + 1.252972309744999e-02 + 1.249973624798444e-02 + 1.246982067106488e-02 + 1.243997620568067e-02 + 1.241020267461013e-02 + 1.238049990730756e-02 + 1.235086774662455e-02 + 1.232130602051964e-02 + 1.229181455798258e-02 + 1.226239319685087e-02 + 1.223304176770951e-02 + 1.220376010347082e-02 + 1.217454804486660e-02 + 1.214540542341299e-02 + 1.211633207079968e-02 + 1.208732782681391e-02 + 1.205839252771904e-02 + 1.202952600673308e-02 + 1.200072809652295e-02 + 1.197199864009422e-02 + 1.194333748171596e-02 + 1.191474445503102e-02 + 1.188621939116683e-02 + 1.185776212675846e-02 + 1.182937251288597e-02 + 1.180105038815069e-02 + 1.177279558400601e-02 + 1.174460794107161e-02 + 1.171648730396891e-02 + 1.168843351609301e-02 + 1.166044641171630e-02 + 1.163252583370262e-02 + 1.160467163102319e-02 + 1.157688364263099e-02 + 1.154916170933044e-02 + 1.152150567570712e-02 + 1.149391538555893e-02 + 1.146639068095789e-02 + 1.143893140447911e-02 + 1.141153740781534e-02 + 1.138420853673168e-02 + 1.135694462854312e-02 + 1.132974552839056e-02 + 1.130261108533809e-02 + 1.127554114921167e-02 + 1.124853556496432e-02 + 1.122159417997002e-02 + 1.119471684585664e-02 + 1.116790340283271e-02 + 1.114115369710312e-02 + 1.111446758944324e-02 + 1.108784492177717e-02 + 1.106128553754357e-02 + 1.103478929728668e-02 + 1.100835604711145e-02 + 1.098198563223047e-02 + 1.095567791133747e-02 + 1.092943273475230e-02 + 1.090324994987072e-02 + 1.087712940981543e-02 + 1.085107096608238e-02 + 1.082507447053145e-02 + 1.079913977931768e-02 + 1.077326674497864e-02 + 1.074745521859533e-02 + 1.072170505538783e-02 + 1.069601611066200e-02 + 1.067038823886727e-02 + 1.064482129354728e-02 + 1.061931513219808e-02 + 1.059386961330933e-02 + 1.056848458681972e-02 + 1.054315990640127e-02 + 1.051789543133304e-02 + 1.049269102209236e-02 + 1.046754653920045e-02 + 1.044246184044743e-02 + 1.041743677288729e-02 + 1.039247119540387e-02 + 1.036756497988909e-02 + 1.034271797805630e-02 + 1.031793004647734e-02 + 1.029320105343422e-02 + 1.026853085073464e-02 + 1.024391929539164e-02 + 1.021936625852857e-02 + 1.019487159885800e-02 + 1.017043517132599e-02 + 1.014605683522427e-02 + 1.012173646293816e-02 + 1.009747392058842e-02 + 1.007326905588179e-02 + 1.004912173629082e-02 + 1.002503183346208e-02 + 1.000099920653610e-02 + 9.977023717087764e-03 + 9.953105229386335e-03 + 9.929243608872406e-03 + 9.905438720973011e-03 + 9.881690430827046e-03 + 9.857998603258316e-03 + 9.834363104786413e-03 + 9.810783802355094e-03 + 9.787260560990621e-03 + 9.763793247942843e-03 + 9.740381731591760e-03 + 9.717025877967826e-03 + 9.693725554510663e-03 + 9.670480630152000e-03 + 9.647290973567844e-03 + 9.624156453317901e-03 + 9.601076938062841e-03 + 9.578052297039983e-03 + 9.555082399653962e-03 + 9.532167115690399e-03 + 9.509306316445141e-03 + 9.486499872512266e-03 + 9.463747653553711e-03 + 9.441049530896879e-03 + 9.418405376662433e-03 + 9.395815063104617e-03 + 9.373278461427175e-03 + 9.350795443499088e-03 + 9.328365882617906e-03 + 9.305989651970981e-03 + 9.283666624824176e-03 + 9.261396674723778e-03 + 9.239179675408710e-03 + 9.217015500757821e-03 + 9.194904024947497e-03 + 9.172845123632957e-03 + 9.150838672277813e-03 + 9.128884544858834e-03 + 9.106982617538152e-03 + 9.085132766987650e-03 + 9.063334868396023e-03 + 9.041588798714301e-03 + 9.019894435479832e-03 + 8.998251654894885e-03 + 8.976660334271958e-03 + 8.955120351755344e-03 + 8.933631585369030e-03 + 8.912193913264732e-03 + 8.890807213885918e-03 + 8.869471366214972e-03 + 8.848186249259440e-03 + 8.826951742251321e-03 + 8.805767725383001e-03 + 8.784634078744540e-03 + 8.763550682317645e-03 + 8.742517416743529e-03 + 8.721534162832835e-03 + 8.700600801502468e-03 + 8.679717214101214e-03 + 8.658883282492529e-03 + 8.638098888885700e-03 + 8.617363915025602e-03 + 8.596678243443276e-03 + 8.576041757693640e-03 + 8.555454340593486e-03 + 8.534915875110577e-03 + 8.514426244844865e-03 + 8.493985333674233e-03 + 8.473593025909724e-03 + 8.453249206335058e-03 + 8.432953759374154e-03 + 8.412706569850796e-03 + 8.392507523503252e-03 + 8.372356505624580e-03 + 8.352253401838657e-03 + 8.332198098800467e-03 + 8.312190482450988e-03 + 8.292230438871008e-03 + 8.272317855399050e-03 + 8.252452619663047e-03 + 8.232634619050869e-03 + 8.212863740331295e-03 + 8.193139871755427e-03 + 8.173462902270600e-03 + 8.153832719956143e-03 + 8.134249213438031e-03 + 8.114712271802299e-03 + 8.095221784035302e-03 + 8.075777639824947e-03 + 8.056379729417138e-03 + 8.037027942938956e-03 + 8.017722170154111e-03 + 7.998462301028743e-03 + 7.979248227381455e-03 + 7.960079840401237e-03 + 7.940957030626293e-03 + 7.921879690329333e-03 + 7.902847711453821e-03 + 7.883860985421759e-03 + 7.864919405368889e-03 + 7.846022864037674e-03 + 7.827171253341911e-03 + 7.808364466777525e-03 + 7.789602397948123e-03 + 7.770884940051978e-03 + 7.752211987488857e-03 + 7.733583434567862e-03 + 7.714999174963997e-03 + 7.696459103766686e-03 + 7.677963116338556e-03 + 7.659511107448910e-03 + 7.641102971817726e-03 + 7.622738605046243e-03 + 7.604417904226960e-03 + 7.586140765207791e-03 + 7.567907083543369e-03 + 7.549716755813157e-03 + 7.531569679511053e-03 + 7.513465752109027e-03 + 7.495404870064185e-03 + 7.477386931039745e-03 + 7.459411833357052e-03 + 7.441479474804043e-03 + 7.423589753808636e-03 + 7.405742569108545e-03 + 7.387937818890154e-03 + 7.370175402616828e-03 + 7.352455220325825e-03 + 7.334777170378787e-03 + 7.317141152312192e-03 + 7.299547066866589e-03 + 7.281994814230038e-03 + 7.264484294760120e-03 + 7.247015409129196e-03 + 7.229588058070396e-03 + 7.212202142721502e-03 + 7.194857564639825e-03 + 7.177554225359851e-03 + 7.160292026920549e-03 + 7.143070871828701e-03 + 7.125890661781023e-03 + 7.108751299151369e-03 + 7.091652687463649e-03 + 7.074594729473165e-03 + 7.057577328092346e-03 + 7.040600386920140e-03 + 7.023663809440293e-03 + 7.006767499621119e-03 + 6.989911362283671e-03 + 6.973095301761098e-03 + 6.956319222246642e-03 + 6.939583028242014e-03 + 6.922886624933184e-03 + 6.906229917977133e-03 + 6.889612813237578e-03 + 6.873035215956658e-03 + 6.856497031627250e-03 + 6.839998167124653e-03 + 6.823538528849983e-03 + 6.807118022950776e-03 + 6.790736556102037e-03 + 6.774394036012678e-03 + 6.758090370503938e-03 + 6.741825465874382e-03 + 6.725599229831710e-03 + 6.709411571076730e-03 + 6.693262397262002e-03 + 6.677151616823099e-03 + 6.661079138883049e-03 + 6.645044871881905e-03 + 6.629048724424302e-03 + 6.613090605727208e-03 + 6.597170426169362e-03 + 6.581288095176429e-03 + 6.565443521393368e-03 + 6.549636616015259e-03 + 6.533867289690503e-03 + 6.518135451870410e-03 + 6.502441013826720e-03 + 6.486783886693610e-03 + 6.471163980906190e-03 + 6.455581208648475e-03 + 6.440035481784445e-03 + 6.424526710952946e-03 + 6.409054808194274e-03 + 6.393619686148648e-03 + 6.378221257389647e-03 + 6.362859434216829e-03 + 6.347534129405275e-03 + 6.332245256702231e-03 + 6.316992728926398e-03 + 6.301776458848883e-03 + 6.286596360249587e-03 + 6.271452347303485e-03 + 6.256344334335297e-03 + 6.241272235637497e-03 + 6.226235965289892e-03 + 6.211235437617355e-03 + 6.196270567881593e-03 + 6.181341271239764e-03 + 6.166447462747857e-03 + 6.151589057705613e-03 + 6.136765971896623e-03 + 6.121978121401951e-03 + 6.107225422158738e-03 + 6.092507790132517e-03 + 6.077825141615326e-03 + 6.063177393866219e-03 + 6.048564463915550e-03 + 6.033986268442907e-03 + 6.019442724444331e-03 + 6.004933749322879e-03 + 5.990459260846786e-03 + 5.976019176895193e-03 + 5.961613415583682e-03 + 5.947241895231883e-03 + 5.932904533953020e-03 + 5.918601250164555e-03 + 5.904331962789744e-03 + 5.890096590861083e-03 + 5.875895053571317e-03 + 5.861727270328322e-03 + 5.847593160694864e-03 + 5.833492644473149e-03 + 5.819425641715960e-03 + 5.805392072163253e-03 + 5.791391855962741e-03 + 5.777424914193254e-03 + 5.763491167389796e-03 + 5.749590536018355e-03 + 5.735722941127108e-03 + 5.721888304505554e-03 + 5.708086547766434e-03 + 5.694317591457392e-03 + 5.680581357894303e-03 + 5.666877769868901e-03 + 5.653206748498332e-03 + 5.639568216215936e-03 + 5.625962096156960e-03 + 5.612388310444156e-03 + 5.598846782244085e-03 + 5.585337435180374e-03 + 5.571860191464977e-03 + 5.558414974887664e-03 + 5.545001710146245e-03 + 5.531620319651646e-03 + 5.518270727342278e-03 + 5.504952858629092e-03 + 5.491666637149790e-03 + 5.478411987013346e-03 + 5.465188833301630e-03 + 5.451997101098349e-03 + 5.438836715460566e-03 + 5.425707601454353e-03 + 5.412609684284744e-03 + 5.399542889597323e-03 + 5.386507143481766e-03 + 5.373502371672819e-03 + 5.360528500347250e-03 + 5.347585456320476e-03 + 5.334673165238440e-03 + 5.321791553290573e-03 + 5.308940548233661e-03 + 5.296120077396591e-03 + 5.283330067535706e-03 + 5.270570445063373e-03 + 5.257841138170363e-03 + 5.245142075198895e-03 + 5.232473183108559e-03 + 5.219834389656078e-03 + 5.207225623450342e-03 + 5.194646813662877e-03 + 5.182097888019903e-03 + 5.169578774433769e-03 + 5.157089403379224e-03 + 5.144629703265777e-03 + 5.132199601938705e-03 + 5.119799030212023e-03 + 5.107427917986021e-03 + 5.095086194230444e-03 + 5.082773788422456e-03 + 5.070490630851877e-03 + 5.058236652191947e-03 + 5.046011782411770e-03 + 5.033815952050060e-03 + 5.021649092096963e-03 + 5.009511132708167e-03 + 4.997402004872952e-03 + 4.985321640554899e-03 + 4.973269971256597e-03 + 4.961246927915243e-03 + 4.949252441362114e-03 + 4.937286444284467e-03 + 4.925348869112448e-03 + 4.913439647407981e-03 + 4.901558711796497e-03 + 4.889705994683389e-03 + 4.877881427891544e-03 + 4.866084944974813e-03 + 4.854316479151838e-03 + 4.842575962313763e-03 + 4.830863328342665e-03 + 4.819178511323663e-03 + 4.807521444209243e-03 + 4.795892060902922e-03 + 4.784290295384500e-03 + 4.772716080911616e-03 + 4.761169351999607e-03 + 4.749650043687660e-03 + 4.738158090505544e-03 + 4.726693426328077e-03 + 4.715255985490762e-03 + 4.703845704177584e-03 + 4.692462517255530e-03 + 4.681106359266913e-03 + 4.669777166671382e-03 + 4.658474874679097e-03 + 4.647199418051047e-03 + 4.635950733693824e-03 + 4.624728757814554e-03 + 4.613533426035275e-03 + 4.602364675085555e-03 + 4.591222441193854e-03 + 4.580106660340010e-03 + 4.569017270133631e-03 + 4.557954207729684e-03 + 4.546917409606468e-03 + 4.535906813063030e-03 + 4.524922355662980e-03 + 4.513963975024341e-03 + 4.503031608847450e-03 + 4.492125194604706e-03 + 4.481244669721014e-03 + 4.470389973075004e-03 + 4.459561043403103e-03 + 4.448757818627942e-03 + 4.437980237157132e-03 + 4.427228237584914e-03 + 4.416501758569245e-03 + 4.405800739475310e-03 + 4.395125119625431e-03 + 4.384474837838549e-03 + 4.373849833628650e-03 + 4.363250046735974e-03 + 4.352675416676511e-03 + 4.342125883231542e-03 + 4.331601386396421e-03 + 4.321101866314548e-03 + 4.310627263408189e-03 + 4.300177518182486e-03 + 4.289752570999768e-03 + 4.279352362569397e-03 + 4.268976833692708e-03 + 4.258625924837488e-03 + 4.248299577436886e-03 + 4.237997733292803e-03 + 4.227720333115902e-03 + 4.217467318344088e-03 + 4.207238631130676e-03 + 4.197034213451204e-03 + 4.186854006892898e-03 + 4.176697952964437e-03 + 4.166565994071401e-03 + 4.156458073034852e-03 + 4.146374132669977e-03 + 4.136314114934178e-03 + 4.126277962231265e-03 + 4.116265617785051e-03 + 4.106277025130100e-03 + 4.096312127225507e-03 + 4.086370866498099e-03 + 4.076453186851077e-03 + 4.066559032318193e-03 + 4.056688346384167e-03 + 4.046841072375552e-03 + 4.037017154234894e-03 + 4.027216536753351e-03 + 4.017439163691562e-03 + 4.007684978984143e-03 + 3.997953927616382e-03 + 3.988245954341580e-03 + 3.978561003853811e-03 + 3.968899020987378e-03 + 3.959259950174110e-03 + 3.949643736363211e-03 + 3.940050325917084e-03 + 3.930479663940112e-03 + 3.920931695287086e-03 + 3.911406366153988e-03 + 3.901903622192779e-03 + 3.892423408886956e-03 + 3.882965672529350e-03 + 3.873530359308857e-03 + 3.864117415340676e-03 + 3.854726787040829e-03 + 3.845358421069152e-03 + 3.836012264203419e-03 + 3.826688263119290e-03 + 3.817386364639579e-03 + 3.808106515834866e-03 + 3.798848664119608e-03 + 3.789612756624806e-03 + 3.780398740246358e-03 + 3.771206562411011e-03 + 3.762036171140922e-03 + 3.752887514765301e-03 + 3.743760540740341e-03 + 3.734655196719529e-03 + 3.725571430980986e-03 + 3.716509192135922e-03 + 3.707468428387338e-03 + 3.698449087559120e-03 + 3.689451119263604e-03 + 3.680474472533827e-03 + 3.671519094850921e-03 + 3.662584935894309e-03 + 3.653671945297336e-03 + 3.644780071212910e-03 + 3.635909263417586e-03 + 3.627059471971036e-03 + 3.618230645951014e-03 + 3.609422734605362e-03 + 3.600635687799489e-03 + 3.591869456282116e-03 + 3.583123989955615e-03 + 3.574399238584414e-03 + 3.565695152845392e-03 + 3.557011682930304e-03 + 3.548348778903569e-03 + 3.539706391532652e-03 + 3.531084471778191e-03 + 3.522482970666699e-03 + 3.513901839239252e-03 + 3.505341028388093e-03 + 3.496800489111114e-03 + 3.488280173119797e-03 + 3.479780031892506e-03 + 3.471300016604386e-03 + 3.462840078578430e-03 + 3.454400170006921e-03 + 3.445980243580072e-03 + 3.437580250715228e-03 + 3.429200142992534e-03 + 3.420839872683964e-03 + 3.412499392748279e-03 + 3.404178655562702e-03 + 3.395877612913789e-03 + 3.387596218304602e-03 + 3.379334424898988e-03 + 3.371092184764359e-03 + 3.362869450661228e-03 + 3.354666175991870e-03 + 3.346482314543412e-03 + 3.338317819175880e-03 + 3.330172642972068e-03 + 3.322046740007612e-03 + 3.313940064235304e-03 + 3.305852569386344e-03 + 3.297784208976364e-03 + 3.289734936574093e-03 + 3.281704706272533e-03 + 3.273693473070924e-03 + 3.265701191257636e-03 + 3.257727814803984e-03 + 3.249773298074863e-03 + 3.241837595979994e-03 + 3.233920663596386e-03 + 3.226022455622204e-03 + 3.218142926555998e-03 + 3.210282031163023e-03 + 3.202439725215123e-03 + 3.194615964134223e-03 + 3.186810703067027e-03 + 3.179023897602838e-03 + 3.171255503398923e-03 + 3.163505475998486e-03 + 3.155773770652277e-03 + 3.148060343553685e-03 + 3.140365151430291e-03 + 3.132688149313277e-03 + 3.125029293443853e-03 + 3.117388541383308e-03 + 3.109765848736372e-03 + 3.102161171408776e-03 + 3.094574466373093e-03 + 3.087005690728782e-03 + 3.079454801280003e-03 + 3.071921754555924e-03 + 3.064406507880629e-03 + 3.056909018530570e-03 + 3.049429243338136e-03 + 3.041967139485151e-03 + 3.034522664377966e-03 + 3.027095775562892e-03 + 3.019686430816400e-03 + 3.012294588055786e-03 + 3.004920205212209e-03 + 2.997563239705405e-03 + 2.990223649173174e-03 + 2.982901392198357e-03 + 2.975596427311395e-03 + 2.968308712822513e-03 + 2.961038206776656e-03 + 2.953784867392937e-03 + 2.946548653201694e-03 + 2.939329523194637e-03 + 2.932127436582907e-03 + 2.924942352269057e-03 + 2.917774228121468e-03 + 2.910623023643702e-03 + 2.903488699056754e-03 + 2.896371212514376e-03 + 2.889270523304112e-03 + 2.882186591679052e-03 + 2.875119376658924e-03 + 2.868068837676741e-03 + 2.861034934776638e-03 + 2.854017627809473e-03 + 2.847016876513213e-03 + 2.840032640686938e-03 + 2.833064880681456e-03 + 2.826113557103426e-03 + 2.819178630449384e-03 + 2.812260060105674e-03 + 2.805357806481411e-03 + 2.798471831428214e-03 + 2.791602094948384e-03 + 2.784748557109701e-03 + 2.777911179108391e-03 + 2.771089922400815e-03 + 2.764284748007964e-03 + 2.757495616304825e-03 + 2.750722488782177e-03 + 2.743965327115693e-03 + 2.737224092409578e-03 + 2.730498746008289e-03 + 2.723789249648372e-03 + 2.717095565426717e-03 + 2.710417654358784e-03 + 2.703755477800617e-03 + 2.697108999073383e-03 + 2.690478179939920e-03 + 2.683862981660020e-03 + 2.677263366970269e-03 + 2.670679298051182e-03 + 2.664110736894467e-03 + 2.657557646411518e-03 + 2.651019989061974e-03 + 2.644497727126834e-03 + 2.637990823748049e-03 + 2.631499241776077e-03 + 2.625022943767310e-03 + 2.618561892619121e-03 + 2.612116051581085e-03 + 2.605685383984310e-03 + 2.599269852507916e-03 + 2.592869420565223e-03 + 2.586484052278204e-03 + 2.580113710814501e-03 + 2.573758359383719e-03 + 2.567417961609419e-03 + 2.561092481362380e-03 + 2.554781882413309e-03 + 2.548486128464411e-03 + 2.542205184036424e-03 + 2.535939013422343e-03 + 2.529687580238901e-03 + 2.523450848436006e-03 + 2.517228782352248e-03 + 2.511021346720505e-03 + 2.504828506444104e-03 + 2.498650225958320e-03 + 2.492486468967693e-03 + 2.486337200680032e-03 + 2.480202386472821e-03 + 2.474081990653096e-03 + 2.467975978460781e-03 + 2.461884315215007e-03 + 2.455806965243642e-03 + 2.449743893777428e-03 + 2.443695066608286e-03 + 2.437660449328877e-03 + 2.431640007119714e-03 + 2.425633705211946e-03 + 2.419641509634817e-03 + 2.413663385966202e-03 + 2.407699299632079e-03 + 2.401749216909934e-03 + 2.395813103777745e-03 + 2.389890925926915e-03 + 2.383982649399740e-03 + 2.378088240561300e-03 + 2.372207665865560e-03 + 2.366340891258303e-03 + 2.360487883037754e-03 + 2.354648608018378e-03 + 2.348823033118129e-03 + 2.343011124539103e-03 + 2.337212848054228e-03 + 2.331428171390765e-03 + 2.325657062040943e-03 + 2.319899486412222e-03 + 2.314155410784699e-03 + 2.308424802407836e-03 + 2.302707629600105e-03 + 2.297003858807791e-03 + 2.291313456815245e-03 + 2.285636391934661e-03 + 2.279972631361444e-03 + 2.274322142294874e-03 + 2.268684892803898e-03 + 2.263060850445441e-03 + 2.257449982675230e-03 + 2.251852257331842e-03 + 2.246267642469826e-03 + 2.240696106224612e-03 + 2.235137616698782e-03 + 2.229592141925271e-03 + 2.224059650030012e-03 + 2.218540109447705e-03 + 2.213033488566449e-03 + 2.207539755750870e-03 + 2.202058879448589e-03 + 2.196590827868759e-03 + 2.191135569406817e-03 + 2.185693073538250e-03 + 2.180263309295804e-03 + 2.174846245146934e-03 + 2.169441849486111e-03 + 2.164050091629877e-03 + 2.158670941358884e-03 + 2.153304367034930e-03 + 2.147950337590323e-03 + 2.142608822828710e-03 + 2.137279792340324e-03 + 2.131963215445541e-03 + 2.126659061301170e-03 + 2.121367299363035e-03 + 2.116087899310018e-03 + 2.110820830985273e-03 + 2.105566064252215e-03 + 2.100323568822542e-03 + 2.095093314356113e-03 + 2.089875271479949e-03 + 2.084669410319118e-03 + 2.079475699908646e-03 + 2.074294110902819e-03 + 2.069124614088778e-03 + 2.063967179294363e-03 + 2.058821776747180e-03 + 2.053688376991423e-03 + 2.048566950718572e-03 + 2.043457468582260e-03 + 2.038359901240009e-03 + 2.033274219398007e-03 + 2.028200393563254e-03 + 2.023138394473247e-03 + 2.018088193625467e-03 + 2.013049761808302e-03 + 2.008023069533293e-03 + 2.003008088023720e-03 + 1.998004788922133e-03 + 1.993013143739461e-03 + 1.988033123025423e-03 + 1.983064698401174e-03 + 1.978107841939004e-03 + 1.973162523959716e-03 + 1.968228716337500e-03 + 1.963306391971906e-03 + 1.958395521239848e-03 + 1.953496075620802e-03 + 1.948608028019931e-03 + 1.943731350228688e-03 + 1.938866013789267e-03 + 1.934011990485791e-03 + 1.929169252901841e-03 + 1.924337773155020e-03 + 1.919517522737516e-03 + 1.914708474346116e-03 + 1.909910600724645e-03 + 1.905123874118434e-03 + 1.900348266781407e-03 + 1.895583751238830e-03 + 1.890830300301618e-03 + 1.886087886140893e-03 + 1.881356481302734e-03 + 1.876636059328014e-03 + 1.871926593048255e-03 + 1.867228054886164e-03 + 1.862540417322874e-03 + 1.857863653843670e-03 + 1.853197738035545e-03 + 1.848542642643349e-03 + 1.843898340659710e-03 + 1.839264805482605e-03 + 1.834642010884087e-03 + 1.830029929675155e-03 + 1.825428534756660e-03 + 1.820837800826725e-03 + 1.816257700994342e-03 + 1.811688207883416e-03 + 1.807129296515363e-03 + 1.802580940478343e-03 + 1.798043112569654e-03 + 1.793515787775185e-03 + 1.788998939681262e-03 + 1.784492541016137e-03 + 1.779996567290557e-03 + 1.775510992946437e-03 + 1.771035790955467e-03 + 1.766570935188881e-03 + 1.762116400368900e-03 + 1.757672161650980e-03 + 1.753238193179788e-03 + 1.748814468721301e-03 + 1.744400962234815e-03 + 1.739997649147064e-03 + 1.735604504535336e-03 + 1.731221502449809e-03 + 1.726848617398671e-03 + 1.722485824273258e-03 + 1.718133098140551e-03 + 1.713790413411143e-03 + 1.709457744960287e-03 + 1.705135068710917e-03 + 1.700822359316501e-03 + 1.696519591326764e-03 + 1.692226740206531e-03 + 1.687943780998710e-03 + 1.683670689031842e-03 + 1.679407440586236e-03 + 1.675154010255336e-03 + 1.670910372590760e-03 + 1.666676504276520e-03 + 1.662452380680008e-03 + 1.658237976786625e-03 + 1.654033269138933e-03 + 1.649838233189823e-03 + 1.645652843894769e-03 + 1.641477077483747e-03 + 1.637310910181093e-03 + 1.633154317979306e-03 + 1.629007276710043e-03 + 1.624869761977525e-03 + 1.620741749539434e-03 + 1.616623216289868e-03 + 1.612514138626602e-03 + 1.608414492386588e-03 + 1.604324254066237e-03 + 1.600243399893093e-03 + 1.596171905775712e-03 + 1.592109748332882e-03 + 1.588056904380057e-03 + 1.584013350591343e-03 + 1.579979063015354e-03 + 1.575954018417919e-03 + 1.571938194458784e-03 + 1.567931567017785e-03 + 1.563934112240304e-03 + 1.559945807641685e-03 + 1.555966630779868e-03 + 1.551996558434039e-03 + 1.548035566441649e-03 + 1.544083632647983e-03 + 1.540140734902817e-03 + 1.536206849423688e-03 + 1.532281953055403e-03 + 1.528366023373284e-03 + 1.524459038476067e-03 + 1.520560975283225e-03 + 1.516671810636623e-03 + 1.512791522697568e-03 + 1.508920088807869e-03 + 1.505057486015793e-03 + 1.501203692209861e-03 + 1.497358685322736e-03 + 1.493522443062933e-03 + 1.489694942724099e-03 + 1.485876162194853e-03 + 1.482066079697174e-03 + 1.478264672890618e-03 + 1.474471919644568e-03 + 1.470687798121700e-03 + 1.466912286519041e-03 + 1.463145362594138e-03 + 1.459387003990816e-03 + 1.455637189563730e-03 + 1.451895897787254e-03 + 1.448163106500405e-03 + 1.444438793947980e-03 + 1.440722938760434e-03 + 1.437015519703072e-03 + 1.433316514691718e-03 + 1.429625901910583e-03 + 1.425943660193772e-03 + 1.422269768088744e-03 + 1.418604204318803e-03 + 1.414946947952337e-03 + 1.411297977567125e-03 + 1.407657271824714e-03 + 1.404024809794961e-03 + 1.400400570139807e-03 + 1.396784531434119e-03 + 1.393176672533590e-03 + 1.389576972970703e-03 + 1.385985412235932e-03 + 1.382401969107074e-03 + 1.378826622443783e-03 + 1.375259351219489e-03 + 1.371700134514839e-03 + 1.368148952079336e-03 + 1.364605783757036e-03 + 1.361070608632290e-03 + 1.357543405835987e-03 + 1.354024154621594e-03 + 1.350512834317403e-03 + 1.347009424934256e-03 + 1.343513906652181e-03 + 1.340026258589547e-03 + 1.336546459739890e-03 + 1.333074489680093e-03 + 1.329610329681042e-03 + 1.326153959375566e-03 + 1.322705357081586e-03 + 1.319264503327096e-03 + 1.315831378649060e-03 + 1.312405962942001e-03 + 1.308988235813027e-03 + 1.305578177302038e-03 + 1.302175767936687e-03 + 1.298780987783750e-03 + 1.295393816813521e-03 + 1.292014235092867e-03 + 1.288642222828844e-03 + 1.285277760581296e-03 + 1.281920829182064e-03 + 1.278571408463430e-03 + 1.275229478684235e-03 + 1.271895021211065e-03 + 1.268568016022571e-03 + 1.265248443161927e-03 + 1.261936283866222e-03 + 1.258631518851812e-03 + 1.255334128626978e-03 + 1.252044093890807e-03 + 1.248761395151665e-03 + 1.245486013113075e-03 + 1.242217929129610e-03 + 1.238957124307338e-03 + 1.235703579418827e-03 + 1.232457274959363e-03 + 1.229218191838193e-03 + 1.225986311430220e-03 + 1.222761615473843e-03 + 1.219544084856006e-03 + 1.216333699988417e-03 + 1.213130441964228e-03 + 1.209934292565758e-03 + 1.206745233676102e-03 + 1.203563245963205e-03 + 1.200388310525061e-03 + 1.197220409053557e-03 + 1.194059523052293e-03 + 1.190905633902454e-03 + 1.187758723052301e-03 + 1.184618772531461e-03 + 1.181485763720917e-03 + 1.178359677471923e-03 + 1.175240496495629e-03 + 1.172128202790389e-03 + 1.169022777016151e-03 + 1.165924201427151e-03 + 1.162832457974330e-03 + 1.159747527667428e-03 + 1.156669393342115e-03 + 1.153598037513975e-03 + 1.150533441195656e-03 + 1.147475586343000e-03 + 1.144424455192698e-03 + 1.141380029693999e-03 + 1.138342292070108e-03 + 1.135311224784675e-03 + 1.132286810371483e-03 + 1.129269030382949e-03 + 1.126257866687504e-03 + 1.123253302806332e-03 + 1.120255320585528e-03 + 1.117263901594027e-03 + 1.114279029333705e-03 + 1.111300685926414e-03 + 1.108328853214485e-03 + 1.105363515126287e-03 + 1.102404653695880e-03 + 1.099452250276090e-03 + 1.096506289065756e-03 + 1.093566752790314e-03 + 1.090633622975802e-03 + 1.087706882683815e-03 + 1.084786514992603e-03 + 1.081872502714538e-03 + 1.078964828755093e-03 + 1.076063476054719e-03 + 1.073168427526746e-03 + 1.070279665938792e-03 + 1.067397174442816e-03 + 1.064520936417689e-03 + 1.061650934087830e-03 + 1.058787150577005e-03 + 1.055929570192933e-03 + 1.053078175257412e-03 + 1.050232948596978e-03 + 1.047393874491941e-03 + 1.044560935952646e-03 + 1.041734115621369e-03 + 1.038913396581877e-03 + 1.036098763274602e-03 + 1.033290199394467e-03 + 1.030487686748966e-03 + 1.027691209748195e-03 + 1.024900752824401e-03 + 1.022116298137994e-03 + 1.019337829638554e-03 + 1.016565331581354e-03 + 1.013798786681050e-03 + 1.011038178744547e-03 + 1.008283492069452e-03 + 1.005534710312413e-03 + 1.002791817167788e-03 + 1.000054796401326e-03 + 9.973236317525795e-04 + 9.945983071579347e-04 + 9.918788066535795e-04 + 9.891651141399331e-04 + 9.864572135795289e-04 + 9.837550890187182e-04 + 9.810587245440681e-04 + 9.783681042165289e-04 + 9.756832121036981e-04 + 9.730040324093254e-04 + 9.703305493295432e-04 + 9.676627470403841e-04 + 9.650006098016114e-04 + 9.623441219373007e-04 + 9.596932677964791e-04 + 9.570480316324495e-04 + 9.544083978205768e-04 + 9.517743508934125e-04 + 9.491458752396962e-04 + 9.465229553012878e-04 + 9.439055756501852e-04 + 9.412937207895038e-04 + 9.386873752809525e-04 + 9.360865238076440e-04 + 9.334911509791107e-04 + 9.309012414140627e-04 + 9.283167798171848e-04 + 9.257377509839119e-04 + 9.231641396993918e-04 + 9.205959306774516e-04 + 9.180331088080401e-04 + 9.154756590138127e-04 + 9.129235660962607e-04 + 9.103768149701878e-04 + 9.078353906321772e-04 + 9.052992780955343e-04 + 9.027684623946166e-04 + 9.002429285800444e-04 + 8.977226617159664e-04 + 8.952076469091944e-04 + 8.926978693026045e-04 + 8.901933140640126e-04 + 8.876939664445818e-04 + 8.851998117353022e-04 + 8.827108351548252e-04 + 8.802270219825557e-04 + 8.777483575705740e-04 + 8.752748272927118e-04 + 8.728064165680979e-04 + 8.703431108522175e-04 + 8.678848955854838e-04 + 8.654317562117358e-04 + 8.629836782258149e-04 + 8.605406473168484e-04 + 8.581026490766350e-04 + 8.556696689660546e-04 + 8.532416927295966e-04 + 8.508187061075220e-04 + 8.484006947265324e-04 + 8.459876443379812e-04 + 8.435795407180884e-04 + 8.411763696276702e-04 + 8.387781169670006e-04 + 8.363847686436173e-04 + 8.339963104952486e-04 + 8.316127284703911e-04 + 8.292340085393665e-04 + 8.268601366269382e-04 + 8.244910987818749e-04 + 8.221268810991789e-04 + 8.197674696351961e-04 + 8.174128504795555e-04 + 8.150630097863373e-04 + 8.127179337987174e-04 + 8.103776086785577e-04 + 8.080420206116735e-04 + 8.057111559859151e-04 + 8.033850010442538e-04 + 8.010635420033148e-04 + 7.987467653572807e-04 + 7.964346575211490e-04 + 7.941272048473775e-04 + 7.918243938145987e-04 + 7.895262109139402e-04 + 7.872326426395918e-04 + 7.849436755404454e-04 + 7.826592961986925e-04 + 7.803794912312032e-04 + 7.781042473133076e-04 + 7.758335511109105e-04 + 7.735673892815593e-04 + 7.713057485643111e-04 + 7.690486157536814e-04 + 7.667959776727836e-04 + 7.645478211004869e-04 + 7.623041328912646e-04 + 7.600649000053854e-04 + 7.578301093264215e-04 + 7.555997477583347e-04 + 7.533738022910566e-04 + 7.511522599897612e-04 + 7.489351079282785e-04 + 7.467223331578882e-04 + 7.445139227962291e-04 + 7.423098639775247e-04 + 7.401101438263767e-04 + 7.379147496119391e-04 + 7.357236686066337e-04 + 7.335368879604144e-04 + 7.313543950177981e-04 + 7.291761771981462e-04 + 7.270022218267318e-04 + 7.248325162821272e-04 + 7.226670479949025e-04 + 7.205058044241288e-04 + 7.183487730730944e-04 + 7.161959414766152e-04 + 7.140472971809699e-04 + 7.119028277561113e-04 + 7.097625208231147e-04 + 7.076263641063239e-04 + 7.054943452576004e-04 + 7.033664519218169e-04 + 7.012426719956601e-04 + 6.991229932510109e-04 + 6.970074033540157e-04 + 6.948958902400331e-04 + 6.927884418689321e-04 + 6.906850461485944e-04 + 6.885856909689247e-04 + 6.864903643067344e-04 + 6.843990542306164e-04 + 6.823117487354689e-04 + 6.802284359113009e-04 + 6.781491039842488e-04 + 6.760737410664455e-04 + 6.740023352773578e-04 + 6.719348748282754e-04 + 6.698713480434448e-04 + 6.678117432465053e-04 + 6.657560487034911e-04 + 6.637042527655789e-04 + 6.616563438432147e-04 + 6.596123103782944e-04 + 6.575721408025738e-04 + 6.555358236048545e-04 + 6.535033473936704e-04 + 6.514747007198682e-04 + 6.494498721244171e-04 + 6.474288502266283e-04 + 6.454116237722280e-04 + 6.433981815196293e-04 + 6.413885120963034e-04 + 6.393826043225120e-04 + 6.373804471016686e-04 + 6.353820291728463e-04 + 6.333873394523912e-04 + 6.313963669651093e-04 + 6.294091005879305e-04 + 6.274255292885601e-04 + 6.254456421348147e-04 + 6.234694281911996e-04 + 6.214968766042923e-04 + 6.195279765738893e-04 + 6.175627172048796e-04 + 6.156010877068991e-04 + 6.136430774102346e-04 + 6.116886755862946e-04 + 6.097378715435904e-04 + 6.077906546697107e-04 + 6.058470144286753e-04 + 6.039069402835228e-04 + 6.019704216802507e-04 + 6.000374481409778e-04 + 5.981080092530985e-04 + 5.961820946588096e-04 + 5.942596940083503e-04 + 5.923407969721301e-04 + 5.904253932616448e-04 + 5.885134726736782e-04 + 5.866050250202793e-04 + 5.847000400980857e-04 + 5.827985078791194e-04 + 5.809004183470366e-04 + 5.790057613579966e-04 + 5.771145269606196e-04 + 5.752267052829598e-04 + 5.733422863841403e-04 + 5.714612604000550e-04 + 5.695836175305099e-04 + 5.677093480037018e-04 + 5.658384420678564e-04 + 5.639708900266142e-04 + 5.621066823022896e-04 + 5.602458092869138e-04 + 5.583882613744714e-04 + 5.565340290827777e-04 + 5.546831029664671e-04 + 5.528354736013446e-04 + 5.509911316015094e-04 + 5.491500676402182e-04 + 5.473122724461832e-04 + 5.454777367810608e-04 + 5.436464514412940e-04 + 5.418184072613321e-04 + 5.399935951226492e-04 + 5.381720060069473e-04 + 5.363536309736959e-04 + 5.345384609776805e-04 + 5.327264870731968e-04 + 5.309177004752346e-04 + 5.291120923593077e-04 + 5.273096539494221e-04 + 5.255103765544823e-04 + 5.237142514407170e-04 + 5.219212699733492e-04 + 5.201314236865307e-04 + 5.183447040146093e-04 + 5.165611024246747e-04 + 5.147806105331874e-04 + 5.130032200114061e-04 + 5.112289225530487e-04 + 5.094577098620987e-04 + 5.076895736854020e-04 + 5.059245058583464e-04 + 5.041624983517820e-04 + 5.024035430839573e-04 + 5.006476319990773e-04 + 4.988947572035529e-04 + 4.971449107942239e-04 + 4.953980848818115e-04 + 4.936542716805855e-04 + 4.919134634956966e-04 + 4.901756526846954e-04 + 4.884408316006080e-04 + 4.867089926403434e-04 + 4.849801282731475e-04 + 4.832542310892493e-04 + 4.815312937309208e-04 + 4.798113088560132e-04 + 4.780942690954599e-04 + 4.763801672259719e-04 + 4.746689961731605e-04 + 4.729607488395965e-04 + 4.712554181333483e-04 + 4.695529970142063e-04 + 4.678534786182833e-04 + 4.661568561056742e-04 + 4.644631226216267e-04 + 4.627722714285577e-04 + 4.610842958399921e-04 + 4.593991892076899e-04 + 4.577169450300335e-04 + 4.560375568336446e-04 + 4.543610181249096e-04 + 4.526873225594637e-04 + 4.510164638467393e-04 + 4.493484356950898e-04 + 4.476832319569833e-04 + 4.460208465382815e-04 + 4.443612733350173e-04 + 4.427045064285867e-04 + 4.410505399510805e-04 + 4.393993679504339e-04 + 4.377509846774214e-04 + 4.361053844834223e-04 + 4.344625616662297e-04 + 4.328225106365193e-04 + 4.311852259097922e-04 + 4.295507020701359e-04 + 4.279189337521271e-04 + 4.262899156467061e-04 + 4.246636425301464e-04 + 4.230401092565931e-04 + 4.214193107478168e-04 + 4.198012419839599e-04 + 4.181858979868060e-04 + 4.165732738692198e-04 + 4.149633649793705e-04 + 4.133561665739344e-04 + 4.117516738453971e-04 + 4.101498823326724e-04 + 4.085507876181731e-04 + 4.069543852399522e-04 + 4.053606707914893e-04 + 4.037696400086752e-04 + 4.021812887746215e-04 + 4.005956129436248e-04 + 3.990126084760660e-04 + 3.974322714846641e-04 + 3.958545980293214e-04 + 3.942795842513983e-04 + 3.927072264688036e-04 + 3.911375211195356e-04 + 3.895704646516968e-04 + 3.880060534869329e-04 + 3.864442842922655e-04 + 3.848851538066537e-04 + 3.833286587063809e-04 + 3.817747958651508e-04 + 3.802235622593628e-04 + 3.786749548688228e-04 + 3.771289708205180e-04 + 3.755856073392733e-04 + 3.740448616816441e-04 + 3.725067311649692e-04 + 3.709712132043737e-04 + 3.694383053790672e-04 + 3.679080053579325e-04 + 3.663803108503057e-04 + 3.648552195606523e-04 + 3.633327293646475e-04 + 3.618128382932607e-04 + 3.602955444344858e-04 + 3.587808459078391e-04 + 3.572687408998931e-04 + 3.557592277792442e-04 + 3.542523049927096e-04 + 3.527479710408145e-04 + 3.512462245019904e-04 + 3.497470640500036e-04 + 3.482504884770882e-04 + 3.467564967437580e-04 + 3.452650877985240e-04 + 3.437762605764859e-04 + 3.422900143250168e-04 + 3.408063483640538e-04 + 3.393252619896345e-04 + 3.378467546103840e-04 + 3.363708257653936e-04 + 3.348974751341996e-04 + 3.334267024660963e-04 + 3.319585075646510e-04 + 3.304928903038992e-04 + 3.290298507212495e-04 + 3.275693889604049e-04 + 3.261115052402690e-04 + 3.246561999078441e-04 + 3.232034733603981e-04 + 3.217533259984899e-04 + 3.203057585048498e-04 + 3.188607716794152e-04 + 3.174183662346357e-04 + 3.159785430822741e-04 + 3.145413032833238e-04 + 3.131066479369465e-04 + 3.116745782136557e-04 + 3.102450954058189e-04 + 3.088182010197684e-04 + 3.073938965650610e-04 + 3.059721835802734e-04 + 3.045530637916020e-04 + 3.031365390386899e-04 + 3.017226112552691e-04 + 3.003112824793533e-04 + 2.989025548234045e-04 + 2.974964304955006e-04 + 2.960929118913988e-04 + 2.946920014422941e-04 + 2.932937016150965e-04 + 2.918980151023185e-04 + 2.905049446767149e-04 + 2.891144931550728e-04 + 2.877266634923031e-04 + 2.863414587568181e-04 + 2.849588821248430e-04 + 2.835789368717610e-04 + 2.822016263526234e-04 + 2.808269540128114e-04 + 2.794549234525530e-04 + 2.780855383858511e-04 + 2.767188026160350e-04 + 2.753547199733251e-04 + 2.739932944362576e-04 + 2.726345302253411e-04 + 2.712784314751994e-04 + 2.699250024118205e-04 + 2.685742475595017e-04 + 2.672261714415124e-04 + 2.658807786371782e-04 + 2.645380739000219e-04 + 2.631980619957597e-04 + 2.618607478073471e-04 + 2.605261365239288e-04 + 2.591942332540738e-04 + 2.578650431168888e-04 + 2.565385715181067e-04 + 2.552148239188000e-04 + 2.538938058269625e-04 + 2.525755228940477e-04 + 2.512599808919393e-04 + 2.499471856899045e-04 + 2.486371432244064e-04 + 2.473298595107682e-04 + 2.460253406646765e-04 + 2.447235929605613e-04 + 2.434246227773826e-04 + 2.421284365723855e-04 + 2.408350408491505e-04 + 2.395444422583646e-04 + 2.382566475978551e-04 + 2.369716636126905e-04 + 2.356894972066169e-04 + 2.344101555067014e-04 + 2.331336455922197e-04 + 2.318599746393059e-04 + 2.305891500038487e-04 + 2.293211790903458e-04 + 2.280560693749351e-04 + 2.267938284406659e-04 + 2.255344639796373e-04 + 2.242779837481179e-04 + 2.230243955584571e-04 + 2.217737074088738e-04 + 2.205259273750273e-04 + 2.192810635194828e-04 + 2.180391240377155e-04 + 2.168001172473318e-04 + 2.155640515683406e-04 + 2.143309354202995e-04 + 2.131007772856537e-04 + 2.118735858306198e-04 + 2.106493698084787e-04 + 2.094281380089542e-04 + 2.082098992208740e-04 + 2.069946624015073e-04 + 2.057824366286796e-04 + 2.045732309577213e-04 + 2.033670545521807e-04 + 2.021639166762670e-04 + 2.009638266139778e-04 + 1.997667937615743e-04 + 1.985728276099416e-04 + 1.973819376249420e-04 + 1.961941334186818e-04 + 1.950094247288422e-04 + 1.938278211445808e-04 + 1.926493324216112e-04 + 1.914739685443153e-04 + 1.903017394244089e-04 + 1.891326549660665e-04 + 1.879667251315557e-04 + 1.868039600691931e-04 + 1.856443699312781e-04 + 1.844879648170397e-04 + 1.833347550402995e-04 + 1.821847509305592e-04 + 1.810379627269367e-04 + 1.798944008330572e-04 + 1.787540757117569e-04 + 1.776169978023503e-04 + 1.764831776083264e-04 + 1.753526257075039e-04 + 1.742253527510645e-04 + 1.731013693254543e-04 + 1.719806860311682e-04 + 1.708633135911149e-04 + 1.697492627671219e-04 + 1.686385443331759e-04 + 1.675311690584120e-04 + 1.664271477052953e-04 + 1.653264910845736e-04 + 1.642292101438652e-04 + 1.631353157524277e-04 + 1.620448187436031e-04 + 1.609577300754615e-04 + 1.598740606888992e-04 + 1.587938214812250e-04 + 1.577170233258566e-04 + 1.566436772168694e-04 + 1.555737942159133e-04 + 1.545073851968179e-04 + 1.534444610998845e-04 + 1.523850329588589e-04 + 1.513291117056277e-04 + 1.502767082718022e-04 + 1.492278336091406e-04 + 1.481824986061289e-04 + 1.471407142013266e-04 + 1.461024913960903e-04 + 1.450678410475480e-04 + 1.440367739871709e-04 + 1.430093010797408e-04 + 1.419854331666308e-04 + 1.409651810501559e-04 + 1.399485554884026e-04 + 1.389355672434459e-04 + 1.379262270311977e-04 + 1.369205454852166e-04 + 1.359185332657421e-04 + 1.349202010239166e-04 + 1.339255593445051e-04 + 1.329346186642912e-04 + 1.319473893926901e-04 + 1.309638820305585e-04 + 1.299841069283569e-04 + 1.290080743627276e-04 + 1.280357946577360e-04 + 1.270672779908064e-04 + 1.261025344295187e-04 + 1.251415740258344e-04 + 1.241844068414831e-04 + 1.232310428585276e-04 + 1.222814917974582e-04 + 1.213357634573326e-04 + 1.203938676644921e-04 + 1.194558139769228e-04 + 1.185216119071049e-04 + 1.175912709437735e-04 + 1.166648004395201e-04 + 1.157422096820540e-04 + 1.148235078941047e-04 + 1.139087041565015e-04 + 1.129978074560730e-04 + 1.120908266963163e-04 + 1.111877706801771e-04 + 1.102886481165339e-04 + 1.093934676008822e-04 + 1.085022375478793e-04 + 1.076149663238430e-04 + 1.067316622735248e-04 + 1.058523335072715e-04 + 1.049769880021785e-04 + 1.041056336506420e-04 + 1.032382782134568e-04 + 1.023749293560659e-04 + 1.015155946454073e-04 + 1.006602814254268e-04 + 9.980899691604087e-05 + 9.896174827822380e-05 + 9.811854249648769e-05 + 9.727938640362722e-05 + 9.644428669167931e-05 + 9.561324990949941e-05 + 9.478628247235907e-05 + 9.396339064407664e-05 + 9.314458052650697e-05 + 9.232985807344924e-05 + 9.151922908266137e-05 + 9.071269919463392e-05 + 8.991027389908417e-05 + 8.911195851702552e-05 + 8.831775819982511e-05 + 8.752767793866873e-05 + 8.674172256023528e-05 + 8.595989672073575e-05 + 8.518220490590348e-05 + 8.440865141920385e-05 + 8.363924039733809e-05 + 8.287397580722622e-05 + 8.211286142104977e-05 + 8.135590084051729e-05 + 8.060309749193613e-05 + 7.985445459658365e-05 + 7.910997520302950e-05 + 7.836966218298747e-05 + 7.763351820320998e-05 + 7.690154574559998e-05 + 7.617374709914362e-05 + 7.545012436145964e-05 + 7.473067944430294e-05 + 7.401541405207780e-05 + 7.330432968888297e-05 + 7.259742767376570e-05 + 7.189470911942744e-05 + 7.119617492698978e-05 + 7.050182580949816e-05 + 6.981166227552030e-05 + 6.912568461899398e-05 + 6.844389294104750e-05 + 6.776628713300132e-05 + 6.709286687130013e-05 + 6.642363163388969e-05 + 6.575858067871575e-05 + 6.509771305493562e-05 + 6.444102761503343e-05 + 6.378852298738557e-05 + 6.314019758583242e-05 + 6.249604962295056e-05 + 6.185607708979005e-05 + 6.122027775903875e-05 + 6.058864919749181e-05 + 5.996118875684596e-05 + 5.933789356597989e-05 + 5.871876053888907e-05 + 5.810378637807827e-05 + 5.749296757228591e-05 + 5.688630039241354e-05 + 5.628378087841946e-05 + 5.568540486517338e-05 + 5.509116797959273e-05 + 5.450106561933422e-05 + 5.391509296529475e-05 + 5.333324497665832e-05 + 5.275551640753726e-05 + 5.218190179953343e-05 + 5.161239545977182e-05 + 5.104699148146081e-05 + 5.048568375913861e-05 + 4.992846596763329e-05 + 4.937533154817934e-05 + 4.882627374147808e-05 + 4.828128557792042e-05 + 4.774035986717692e-05 + 4.720348921279551e-05 + 4.667066599018748e-05 + 4.614188237688400e-05 + 4.561713035194944e-05 + 4.509640165438987e-05 + 4.457968783032909e-05 + 4.406698023190481e-05 + 4.355826996891768e-05 + 4.305354796420506e-05 + 4.255280495560636e-05 + 4.205603144616347e-05 + 4.156321774032957e-05 + 4.107435395613752e-05 + 4.058942999665175e-05 + 4.010843556869507e-05 + 3.963136019085871e-05 + 3.915819316952545e-05 + 3.868892361719641e-05 + 3.822354046226270e-05 + 3.776203244031561e-05 + 3.730438809005640e-05 + 3.685059575689182e-05 + 3.640064360230198e-05 + 3.595451960373741e-05 + 3.551221155134442e-05 + 3.507370705450421e-05 + 3.463899354241650e-05 + 3.420805825588243e-05 + 3.378088825986720e-05 + 3.335747045272196e-05 + 3.293779155463552e-05 + 3.252183811026268e-05 + 3.210959649877579e-05 + 3.170105292073684e-05 + 3.129619341367949e-05 + 3.089500386055728e-05 + 3.049746997239612e-05 + 3.010357730585059e-05 + 2.971331126487755e-05 + 2.932665707619367e-05 + 2.894359982344077e-05 + 2.856412445132613e-05 + 2.818821573877814e-05 + 2.781585831987375e-05 + 2.744703669454755e-05 + 2.708173520948728e-05 + 2.671993807124836e-05 + 2.636162935767626e-05 + 2.600679300301263e-05 + 2.565541280257304e-05 + 2.530747242805460e-05 + 2.496295542635910e-05 + 2.462184521206604e-05 + 2.428412507785163e-05 + 2.394977819771233e-05 + 2.361878762181457e-05 + 2.329113628448998e-05 + 2.296680701293541e-05 + 2.264578251757474e-05 + 2.232804539557172e-05 + 2.201357814530585e-05 + 2.170236315904943e-05 + 2.139438272306276e-05 + 2.108961903561866e-05 + 2.078805419745760e-05 + 2.048967020582218e-05 + 2.019444897050880e-05 + 1.990237231614296e-05 + 1.961342197777572e-05 + 1.932757961462368e-05 + 1.904482680353188e-05 + 1.876514502948172e-05 + 1.848851571515060e-05 + 1.821492021640795e-05 + 1.794433980831370e-05 + 1.767675569461099e-05 + 1.741214902258527e-05 + 1.715050088166273e-05 + 1.689179228451716e-05 + 1.663600419203502e-05 + 1.638311752659296e-05 + 1.613311314360082e-05 + 1.588597184706167e-05 + 1.564167440067926e-05 + 1.540020151829502e-05 + 1.516153387453479e-05 + 1.492565211244357e-05 + 1.469253682958783e-05 + 1.446216858599219e-05 + 1.423452791421707e-05 + 1.400959531801727e-05 + 1.378735127470914e-05 + 1.356777623611474e-05 + 1.335085063337062e-05 + 1.313655487876277e-05 + 1.292486936494610e-05 + 1.271577446639333e-05 + 1.250925054517824e-05 + 1.230527796605363e-05 + 1.210383708165599e-05 + 1.190490823144681e-05 + 1.170847175649016e-05 + 1.151450799717159e-05 + 1.132299729345123e-05 + 1.113391999104486e-05 + 1.094725644586371e-05 + 1.076298702423590e-05 + 1.058109209097755e-05 + 1.040155202830912e-05 + 1.022434724511555e-05 + 1.004945815519835e-05 + 9.876865191434875e-06 + 9.706548817576976e-06 + 9.538489515829050e-06 + 9.372667791395497e-06 + 9.209064181113714e-06 + 9.047659251528417e-06 + 8.888433598428113e-06 + 8.731367852817854e-06 + 8.576442682615395e-06 + 8.423638791088146e-06 + 8.272936922653283e-06 + 8.124317863654203e-06 + 7.977762440936675e-06 + 7.833251528895281e-06 + 7.690766049713759e-06 + 7.550286971453687e-06 + 7.411795314787993e-06 + 7.275272152381047e-06 + 7.140698608042407e-06 + 7.008055865069554e-06 + 6.877325162999093e-06 + 6.748487797069182e-06 + 6.621525126924672e-06 + 6.496418573295794e-06 + 6.373149617974847e-06 + 6.251699810845691e-06 + 6.132050767139283e-06 + 6.014184168887862e-06 + 5.898081770597224e-06 + 5.783725394973654e-06 + 5.671096936050662e-06 + 5.560178365641361e-06 + 5.450951726966341e-06 + 5.343399138917566e-06 + 5.237502802602254e-06 + 5.133244993336976e-06 + 5.030608065950253e-06 + 4.929574461382145e-06 + 4.830126698086031e-06 + 4.732247378331277e-06 + 4.635919192753656e-06 + 4.541124912789582e-06 + 4.447847398958468e-06 + 4.356069602328335e-06 + 4.265774557056776e-06 + 4.176945389932598e-06 + 4.089565320281935e-06 + 4.003617653697645e-06 + 3.919085793399028e-06 + 3.835953236362346e-06 + 3.754203567594348e-06 + 3.673820472678765e-06 + 3.594787733829004e-06 + 3.517089224074456e-06 + 3.440708918621393e-06 + 3.365630891050476e-06 + 3.291839308866544e-06 + 3.219318444550688e-06 + 3.148052669614368e-06 + 3.078026451876596e-06 + 3.009224366838337e-06 + 2.941631088840265e-06 + 2.875231390541817e-06 + 2.810010156134118e-06 + 2.745952369256776e-06 + 2.683043113446972e-06 + 2.621267584374269e-06 + 2.560611078236376e-06 + 2.501058994098514e-06 + 2.442596843868032e-06 + 2.385210240033010e-06 + 2.328884899906223e-06 + 2.273606655048426e-06 + 2.219361438179688e-06 + 2.166135288678882e-06 + 2.113914360085187e-06 + 2.062684907096663e-06 + 2.012433292648363e-06 + 1.963145994708785e-06 + 1.914809593193956e-06 + 1.867410777738679e-06 + 1.820936351687259e-06 + 1.775373220609589e-06 + 1.730708401732302e-06 + 1.686929024947410e-06 + 1.644022323067757e-06 + 1.601975642837270e-06 + 1.560776441656341e-06 + 1.520412279298016e-06 + 1.480870830315665e-06 + 1.442139880698395e-06 + 1.404207319625165e-06 + 1.367061149777226e-06 + 1.330689483155573e-06 + 1.295080535963615e-06 + 1.260222640873472e-06 + 1.226104237141460e-06 + 1.192713867093549e-06 + 1.160040189500492e-06 + 1.128071969424555e-06 + 1.096798076008267e-06 + 1.066207493417804e-06 + 1.036289310659725e-06 + 1.007032721493895e-06 + 9.784270338592895e-07 + 9.504616588874799e-07 + 9.231261128777845e-07 + 8.964100256766087e-07 + 8.703031286825232e-07 + 8.447952585150839e-07 + 8.198763641874936e-07 + 7.955364948286083e-07 + 7.717658049346632e-07 + 7.485545598914059e-07 + 7.258931238786546e-07 + 7.037719665410901e-07 + 6.821816664988581e-07 + 6.611128998791421e-07 + 6.405564483055948e-07 + 6.205032003278225e-07 + 6.009441408121614e-07 + 5.818703600152187e-07 + 5.632730528626383e-07 + 5.451435094097432e-07 + 5.274731247677268e-07 + 5.102533965000488e-07 + 4.934759162480692e-07 + 4.771323800991179e-07 + 4.612145841662162e-07 + 4.457144176881476e-07 + 4.306238737382497e-07 + 4.159350428538716e-07 + 4.016401077051372e-07 + 3.877313539402903e-07 + 3.742011622647174e-07 + 3.610420046499709e-07 + 3.482464547968944e-07 + 3.358071790150475e-07 + 3.237169340794661e-07 + 3.119685770955318e-07 + 3.005550554375413e-07 + 2.894694062090586e-07 + 2.787047650882795e-07 + 2.682543557659438e-07 + 2.581114909968296e-07 + 2.482695801165890e-07 + 2.387221182727572e-07 + 2.294626890008751e-07 + 2.204849703818711e-07 + 2.117827241548630e-07 + 2.033497996067267e-07 + 1.951801384624956e-07 + 1.872677641420447e-07 + 1.796067867093452e-07 + 1.721914062527642e-07 + 1.650159027896018e-07 + 1.580746421942365e-07 + 1.513620776690125e-07 + 1.448727406655565e-07 + 1.386012477684331e-07 + 1.325423000867771e-07 + 1.266906752868051e-07 + 1.210412352836648e-07 + 1.155889239131470e-07 + 1.103287600176973e-07 + 1.052558455547364e-07 + 1.003653617115032e-07 + 9.565256321109978e-08 + 9.111278671292873e-08 + 8.674144556415159e-08 + 8.253402527034383e-08 + 7.848609177536523e-08 + 7.459328520671134e-08 + 7.085131661573615e-08 + 6.725597593242166e-08 + 6.380312478868793e-08 + 6.048869454868003e-08 + 5.730869376312298e-08 + 5.425920030057914e-08 + 5.133636064443935e-08 + 4.853639660958214e-08 + 4.585559704969457e-08 + 4.329031836991616e-08 + 4.083699031045398e-08 + 3.849210751108346e-08 + 3.625223113812888e-08 + 3.411399355038575e-08 + 3.207408998824199e-08 + 3.012928121511996e-08 + 2.827639696078167e-08 + 2.651232793166280e-08 + 2.483402934338396e-08 + 2.323852311474514e-08 + 2.172289034680298e-08 + 2.028427559044146e-08 + 1.891988778877689e-08 + 1.762699339728036e-08 + 1.640292125402531e-08 + 1.524506224991720e-08 + 1.415086320949184e-08 + 1.311783222446413e-08 + 1.214353712508316e-08 + 1.122560019257377e-08 + 1.036170378687137e-08 + 9.549587722603005e-09 + 8.787044877843281e-09 + 8.071926961605364e-09 + 7.402140903161946e-09 + 6.775645388472587e-09 + 6.190456604129221e-09 + 5.644643792323680e-09 + 5.136326726146161e-09 + 4.663681262833891e-09 + 4.224934232596794e-09 + 3.818361845837701e-09 + 3.442294891302550e-09 + 3.095113141246544e-09 + 2.775244665814773e-09 + 2.481170514027403e-09 + 2.211418834557044e-09 + 1.964565051498559e-09 + 1.739235878927056e-09 + 1.534103359980556e-09 + 1.347885831336700e-09 + 1.179351163642234e-09 + 1.027310894648576e-09 + 8.906218977827446e-10 + 7.681887799049897e-10 + 6.589582548472836e-10 + 5.619214238374888e-10 + 4.761152985830568e-10 + 4.006175343477256e-10 + 3.345492227248809e-10 + 2.770755362128679e-10 + 2.274009179016229e-10 + 1.847722840116674e-10 + 1.484788129161787e-10 + 1.178476648704315e-10 + 9.224748788184573e-11 + 7.108739902161543e-11 + 5.381328706422649e-11 + 3.991151299277479e-11 + 2.890715266694438e-11 + 2.036091648096713e-11 + 1.387293445657948e-11 + 9.080348320515963e-12 + 5.654863833225597e-12 + 3.306507674167889e-12 + 1.780689492983021e-12 + 8.563646514161430e-13 + 3.496037276049943e-13 + 1.104943293034196e-13 + 2.161027650041889e-14 + 5.028049336448720e-16 + 0.000000000000000e+00 + 0.000000000000000e+00 + 7.485272593961791e+04 + 1.493519318697679e+05 + 2.234970831294996e+05 + 2.972876533345640e+05 + 3.707231243774131e+05 + 4.438029864272003e+05 + 5.165267379297798e+05 + 5.888938856077072e+05 + 6.609039444602393e+05 + 7.325564377633340e+05 + 8.038508970696502e+05 + 8.747868622085482e+05 + 9.453638812860892e+05 + 1.015581510685036e+06 + 1.085439315064852e+06 + 1.154936867361703e+06 + 1.224073748788454e+06 + 1.292849548834672e+06 + 1.361263865266626e+06 + 1.429316304127287e+06 + 1.497006479736322e+06 + 1.564334014690106e+06 + 1.631298539861710e+06 + 1.697899694400910e+06 + 1.764137125734180e+06 + 1.830010489564698e+06 + 1.895519449872339e+06 + 1.960663678913685e+06 + 2.025442857222013e+06 + 2.089856673607306e+06 + 2.153904825156247e+06 + 2.217587017232217e+06 + 2.280902963475303e+06 + 2.343852385802290e+06 + 2.406435014406665e+06 + 2.468650587758616e+06 + 2.530498852605031e+06 + 2.591979563969503e+06 + 2.653092485152322e+06 + 2.713837387730482e+06 + 2.774214051557675e+06 + 2.834222264764299e+06 + 2.893861823757447e+06 + 2.953132533220919e+06 + 3.012034206115212e+06 + 3.070566663677527e+06 + 3.128729735421764e+06 + 3.186523259138526e+06 + 3.243947080895115e+06 + 3.301001055035538e+06 + 3.357685044180497e+06 + 3.413998919227402e+06 + 3.469942559350358e+06 + 3.525515852000176e+06 + 3.580718692904367e+06 + 3.635550986067140e+06 + 3.690012643769410e+06 + 3.744103586568788e+06 + 3.797823743299592e+06 + 3.851173051072835e+06 + 3.904151455276238e+06 + 3.956758909574216e+06 + 4.008995375907890e+06 + 4.060860824495079e+06 + 4.112355233830309e+06 + 4.163478590684799e+06 + 4.214230890106475e+06 + 4.264612135419963e+06 + 4.314622338226588e+06 + 4.364261518404378e+06 + 4.413529704108060e+06 + 4.462426931769069e+06 + 4.510953246095533e+06 + 4.559108700072286e+06 + 4.606893354960858e+06 + 4.654307280299488e+06 + 4.701350553903108e+06 + 4.748023261863358e+06 + 4.794325498548575e+06 + 4.840257366603798e+06 + 4.885818976950768e+06 + 4.931010448787929e+06 + 4.975831909590418e+06 + 5.020283495110085e+06 + 5.064365349375471e+06 + 5.108077624691823e+06 + 5.151420481641092e+06 + 5.194394089081923e+06 + 5.236998624149668e+06 + 5.279234272256376e+06 + 5.321101227090800e+06 + 5.362599690618395e+06 + 5.403729873081312e+06 + 5.444491992998408e+06 + 5.484886277165242e+06 + 5.524912960654069e+06 + 5.564572286813851e+06 + 5.603864507270245e+06 + 5.642789881925615e+06 + 5.681348678959022e+06 + 5.719541174826232e+06 + 5.757367654259708e+06 + 5.794828410268617e+06 + 5.831923744138825e+06 + 5.868653965432902e+06 + 5.905019391990117e+06 + 5.941020349926441e+06 + 5.976657173634544e+06 + 6.011930205783804e+06 + 6.046839797320290e+06 + 6.081386307466779e+06 + 6.115570103722748e+06 + 6.149391561864376e+06 + 6.182851065944540e+06 + 6.215949008292822e+06 + 6.248685789515501e+06 + 6.281061818495561e+06 + 6.313077512392686e+06 + 6.344733296643257e+06 + 6.376029604960365e+06 + 6.406966879333793e+06 + 6.437545570030033e+06 + 6.467766135592271e+06 + 6.497629042840399e+06 + 6.527134766871008e+06 + 6.556283791057392e+06 + 6.585076607049545e+06 + 6.613513714774162e+06 + 6.641595622434638e+06 + 6.669322846511072e+06 + 6.696695911760264e+06 + 6.723715351215711e+06 + 6.750381706187615e+06 + 6.776695526262878e+06 + 6.802657369305105e+06 + 6.828267801454599e+06 + 6.853527397128366e+06 + 6.878436739020114e+06 + 6.902996418100249e+06 + 6.927207033615881e+06 + 6.951069193090819e+06 + 6.974583512325577e+06 + 6.997750615397365e+06 + 7.020571134660101e+06 + 7.043045710744397e+06 + 7.065174992557568e+06 + 7.086959637283633e+06 + 7.108400310383311e+06 + 7.129497685594021e+06 + 7.150252444929884e+06 + 7.170665278681723e+06 + 7.190736885417059e+06 + 7.210467971980117e+06 + 7.229859253491821e+06 + 7.248911453349800e+06 + 7.267625303228384e+06 + 7.286001543078595e+06 + 7.304040921128170e+06 + 7.321744193881537e+06 + 7.339112126119829e+06 + 7.356145490900878e+06 + 7.372845069559221e+06 + 7.389211651706094e+06 + 7.405246035229432e+06 + 7.420949026293878e+06 + 7.436321439340766e+06 + 7.451364097088138e+06 + 7.466077830530737e+06 + 7.480463478940005e+06 + 7.494521889864086e+06 + 7.508253919127826e+06 + 7.521660430832772e+06 + 7.534742297357169e+06 + 7.547500399355968e+06 + 7.559935625760819e+06 + 7.572048873780073e+06 + 7.583841048898782e+06 + 7.595313064878696e+06 + 7.606465843758276e+06 + 7.617300315852673e+06 + 7.627817419753745e+06 + 7.638018102330050e+06 + 7.647903318726848e+06 + 7.657474032366097e+06 + 7.666731214946462e+06 + 7.675675846443305e+06 + 7.684308915108687e+06 + 7.692631417471377e+06 + 7.700644358336839e+06 + 7.708348750787240e+06 + 7.715745616181449e+06 + 7.722835984155037e+06 + 7.729620892620270e+06 + 7.736101387766127e+06 + 7.742278524058277e+06 + 7.748153364239094e+06 + 7.753726979327655e+06 + 7.759000448619737e+06 + 7.763974859687817e+06 + 7.768651308381077e+06 + 7.773030898825390e+06 + 7.777114743423343e+06 + 7.780903962854218e+06 + 7.784399686073998e+06 + 7.787603050315368e+06 + 7.790515201087713e+06 + 7.793137292177119e+06 + 7.795470485646380e+06 + 7.797515951834979e+06 + 7.799274869359110e+06 + 7.800748425111664e+06 + 7.801937814262233e+06 + 7.802844240257111e+06 + 7.803468914819297e+06 + 7.803813057948484e+06 + 7.803877897921070e+06 + 7.803664671290156e+06 + 7.803174622885538e+06 + 7.802409005813722e+06 + 7.801369081457905e+06 + 7.800056119477995e+06 + 7.798471397810592e+06 + 7.796616202669007e+06 + 7.794491828543244e+06 + 7.792099578200010e+06 + 7.789440762682715e+06 + 7.786516701311473e+06 + 7.783328721683091e+06 + 7.779878159671083e+06 + 7.776166359425665e+06 + 7.772194673373749e+06 + 7.767964462218953e+06 + 7.763477094941595e+06 + 7.758733948798692e+06 + 7.753736409323964e+06 + 7.748485870327833e+06 + 7.742983733897421e+06 + 7.737231410396548e+06 + 7.731230318465743e+06 + 7.724981885022229e+06 + 7.718487545259933e+06 + 7.711748742649483e+06 + 7.704766928938209e+06 + 7.697543564150140e+06 + 7.690080116586007e+06 + 7.682378062823243e+06 + 7.674438887715982e+06 + 7.666264084395060e+06 + 7.657855154268012e+06 + 7.649213607019073e+06 + 7.640340960609185e+06 + 7.631238741275986e+06 + 7.621908483533815e+06 + 7.612351730173716e+06 + 7.602570032263434e+06 + 7.592564949147409e+06 + 7.582338048446788e+06 + 7.571890906059416e+06 + 7.561225106159844e+06 + 7.550342241199316e+06 + 7.539243911905785e+06 + 7.527931727283904e+06 + 7.516407304615024e+06 + 7.504672269457198e+06 + 7.492728241314434e+06 + 7.480576638153253e+06 + 7.468218723741564e+06 + 7.455655760425863e+06 + 7.442889013018662e+06 + 7.429919748798485e+06 + 7.416749237509867e+06 + 7.403378751363355e+06 + 7.389809565035504e+06 + 7.376042955668893e+06 + 7.362080202872100e+06 + 7.347922588719724e+06 + 7.333571397752370e+06 + 7.319027916976659e+06 + 7.304293435865226e+06 + 7.289369246356713e+06 + 7.274256642855777e+06 + 7.258956922233087e+06 + 7.243471383825325e+06 + 7.227801329435183e+06 + 7.211948063331366e+06 + 7.195912892248592e+06 + 7.179697125387593e+06 + 7.163302074415106e+06 + 7.146729053463890e+06 + 7.129979379132709e+06 + 7.113054370486340e+06 + 7.095955349055575e+06 + 7.078683638837219e+06 + 7.061240566294081e+06 + 7.043627460354994e+06 + 7.025845652414794e+06 + 7.007896476334333e+06 + 6.989781268440476e+06 + 6.971501367526096e+06 + 6.953058114850082e+06 + 6.934452854137335e+06 + 6.915686931578767e+06 + 6.896761695831302e+06 + 6.877678498017876e+06 + 6.858438691727439e+06 + 6.839043633014950e+06 + 6.819494680401385e+06 + 6.799793194873727e+06 + 6.779940539884974e+06 + 6.759938081354136e+06 + 6.739787187666233e+06 + 6.719489229672301e+06 + 6.699045580689385e+06 + 6.678457616500544e+06 + 6.657726715354848e+06 + 6.636854257967380e+06 + 6.615841627519235e+06 + 6.594690209657516e+06 + 6.573401392495350e+06 + 6.551976566611862e+06 + 6.530417125052197e+06 + 6.508724463327511e+06 + 6.486899979414972e+06 + 6.464945073757762e+06 + 6.442861149265066e+06 + 6.420649611312096e+06 + 6.398311867740063e+06 + 6.375849328856199e+06 + 6.353263407433745e+06 + 6.330555518711952e+06 + 6.307727080396090e+06 + 6.284779512657427e+06 + 6.261714238133261e+06 + 6.238532681926889e+06 + 6.215236271607627e+06 + 6.191826437210800e+06 + 6.168304611237747e+06 + 6.144672228655818e+06 + 6.120930726898375e+06 + 6.097081545864792e+06 + 6.073126127920459e+06 + 6.049065917896771e+06 + 6.024902363091142e+06 + 6.000636913266994e+06 + 5.976271020653763e+06 + 5.951806139946896e+06 + 5.927243728307854e+06 + 5.902585245364108e+06 + 5.877832153209144e+06 + 5.852985916402456e+06 + 5.828048001969552e+06 + 5.803019879401957e+06 + 5.777903020657199e+06 + 5.752698900158828e+06 + 5.727408994796395e+06 + 5.702034783925475e+06 + 5.676577749367647e+06 + 5.651039375410506e+06 + 5.625421148807654e+06 + 5.599724558778713e+06 + 5.573951097009313e+06 + 5.548102257651095e+06 + 5.522179537321717e+06 + 5.496184435104840e+06 + 5.470118452550147e+06 + 5.443983093673326e+06 + 5.417779864956085e+06 + 5.391510275346133e+06 + 5.365175836257203e+06 + 5.338778061569033e+06 + 5.312318467627374e+06 + 5.285798573243992e+06 + 5.259219899696662e+06 + 5.232583970729172e+06 + 5.205892312551324e+06 + 5.179146453838928e+06 + 5.152347925733812e+06 + 5.125498261843814e+06 + 5.098598998242779e+06 + 5.071651673470572e+06 + 5.044657828533064e+06 + 5.017619006902143e+06 + 4.990536754515706e+06 + 4.963412619777663e+06 + 4.936248153557934e+06 + 4.909044909192459e+06 + 4.881804442483181e+06 + 4.854528311698059e+06 + 4.827218077571064e+06 + 4.799875303302180e+06 + 4.772501554557402e+06 + 4.745098399468737e+06 + 4.717667408634203e+06 + 4.690210155117837e+06 + 4.662728214449679e+06 + 4.635223164625784e+06 + 4.607696586108224e+06 + 4.580150061825077e+06 + 4.552585177170437e+06 + 4.525003520004408e+06 + 4.497406680653105e+06 + 4.469796251908661e+06 + 4.442173829029216e+06 + 4.414541009738924e+06 + 4.386899394227949e+06 + 4.359250585152469e+06 + 4.331596187634676e+06 + 4.303937809262771e+06 + 4.276277060090966e+06 + 4.248615552639492e+06 + 4.220954901894585e+06 + 4.193296725308497e+06 + 4.165642642799490e+06 + 4.137994276751838e+06 + 4.110353252015833e+06 + 4.082721195907769e+06 + 4.055099738209962e+06 + 4.027490511170732e+06 + 3.999895149504419e+06 + 3.972315290391369e+06 + 3.944752573477943e+06 + 3.917208640876513e+06 + 3.889685137165464e+06 + 3.862183709389194e+06 + 3.834706007058111e+06 + 3.807253682148634e+06 + 3.779828389103201e+06 + 3.752431784830255e+06 + 3.725065528704253e+06 + 3.697731282565667e+06 + 3.670430710720977e+06 + 3.643165479942679e+06 + 3.615937259469279e+06 + 3.588747721005293e+06 + 3.561598538721256e+06 + 3.534491389253709e+06 + 3.507427951705208e+06 + 3.480409907644318e+06 + 3.453438941105620e+06 + 3.426516738589706e+06 + 3.399644989063177e+06 + 3.372825383958653e+06 + 3.346070405300663e+06 + 3.319465764034668e+06 + 3.293075510459727e+06 + 3.266878224028187e+06 + 3.240869303162930e+06 + 3.215052515525467e+06 + 3.189425288821125e+06 + 3.163986116962672e+06 + 3.138734196986282e+06 + 3.113668261011591e+06 + 3.088787111422079e+06 + 3.064089610843659e+06 + 3.039574592322599e+06 + 3.015240898358055e+06 + 2.991087379108502e+06 + 2.967112887579310e+06 + 2.943316282301805e+06 + 2.919696425519270e+06 + 2.896252185185970e+06 + 2.872982434032476e+06 + 2.849886048305633e+06 + 2.826961910405915e+06 + 2.804208907130155e+06 + 2.781625929123051e+06 + 2.759211873644148e+06 + 2.736965641930875e+06 + 2.714886139213983e+06 + 2.692972277509980e+06 + 2.671222972611625e+06 + 2.649637144535781e+06 + 2.628213719738566e+06 + 2.606951628696007e+06 + 2.585849806893141e+06 + 2.564907195391811e+06 + 2.544122739173453e+06 + 2.523495389008342e+06 + 2.503024101022326e+06 + 2.482707834844927e+06 + 2.462545556054412e+06 + 2.442536236120740e+06 + 2.422678849863131e+06 + 2.402972377820649e+06 + 2.383415806334849e+06 + 2.364008125120872e+06 + 2.344748329835404e+06 + 2.325635421794777e+06 + 2.306668405813749e+06 + 2.287846292982958e+06 + 2.269168099757709e+06 + 2.250632846050586e+06 + 2.232239558087812e+06 + 2.213987266957964e+06 + 2.195875007435864e+06 + 2.177901821473463e+06 + 2.160066755143500e+06 + 2.142368857948063e+06 + 2.124807187017495e+06 + 2.107380803567688e+06 + 2.090088772295509e+06 + 2.072930165358991e+06 + 2.055904058912681e+06 + 2.039009532618940e+06 + 2.022245673041180e+06 + 2.005611571515091e+06 + 1.989106323649552e+06 + 1.972729030298793e+06 + 1.956478797095825e+06 + 1.940354734917566e+06 + 1.924355960016194e+06 + 1.908481592593613e+06 + 1.892730757829583e+06 + 1.877102587255698e+06 + 1.861596215902473e+06 + 1.846210783186639e+06 + 1.830945435012332e+06 + 1.815799321323061e+06 + 1.800771597138300e+06 + 1.785861422681660e+06 + 1.771067961851037e+06 + 1.756390384114008e+06 + 1.741827864373811e+06 + 1.727379581118251e+06 + 1.713044718232857e+06 + 1.698822464839031e+06 + 1.684712013805151e+06 + 1.670712563808684e+06 + 1.656823318006428e+06 + 1.643043483055245e+06 + 1.629372271790869e+06 + 1.615808901794204e+06 + 1.602352594044968e+06 + 1.589002574975609e+06 + 1.575758076322615e+06 + 1.562618333634041e+06 + 1.549582587186907e+06 + 1.536650081999105e+06 + 1.523820067124634e+06 + 1.511091797064626e+06 + 1.498464530780005e+06 + 1.485937531189585e+06 + 1.473510066471004e+06 + 1.461181408848471e+06 + 1.448950834625962e+06 + 1.436817625926470e+06 + 1.424781068397768e+06 + 1.412840451407005e+06 + 1.400995070309470e+06 + 1.389244224245752e+06 + 1.377587216393601e+06 + 1.366023355231714e+06 + 1.354551952598923e+06 + 1.343172324427429e+06 + 1.331883792566280e+06 + 1.320685682335073e+06 + 1.309577322956444e+06 + 1.298558048406515e+06 + 1.287627196362487e+06 + 1.276784109496439e+06 + 1.266028134784103e+06 + 1.255358622320226e+06 + 1.244774926866806e+06 + 1.234276408467881e+06 + 1.223862430240984e+06 + 1.213532358913532e+06 + 1.203285566621101e+06 + 1.193121428991948e+06 + 1.183039325468658e+06 + 1.173038640304339e+06 + 1.163118761212139e+06 + 1.153279079987504e+06 + 1.143518992564451e+06 + 1.133837898317260e+06 + 1.124235201394960e+06 + 1.114710310121780e+06 + 1.105262636013577e+06 + 1.095891594215875e+06 + 1.086596604396163e+06 + 1.077377090233802e+06 + 1.068232479242110e+06 + 1.059162202266971e+06 + 1.050165693700918e+06 + 1.041242393088349e+06 + 1.032391743039430e+06 + 1.023613189178869e+06 + 1.014906181731556e+06 + 1.006270174355871e+06 + 9.977046244036794e+05 + 9.892089935677024e+05 + 9.807827465832273e+05 + 9.724253516436865e+05 + 9.641362810046984e+05 + 9.559150102252770e+05 + 9.477610188719215e+05 + 9.396737899139688e+05 + 9.316528091743080e+05 + 9.236975665277143e+05 + 9.158075561735997e+05 + 9.079822751134991e+05 + 9.002212236533747e+05 + 8.925239057014534e+05 + 8.848898281604102e+05 + 8.773185021612709e+05 + 8.698094424648809e+05 + 8.623621663528095e+05 + 8.549761944053203e+05 + 8.476510510446843e+05 + 8.403862637513808e+05 + 8.331813638606276e+05 + 8.260358860837971e+05 + 8.189493675787508e+05 + 8.119213484350282e+05 + 8.049513730504208e+05 + 7.980389891667470e+05 + 7.911837471192866e+05 + 7.843852003599521e+05 + 7.776429053419405e+05 + 7.709564221446526e+05 + 7.643253140173181e+05 + 7.577491471803443e+05 + 7.512274911598503e+05 + 7.447599180231568e+05 + 7.383460026539576e+05 + 7.319853241781880e+05 + 7.256774641850799e+05 + 7.194220067146936e+05 + 7.132185395309449e+05 + 7.070666529520636e+05 + 7.009659400728574e+05 + 6.949159975085063e+05 + 6.889164244086053e+05 + 6.829668228095188e+05 + 6.770667979456050e+05 + 6.712159574726904e+05 + 6.654139120281750e+05 + 6.596602754201142e+05 + 6.539546638052445e+05 + 6.482966962660485e+05 + 6.426859949531512e+05 + 6.371221843431445e+05 + 6.316048918617199e+05 + 6.261337478197705e+05 + 6.207083848228143e+05 + 6.153284385055641e+05 + 6.099935472682050e+05 + 6.047033517354903e+05 + 5.994574954765684e+05 + 5.942556247644721e+05 + 5.890973881315375e+05 + 5.839824370501742e+05 + 5.789104255366600e+05 + 5.738810098259439e+05 + 5.688938490610282e+05 + 5.639486048510919e+05 + 5.590449410545636e+05 + 5.541825244030216e+05 + 5.493610239467514e+05 + 5.445801109611018e+05 + 5.398394595653637e+05 + 5.351387461239359e+05 + 5.304776492529869e+05 + 5.258558503354813e+05 + 5.212730328842620e+05 + 5.167288826663374e+05 + 5.122230881980868e+05 + 5.077553400369428e+05 + 5.033253309731948e+05 + 4.989327564601310e+05 + 4.945773139811803e+05 + 4.902587033143605e+05 + 4.859766266490913e+05 + 4.817307881770549e+05 + 4.775208945372272e+05 + 4.733466546121029e+05 + 4.692077791818720e+05 + 4.651039814988278e+05 + 4.610349770333872e+05 + 4.570004830895544e+05 + 4.530002193970305e+05 + 4.490339078641191e+05 + 4.451012721716975e+05 + 4.412020383482487e+05 + 4.373359346346440e+05 + 4.335026910461138e+05 + 4.297020398291695e+05 + 4.259337153289251e+05 + 4.221974536699117e+05 + 4.184929932504728e+05 + 4.148200744355357e+05 + 4.111784393468680e+05 + 4.075678324287459e+05 + 4.039879999559653e+05 + 4.004386899065928e+05 + 3.969196525836200e+05 + 3.934306400492930e+05 + 3.899714060581240e+05 + 3.865417066557175e+05 + 3.831412995849459e+05 + 3.797699442729113e+05 + 3.764274023566474e+05 + 3.731134371233769e+05 + 3.698278135622827e+05 + 3.665702987827645e+05 + 3.633406615280775e+05 + 3.601386722825769e+05 + 3.569641034535824e+05 + 3.538167290782596e+05 + 3.506963250581825e+05 + 3.476026690576235e+05 + 3.445355402744773e+05 + 3.414947198014101e+05 + 3.384799905288533e+05 + 3.354911367821696e+05 + 3.325279446663284e+05 + 3.295902021531574e+05 + 3.266776986085196e+05 + 3.237902250531212e+05 + 3.209275743946921e+05 + 3.180895409646722e+05 + 3.152759207164211e+05 + 3.124865112943467e+05 + 3.097211117598721e+05 + 3.069795229448014e+05 + 3.042615472339510e+05 + 3.015669883675390e+05 + 2.988956518738279e+05 + 2.962473447339852e+05 + 2.936218752484120e+05 + 2.910190535228683e+05 + 2.884386911235514e+05 + 2.858806009067059e+05 + 2.833445973150213e+05 + 2.808304963224871e+05 + 2.783381153152796e+05 + 2.758672731341289e+05 + 2.734177900017136e+05 + 2.709894875428715e+05 + 2.685821889451269e+05 + 2.661957187314447e+05 + 2.638299027837526e+05 + 2.614845683836235e+05 + 2.591595441417535e+05 + 2.568546601323142e+05 + 2.545697477628685e+05 + 2.523046396706559e+05 + 2.500591699434964e+05 + 2.478331740644226e+05 + 2.456264886409644e+05 + 2.434389516128850e+05 + 2.412704023657298e+05 + 2.391206814330098e+05 + 2.369896306904384e+05 + 2.348770933197599e+05 + 2.327829136173563e+05 + 2.307069372775990e+05 + 2.286490111745211e+05 + 2.266089832623643e+05 + 2.245867029748616e+05 + 2.225820209237310e+05 + 2.205947887183862e+05 + 2.186248592921233e+05 + 2.166720868039439e+05 + 2.147363264605261e+05 + 2.128174347706827e+05 + 2.109152693889452e+05 + 2.090296889843596e+05 + 2.071605535077064e+05 + 2.053077240108720e+05 + 2.034710625599661e+05 + 2.016504325007521e+05 + 1.998456982030229e+05 + 1.980567250370223e+05 + 1.962833797138016e+05 + 1.945255298870997e+05 + 1.927830441323482e+05 + 1.910557922964590e+05 + 1.893436452442803e+05 + 1.876464748512476e+05 + 1.859641540986015e+05 + 1.842965568967865e+05 + 1.826435581738182e+05 + 1.810050340354470e+05 + 1.793808614993985e+05 + 1.777709185575675e+05 + 1.761750842223128e+05 + 1.745932384546507e+05 + 1.730252623268235e+05 + 1.714710377603550e+05 + 1.699304475698536e+05 + 1.684033757560533e+05 + 1.668897071565261e+05 + 1.653893274220018e+05 + 1.639021233234161e+05 + 1.624279825159149e+05 + 1.609667934564489e+05 + 1.595184456811041e+05 + 1.580828296015031e+05 + 1.566598363968120e+05 + 1.552493582649543e+05 + 1.538512882702573e+05 + 1.524655202573447e+05 + 1.510919491064028e+05 + 1.497304705096718e+05 + 1.483809809029638e+05 + 1.470433777326744e+05 + 1.457175592117312e+05 + 1.444034242942686e+05 + 1.431008729933990e+05 + 1.418098060421684e+05 + 1.405301248671864e+05 + 1.392617319094920e+05 + 1.380045303259556e+05 + 1.367584239714278e+05 + 1.355233176631985e+05 + 1.342991169575939e+05 + 1.330857281367400e+05 + 1.318830583220329e+05 + 1.306910153813551e+05 + 1.295095079605118e+05 + 1.283384454277201e+05 + 1.271777378619642e+05 + 1.260272961737778e+05 + 1.248870320778846e+05 + 1.237568578844075e+05 + 1.226366865814357e+05 + 1.215264320654815e+05 + 1.204260088572700e+05 + 1.193353321256488e+05 + 1.182543179024696e+05 + 1.171828828329888e+05 + 1.161209441971572e+05 + 1.150684200368906e+05 + 1.140252290172446e+05 + 1.129912905290012e+05 + 1.119665246989244e+05 + 1.109508522377596e+05 + 1.099441945145749e+05 + 1.089464735474612e+05 + 1.079576119759672e+05 + 1.069775332658437e+05 + 1.060061613940063e+05 + 1.050434208615313e+05 + 1.040892370245828e+05 + 1.031435357647047e+05 + 1.022062434696232e+05 + 1.012772873208916e+05 + 1.003565950684523e+05 + 9.944409498683986e+04 + 9.853971604124359e+04 + 9.764338777276211e+04 + 9.675504027159151e+04 + 9.587460425195689e+04 + 9.500201099602232e+04 + 9.413719236230559e+04 + 9.328008081668093e+04 + 9.243060935253548e+04 + 9.158871152497102e+04 + 9.075432150057999e+04 + 8.992737394814150e+04 + 8.910780408381460e+04 + 8.829554772812854e+04 + 8.749054118346877e+04 + 8.669272128864085e+04 + 8.590202548133289e+04 + 8.511839166524833e+04 + 8.434175826486869e+04 + 8.357206428147563e+04 + 8.280924917713979e+04 + 8.205325293940101e+04 + 8.130401609766987e+04 + 8.056147962909786e+04 + 7.982558503959214e+04 + 7.909627435336930e+04 + 7.837349003289423e+04 + 7.765717506702039e+04 + 7.694727293156023e+04 + 7.624372753229675e+04 + 7.554648331079309e+04 + 7.485548516457589e+04 + 7.417067840840557e+04 + 7.349200889763620e+04 + 7.281942291879337e+04 + 7.215286716185752e+04 + 7.149228885123666e+04 + 7.083763563584826e+04 + 7.018885556144921e+04 + 6.954589717270633e+04 + 6.890870943981725e+04 + 6.827724174191867e+04 + 6.765144392787226e+04 + 6.703126624692857e+04 + 6.641665935636370e+04 + 6.580757439267423e+04 + 6.520396286738058e+04 + 6.460577668030654e+04 + 6.401296820028803e+04 + 6.342549017134596e+04 + 6.284329573351656e+04 + 6.226633845994343e+04 + 6.169457228450359e+04 + 6.112795154311772e+04 + 6.056643099463883e+04 + 6.000996574136701e+04 + 5.945851127624574e+04 + 5.891202351267684e+04 + 5.837045869755570e+04 + 5.783377345558033e+04 + 5.730192482224794e+04 + 5.677487016169035e+04 + 5.625256720967715e+04 + 5.573497408983271e+04 + 5.522204924870504e+04 + 5.471375150828325e+04 + 5.421004006036229e+04 + 5.371087441354901e+04 + 5.321621444975662e+04 + 5.272602039469258e+04 + 5.224025278120609e+04 + 5.175887251825204e+04 + 5.128184085536604e+04 + 5.080911934597862e+04 + 5.034066989608183e+04 + 4.987645473684455e+04 + 4.941643640415772e+04 + 4.896057778639881e+04 + 4.850884207769758e+04 + 4.806119277190277e+04 + 4.761759372195903e+04 + 4.717800906318111e+04 + 4.674240321587012e+04 + 4.631074096205200e+04 + 4.588298735942246e+04 + 4.545910774537743e+04 + 4.503906779825089e+04 + 4.462283347373459e+04 + 4.421037101686000e+04 + 4.380164697651120e+04 + 4.339662816996042e+04 + 4.299528171729217e+04 + 4.259757504220949e+04 + 4.220347581706527e+04 + 4.181295199605238e+04 + 4.142597183433527e+04 + 4.104250383855765e+04 + 4.066251680332360e+04 + 4.028597979815155e+04 + 3.991286213370509e+04 + 3.954313341370015e+04 + 3.917676350485435e+04 + 3.881372250661203e+04 + 3.845398080843414e+04 + 3.809750905736740e+04 + 3.774427812448402e+04 + 3.739425915595785e+04 + 3.704742356207799e+04 + 3.670374297805164e+04 + 3.636318929176292e+04 + 3.602573463927982e+04 + 3.569135138416072e+04 + 3.536001215672983e+04 + 3.503168981923036e+04 + 3.470635744840465e+04 + 3.438398837321134e+04 + 3.406455615393208e+04 + 3.374803457309094e+04 + 3.343439765983423e+04 + 3.312361965186892e+04 + 3.281567499925836e+04 + 3.251053841177231e+04 + 3.220818480242078e+04 + 3.190858928930939e+04 + 3.161172723179306e+04 + 3.131757418548949e+04 + 3.102610591483113e+04 + 3.073729842710719e+04 + 3.045112791454117e+04 + 3.016757076691534e+04 + 2.988660361584138e+04 + 2.960820327321778e+04 + 2.933234674204399e+04 + 2.905901125674980e+04 + 2.878817423326317e+04 + 2.851981328372868e+04 + 2.825390622682930e+04 + 2.799043106146120e+04 + 2.772936599846356e+04 + 2.747068942978120e+04 + 2.721437991599866e+04 + 2.696041623625263e+04 + 2.670877735469368e+04 + 2.645944239579323e+04 + 2.621239068106394e+04 + 2.596760172331024e+04 + 2.572505519859007e+04 + 2.548473096648493e+04 + 2.524660907253227e+04 + 2.501066972455740e+04 + 2.477689330466365e+04 + 2.454526037995217e+04 + 2.431575168300119e+04 + 2.408834810912627e+04 + 2.386303072521571e+04 + 2.363978076395976e+04 + 2.341857962988761e+04 + 2.319940888531031e+04 + 2.298225025100848e+04 + 2.276708562349589e+04 + 2.255389704384009e+04 + 2.234266670372097e+04 + 2.213337697918792e+04 + 2.192601038403303e+04 + 2.172054957464747e+04 + 2.151697738916041e+04 + 2.131527680276242e+04 + 2.111543093252480e+04 + 2.091742306098243e+04 + 2.072123660422193e+04 + 2.052685512482519e+04 + 2.033426234716574e+04 + 2.014344212098682e+04 + 1.995437843426629e+04 + 1.976705544479930e+04 + 1.958145743480243e+04 + 1.939756881213320e+04 + 1.921537413684756e+04 + 1.903485810052154e+04 + 1.885600554387774e+04 + 1.867880143037429e+04 + 1.850323084111479e+04 + 1.832927901452323e+04 + 1.815693131178678e+04 + 1.798617320994377e+04 + 1.781699034075772e+04 + 1.764936845332217e+04 + 1.748329340510924e+04 + 1.731875119617678e+04 + 1.715572795110660e+04 + 1.699420990919751e+04 + 1.683418344358223e+04 + 1.667563503873475e+04 + 1.651855128954979e+04 + 1.636291893365341e+04 + 1.620872481606836e+04 + 1.605595588699932e+04 + 1.590459923143625e+04 + 1.575464203906202e+04 + 1.560607160533683e+04 + 1.545887535681969e+04 + 1.531304082443131e+04 + 1.516855564487381e+04 + 1.502540756956110e+04 + 1.488358445432479e+04 + 1.474307426938193e+04 + 1.460386509288249e+04 + 1.446594509814283e+04 + 1.432930256776952e+04 + 1.419392589719183e+04 + 1.405980357729324e+04 + 1.392692420658459e+04 + 1.379527648729671e+04 + 1.366484921045415e+04 + 1.353563127160237e+04 + 1.340761167272577e+04 + 1.328077950662285e+04 + 1.315512397004589e+04 + 1.303063434970357e+04 + 1.290730001970431e+04 + 1.278511047006237e+04 + 1.266405528001385e+04 + 1.254412410683634e+04 + 1.242530670538782e+04 + 1.230759293022181e+04 + 1.219097272173875e+04 + 1.207543611008671e+04 + 1.196097321598039e+04 + 1.184757424417080e+04 + 1.173522949287449e+04 + 1.162392934478714e+04 + 1.151366426384512e+04 + 1.140442480834694e+04 + 1.129620161634352e+04 + 1.118898540470730e+04 + 1.108276698378667e+04 + 1.097753724013271e+04 + 1.087328713748510e+04 + 1.077000773143687e+04 + 1.066769015148828e+04 + 1.056632560317495e+04 + 1.046590537899618e+04 + 1.036642084361778e+04 + 1.026786343864257e+04 + 1.017022468912003e+04 + 1.007349618928313e+04 + 9.977669609110566e+03 + 9.882736699245763e+03 + 9.788689277747368e+03 + 9.695519238085319e+03 + 9.603218551128393e+03 + 9.511779252802526e+03 + 9.421193453258842e+03 + 9.331453338088948e+03 + 9.242551155902798e+03 + 9.154479227014926e+03 + 9.067229945268125e+03 + 8.980795767109534e+03 + 8.895169220035050e+03 + 8.810342900070031e+03 + 8.726309463870590e+03 + 8.643061639313548e+03 + 8.560592220958239e+03 + 8.478894062727115e+03 + 8.397960086705119e+03 + 8.317783279679375e+03 + 8.238356687487571e+03 + 8.159673421912185e+03 + 8.081726656780256e+03 + 8.004509624589081e+03 + 7.928015622553179e+03 + 7.852238006535134e+03 + 7.777170190090504e+03 + 7.702805651284038e+03 + 7.629137923269694e+03 + 7.556160595291118e+03 + 7.483867321684783e+03 + 7.412251809739373e+03 + 7.341307820993124e+03 + 7.271029178292266e+03 + 7.201409756944273e+03 + 7.132443488123264e+03 + 7.064124361053455e+03 + 6.996446414793790e+03 + 6.929403743407001e+03 + 6.862990497342477e+03 + 6.797200875139910e+03 + 6.732029129721631e+03 + 6.667469570161823e+03 + 6.603516551523545e+03 + 6.540164480186692e+03 + 6.477407817185619e+03 + 6.415241069687017e+03 + 6.353658796883785e+03 + 6.292655608307650e+03 + 6.232226157518487e+03 + 6.172365149982773e+03 + 6.113067340658933e+03 + 6.054327527853383e+03 + 5.996140560115221e+03 + 5.938501333076975e+03 + 5.881404784939355e+03 + 5.824845903828315e+03 + 5.768819722461626e+03 + 5.713321315085574e+03 + 5.658345805472132e+03 + 5.603888359764917e+03 + 5.549944184846402e+03 + 5.496508537213697e+03 + 5.443576713947663e+03 + 5.391144051368102e+03 + 5.339205932358427e+03 + 5.287757780134069e+03 + 5.236795058291703e+03 + 5.186313275265001e+03 + 5.136307977685731e+03 + 5.086774751617681e+03 + 5.037709226449485e+03 + 4.989107068721900e+03 + 4.940963984504954e+03 + 4.893275721702213e+03 + 4.846038063285039e+03 + 4.799246830729600e+03 + 4.752897888312231e+03 + 4.706987133608333e+03 + 4.661510499922751e+03 + 4.616463963202395e+03 + 4.571843532909213e+03 + 4.527645253893021e+03 + 4.483865209302220e+03 + 4.440499515324517e+03 + 4.397544325863069e+03 + 4.354995830474305e+03 + 4.312850249817291e+03 + 4.271103841242844e+03 + 4.229752898292512e+03 + 4.188793745275323e+03 + 4.148222740634546e+03 + 4.108036277499050e+03 + 4.068230779797937e+03 + 4.028802705831530e+03 + 3.989748546349355e+03 + 3.951064821861195e+03 + 3.912748086813885e+03 + 3.874794926585441e+03 + 3.837201956255947e+03 + 3.799965825835211e+03 + 3.763083213042953e+03 + 3.726550823283946e+03 + 3.690365398229224e+03 + 3.654523706502605e+03 + 3.619022543185401e+03 + 3.583858737044133e+03 + 3.549029144619817e+03 + 3.514530650270483e+03 + 3.480360168271582e+03 + 3.446514639490202e+03 + 3.412991033253400e+03 + 3.379786348393921e+03 + 3.346897609451590e+03 + 3.314321868815865e+03 + 3.282056205761040e+03 + 3.250097724763853e+03 + 3.218443559530467e+03 + 3.187090870156345e+03 + 3.156036839918450e+03 + 3.125278679259401e+03 + 3.094813626126841e+03 + 3.064638941053712e+03 + 3.034751909739014e+03 + 3.005149845581890e+03 + 2.975830084543172e+03 + 2.946789986568892e+03 + 2.918026936797055e+03 + 2.889538343339952e+03 + 2.861321641252218e+03 + 2.833374286700636e+03 + 2.805693756659566e+03 + 2.778277556469224e+03 + 2.751123212016022e+03 + 2.724228269242043e+03 + 2.697590301914011e+03 + 2.671206904109025e+03 + 2.645075688971241e+03 + 2.619194294405700e+03 + 2.593560380141823e+03 + 2.568171626715687e+03 + 2.543025737542386e+03 + 2.518120435211838e+03 + 2.493453462030127e+03 + 2.469022585172818e+03 + 2.444825590018873e+03 + 2.420860280374604e+03 + 2.397124484732366e+03 + 2.373616049394018e+03 + 2.350332838436762e+03 + 2.327272738609171e+03 + 2.304433655218562e+03 + 2.281813512846715e+03 + 2.259410255601743e+03 + 2.237221844685107e+03 + 2.215246260811306e+03 + 2.193481506123082e+03 + 2.171925598942706e+03 + 2.150576574458654e+03 + 2.129432488764117e+03 + 2.108491414442484e+03 + 2.087751441300839e+03 + 2.067210678851041e+03 + 2.046867252353801e+03 + 2.026719303565292e+03 + 2.006764993817073e+03 + 1.987002500493615e+03 + 1.967430017215378e+03 + 1.948045754608514e+03 + 1.928847939062915e+03 + 1.909834815497017e+03 + 1.891004644375446e+03 + 1.872355700452609e+03 + 1.853886275802303e+03 + 1.835594678424827e+03 + 1.817479231062563e+03 + 1.799538272810955e+03 + 1.781770158220284e+03 + 1.764173256646331e+03 + 1.746745953278825e+03 + 1.729486647489427e+03 + 1.712393752933919e+03 + 1.695465699568615e+03 + 1.678700930817119e+03 + 1.662097903850073e+03 + 1.645655091966725e+03 + 1.629370981458450e+03 + 1.613244072202941e+03 + 1.597272880432907e+03 + 1.581455933682248e+03 + 1.565791771294578e+03 + 1.550278951289465e+03 + 1.534916043199639e+03 + 1.519701626827566e+03 + 1.504634298568118e+03 + 1.489712666891078e+03 + 1.474935351996483e+03 + 1.460300988758884e+03 + 1.445808223853118e+03 + 1.431455716350942e+03 + 1.417242138357507e+03 + 1.403166173386480e+03 + 1.389226518010158e+03 + 1.375421881007569e+03 + 1.361750982089843e+03 + 1.348212553926219e+03 + 1.334805341072190e+03 + 1.321528098720046e+03 + 1.308379594477261e+03 + 1.295358607457036e+03 + 1.282463927300085e+03 + 1.269694355823267e+03 + 1.257048705895313e+03 + 1.244525800699936e+03 + 1.232124475203576e+03 + 1.219843575042076e+03 + 1.207681956049559e+03 + 1.195638485404256e+03 + 1.183712040621380e+03 + 1.171901509388141e+03 + 1.160205790292893e+03 + 1.148623791762007e+03 + 1.137154432301071e+03 + 1.125796641257775e+03 + 1.114549357263016e+03 + 1.103411528660030e+03 + 1.092382114454342e+03 + 1.081460082700689e+03 + 1.070644411095300e+03 + 1.059934087579793e+03 + 1.049328108720195e+03 + 1.038825480494848e+03 + 1.028425218971371e+03 + 1.018126348582361e+03 + 1.007927902927668e+03 + 9.978289254104482e+02 + 9.878284675679866e+02 + 9.779255898738762e+02 + 9.681193621623704e+02 + 9.584088623012988e+02 + 9.487931771658360e+02 + 9.392714022763922e+02 + 9.298426409505287e+02 + 9.205060057185552e+02 + 9.112606172437117e+02 + 9.021056036681564e+02 + 8.930401022331439e+02 + 8.840632580637879e+02 + 8.751742235821227e+02 + 8.663721600224155e+02 + 8.576562363107178e+02 + 8.490256285516970e+02 + 8.404795212594149e+02 + 8.320171064683727e+02 + 8.236375833541338e+02 + 8.153401590664150e+02 + 8.071240480244585e+02 + 7.989884718119321e+02 + 7.909326598382663e+02 + 7.829558483527250e+02 + 7.750572805483002e+02 + 7.672362073200991e+02 + 7.594918862313154e+02 + 7.518235817392584e+02 + 7.442305655888715e+02 + 7.367121158939042e+02 + 7.292675176279554e+02 + 7.218960630476328e+02 + 7.145970504767349e+02 + 7.073697847820883e+02 + 7.002135778527235e+02 + 6.931277475695276e+02 + 6.861116184218383e+02 + 6.791645215373005e+02 + 6.722857937677298e+02 + 6.654747784472139e+02 + 6.587308253877042e+02 + 6.520532900189463e+02 + 6.454415341582659e+02 + 6.388949257382737e+02 + 6.324128381494658e+02 + 6.259946511571761e+02 + 6.196397502963263e+02 + 6.133475264197438e+02 + 6.071173768094372e+02 + 6.009487042352257e+02 + 5.948409165626937e+02 + 5.887934278801442e+02 + 5.828056577149104e+02 + 5.768770306335107e+02 + 5.710069770500405e+02 + 5.651949326339243e+02 + 5.594403381224611e+02 + 5.537426401234757e+02 + 5.481012901152631e+02 + 5.425157443662293e+02 + 5.369854648776029e+02 + 5.315099185870243e+02 + 5.260885773011631e+02 + 5.207209179649268e+02 + 5.154064223119497e+02 + 5.101445770921985e+02 + 5.049348740937783e+02 + 4.997768095346806e+02 + 4.946698844046663e+02 + 4.896136047862775e+02 + 4.846074811232588e+02 + 4.796510285618583e+02 + 4.747437670096575e+02 + 4.698852205388910e+02 + 4.650749179071526e+02 + 4.603123926691712e+02 + 4.555971823563725e+02 + 4.509288288727789e+02 + 4.463068788207030e+02 + 4.417308827523721e+02 + 4.372003955530256e+02 + 4.327149765501697e+02 + 4.282741889383891e+02 + 4.238776002586995e+02 + 4.195247821802792e+02 + 4.152153101301976e+02 + 4.109487639564401e+02 + 4.067247273397822e+02 + 4.025427875685178e+02 + 3.984025363858318e+02 + 3.943035692477232e+02 + 3.902454851077117e+02 + 3.862278871965347e+02 + 3.822503823673904e+02 + 3.783125809462488e+02 + 3.744140974100913e+02 + 3.705545496319825e+02 + 3.667335588488600e+02 + 3.629507504654170e+02 + 3.592057531920941e+02 + 3.554981990087823e+02 + 3.518277237997781e+02 + 3.481939667588409e+02 + 3.445965704231110e+02 + 3.410351808614133e+02 + 3.375094474036331e+02 + 3.340190228447264e+02 + 3.305635632260179e+02 + 3.271427276872922e+02 + 3.237561788604219e+02 + 3.204035826343936e+02 + 3.170846077594078e+02 + 3.137989262426811e+02 + 3.105462135925898e+02 + 3.073261480789523e+02 + 3.041384108878384e+02 + 3.009826866110996e+02 + 2.978586626197819e+02 + 2.947660292278245e+02 + 2.917044799774912e+02 + 2.886737110898206e+02 + 2.856734216347172e+02 + 2.827033138205502e+02 + 2.797630925185924e+02 + 2.768524653868921e+02 + 2.739711430162750e+02 + 2.711188386172004e+02 + 2.682952682417776e+02 + 2.655001507075566e+02 + 2.627332073441469e+02 + 2.599941621974095e+02 + 2.572827420191948e+02 + 2.545986760992428e+02 + 2.519416963867954e+02 + 2.493115374004747e+02 + 2.467079361520411e+02 + 2.441306322519956e+02 + 2.415793676749632e+02 + 2.390538868162835e+02 + 2.365539368283623e+02 + 2.340792670618049e+02 + 2.316296291092408e+02 + 2.292047772816256e+02 + 2.268044680583002e+02 + 2.244284601550858e+02 + 2.220765149177526e+02 + 2.197483957886078e+02 + 2.174438683808943e+02 + 2.151627007150871e+02 + 2.129046628991759e+02 + 2.106695273325222e+02 + 2.084570687245048e+02 + 2.062670637405841e+02 + 2.040992912166423e+02 + 2.019535322675416e+02 + 1.998295699356443e+02 + 1.977271893831886e+02 + 1.956461779038128e+02 + 1.935863247355415e+02 + 1.915474213798307e+02 + 1.895292611678483e+02 + 1.875316391958052e+02 + 1.855543528482985e+02 + 1.835972014670904e+02 + 1.816599861324650e+02 + 1.797425099308505e+02 + 1.778445779263947e+02 + 1.759659969484732e+02 + 1.741065756739388e+02 + 1.722661246609050e+02 + 1.704444562587039e+02 + 1.686413847511611e+02 + 1.668567261386198e+02 + 1.650902980928235e+02 + 1.633419201885993e+02 + 1.616114136777071e+02 + 1.598986014805750e+02 + 1.582033083619683e+02 + 1.565253606828442e+02 + 1.548645864376032e+02 + 1.532208154238845e+02 + 1.515938789750561e+02 + 1.499836100268175e+02 + 1.483898432531407e+02 + 1.468124148175615e+02 + 1.452511624739094e+02 + 1.437059256375598e+02 + 1.421765451600804e+02 + 1.406628634671130e+02 + 1.391647245894945e+02 + 1.376819739463268e+02 + 1.362144585029153e+02 + 1.347620267668397e+02 + 1.333245285902542e+02 + 1.319018153476916e+02 + 1.304937399101563e+02 + 1.291001564496927e+02 + 1.277209206089626e+02 + 1.263558894821456e+02 + 1.250049214462411e+02 + 1.236678763269726e+02 + 1.223446153162142e+02 + 1.210350008534558e+02 + 1.197388968258891e+02 + 1.184561684202564e+02 + 1.171866820332312e+02 + 1.159303054814129e+02 + 1.146869078350233e+02 + 1.134563593521917e+02 + 1.122385316718855e+02 + 1.110332976232454e+02 + 1.098405311979204e+02 + 1.086601077608261e+02 + 1.074919038117468e+02 + 1.063357969778381e+02 + 1.051916662270612e+02 + 1.040593916259906e+02 + 1.029388543466298e+02 + 1.018299368341955e+02 + 1.007325226151947e+02 + 9.964649633160360e+01 + 9.857174381861689e+01 + 9.750815195741964e+01 + 9.645560874783912e+01 + 9.541400332430685e+01 + 9.438322582683587e+01 + 9.336316750654088e+01 + 9.235372071959520e+01 + 9.135477878367931e+01 + 9.036623609174318e+01 + 8.938798812110527e+01 + 8.841993128541375e+01 + 8.746196304638978e+01 + 8.651398190997742e+01 + 8.557588728936869e+01 + 8.464757961904559e+01 + 8.372896034435935e+01 + 8.281993180539014e+01 + 8.192039735296085e+01 + 8.103026128111058e+01 + 8.014942874495698e+01 + 7.927780590401203e+01 + 7.841529983858037e+01 + 7.756181847617736e+01 + 7.671727071446733e+01 + 7.588156631674119e+01 + 7.505461587372024e+01 + 7.423633095310755e+01 + 7.342662394299153e+01 + 7.262540803009301e+01 + 7.183259736215271e+01 + 7.104810687905130e+01 + 7.027185230277652e+01 + 6.950375028269488e+01 + 6.874371822566854e+01 + 6.799167430460268e+01 + 6.724753760248480e+01 + 6.651122793902957e+01 + 6.578266588744034e+01 + 6.506177289160765e+01 + 6.434847109859102e+01 + 6.364268339115934e+01 + 6.294433351272004e+01 + 6.225334589309644e+01 + 6.156964567793161e+01 + 6.089315882540817e+01 + 6.022381196253272e+01 + 5.956153243192431e+01 + 5.890624835784802e+01 + 5.825788851805937e+01 + 5.761638239540161e+01 + 5.698166019812880e+01 + 5.635365277455941e+01 + 5.573229169643452e+01 + 5.511750921575248e+01 + 5.450923819422894e+01 + 5.390741220139490e+01 + 5.331196548214332e+01 + 5.272283288224639e+01 + 5.213994992103501e+01 + 5.156325275682114e+01 + 5.099267813956413e+01 + 5.042816349744366e+01 + 4.986964685712531e+01 + 4.931706681520628e+01 + 4.877036264381848e+01 + 4.822947419115650e+01 + 4.769434186145065e+01 + 4.716490671386568e+01 + 4.664111036072763e+01 + 4.612289496053356e+01 + 4.561020331071316e+01 + 4.510297873475514e+01 + 4.460116508782800e+01 + 4.410470685618211e+01 + 4.361354903662819e+01 + 4.312763714409483e+01 + 4.264691728541290e+01 + 4.217133607138089e+01 + 4.170084064118655e+01 + 4.123537869187841e+01 + 4.077489840121474e+01 + 4.031934846724150e+01 + 3.986867813088396e+01 + 3.942283710103097e+01 + 3.898177559914080e+01 + 3.854544435810492e+01 + 3.811379455962712e+01 + 3.768677789368684e+01 + 3.726434656726136e+01 + 3.684645322013298e+01 + 3.643305096011807e+01 + 3.602409338288954e+01 + 3.561953452360934e+01 + 3.521932891649128e+01 + 3.482343153306697e+01 + 3.443179775085747e+01 + 3.404438343901749e+01 + 3.366114490993698e+01 + 3.328203887889210e+01 + 3.290702252020257e+01 + 3.253605343510379e+01 + 3.216908962260722e+01 + 3.180608952699355e+01 + 3.144701200592500e+01 + 3.109181631097627e+01 + 3.074046212448961e+01 + 3.039290952642646e+01 + 3.004911898312090e+01 + 2.970905136572861e+01 + 2.937266792505025e+01 + 2.903993030352619e+01 + 2.871080057194543e+01 + 2.838524113862799e+01 + 2.806321475945346e+01 + 2.774468462701056e+01 + 2.742961427426368e+01 + 2.711796757973106e+01 + 2.680970882898042e+01 + 2.650480264209962e+01 + 2.620321398820359e+01 + 2.590490820737408e+01 + 2.560985096325826e+01 + 2.531800827674159e+01 + 2.502934653543958e+01 + 2.474383242809609e+01 + 2.446143296972707e+01 + 2.418211555345684e+01 + 2.390584787621480e+01 + 2.363259794654911e+01 + 2.336233413255323e+01 + 2.309502509558490e+01 + 2.283063979379989e+01 + 2.256914753657127e+01 + 2.231051792982671e+01 + 2.205472087402378e+01 + 2.180172658443576e+01 + 2.155150556919213e+01 + 2.130402866286656e+01 + 2.105926696778129e+01 + 2.081719185319289e+01 + 2.057777503962135e+01 + 2.034098850331043e+01 + 2.010680447006149e+01 + 1.987519550063876e+01 + 1.964613442528577e+01 + 1.941959432580914e+01 + 1.919554855827313e+01 + 1.897397075515799e+01 + 1.875483482712853e+01 + 1.853811494931854e+01 + 1.832378554313712e+01 + 1.811182128985819e+01 + 1.790219714756028e+01 + 1.769488831219516e+01 + 1.748987022919803e+01 + 1.728711861052799e+01 + 1.708660939866258e+01 + 1.688831878332332e+01 + 1.669222321036695e+01 + 1.649829934690638e+01 + 1.630652410217020e+01 + 1.611687463464447e+01 + 1.592932831698975e+01 + 1.574386275792588e+01 + 1.556045580679505e+01 + 1.537908552081595e+01 + 1.519973018767385e+01 + 1.502236832516821e+01 + 1.484697865494502e+01 + 1.467354012841447e+01 + 1.450203191274907e+01 + 1.433243337158686e+01 + 1.416472409674524e+01 + 1.399888388846149e+01 + 1.383489273805109e+01 + 1.367273085728561e+01 + 1.351237865788487e+01 + 1.335381673920349e+01 + 1.319702591751814e+01 + 1.304198719993403e+01 + 1.288868177614133e+01 + 1.273709104714046e+01 + 1.258719659672914e+01 + 1.243898018736778e+01 + 1.229242378719869e+01 + 1.214750954184370e+01 + 1.200421977308728e+01 + 1.186253699764556e+01 + 1.172244390266849e+01 + 1.158392335040491e+01 + 1.144695839482221e+01 + 1.131153225250660e+01 + 1.117762830926809e+01 + 1.104523013501555e+01 + 1.091432145806843e+01 + 1.078488617566414e+01 + 1.065690836231677e+01 + 1.053037224372639e+01 + 1.040526220941685e+01 + 1.028156282144551e+01 + 1.015925878941852e+01 + 1.003833498402321e+01 + 9.918776439618298e+00 + 9.800568334039683e+00 + 9.683696005431957e+00 + 9.568144946174611e+00 + 9.453900787919052e+00 + 9.340949322860192e+00 + 9.229276489267212e+00 + 9.118868359446189e+00 + 9.009711163642070e+00 + 8.901791273796370e+00 + 8.795095191785947e+00 + 8.689609570648638e+00 + 8.585321201853549e+00 + 8.482217005296002e+00 + 8.380284046581689e+00 + 8.279509523576925e+00 + 8.179880760330461e+00 + 8.081385224417300e+00 + 7.984010508389771e+00 + 7.887744326863491e+00 + 7.792574535206823e+00 + 7.698489109519010e+00 + 7.605476146133772e+00 + 7.513523878187234e+00 + 7.422620653280592e+00 + 7.332754935473541e+00 + 7.243915322674125e+00 + 7.156090524556396e+00 + 7.069269365741123e+00 + 6.983440797017198e+00 + 6.898593877074525e+00 + 6.814717779007339e+00 + 6.731801797516630e+00 + 6.649835331100769e+00 + 6.568807890289103e+00 + 6.488709103685009e+00 + 6.409528699865692e+00 + 6.331256515961359e+00 + 6.253882503643640e+00 + 6.177396712290002e+00 + 6.101789297547966e+00 + 6.027050522250886e+00 + 5.953170745278158e+00 + 5.880140434362498e+00 + 5.807950156612827e+00 + 5.736590570385212e+00 + 5.666052441485230e+00 + 5.596326632559173e+00 + 5.527404095182261e+00 + 5.459275884510260e+00 + 5.391933149962507e+00 + 5.325367127621260e+00 + 5.259569151260657e+00 + 5.194530647433734e+00 + 5.130243129823101e+00 + 5.066698206416531e+00 + 5.003887570881058e+00 + 4.941803001453549e+00 + 4.880436373900456e+00 + 4.819779643419130e+00 + 4.759824844738533e+00 + 4.700564110200850e+00 + 4.641989649988083e+00 + 4.584093751736625e+00 + 4.526868793565781e+00 + 4.470307230074105e+00 + 4.414401594729965e+00 + 4.359144507919768e+00 + 4.304528662004969e+00 + 4.250546825086277e+00 + 4.197191851627622e+00 + 4.144456666159979e+00 + 4.092334266478478e+00 + 4.040817732600400e+00 + 3.989900213340690e+00 + 3.939574930757155e+00 + 3.889835183170760e+00 + 3.840674335613269e+00 + 3.792085827136757e+00 + 3.744063169627271e+00 + 3.696599938809659e+00 + 3.649689781941428e+00 + 3.603326418080199e+00 + 3.557503628382808e+00 + 3.512215261374414e+00 + 3.467455236465362e+00 + 3.423217534816642e+00 + 3.379496201686086e+00 + 3.336285348654454e+00 + 3.293579148649655e+00 + 3.251371841226642e+00 + 3.209657725464606e+00 + 3.168431157939940e+00 + 3.127686563504146e+00 + 3.087418425005168e+00 + 3.047621281181530e+00 + 3.008289735457784e+00 + 2.969418446796439e+00 + 2.931002129285814e+00 + 2.893035560502260e+00 + 2.855513571053400e+00 + 2.818431044951714e+00 + 2.781782927380857e+00 + 2.745564215550934e+00 + 2.709769959926351e+00 + 2.674395268211684e+00 + 2.639435297946605e+00 + 2.604885259637663e+00 + 2.570740420206636e+00 + 2.536996093812299e+00 + 2.503647644969884e+00 + 2.470690493614489e+00 + 2.438120105903952e+00 + 2.405931997235836e+00 + 2.374121735683807e+00 + 2.342684934097179e+00 + 2.311617253784086e+00 + 2.280914406216957e+00 + 2.250572146781167e+00 + 2.220586279273893e+00 + 2.190952653478714e+00 + 2.161667162009881e+00 + 2.132725747880213e+00 + 2.104124396422602e+00 + 2.075859132902099e+00 + 2.047926031580743e+00 + 2.020321209266692e+00 + 1.993040822266800e+00 + 1.966081073072867e+00 + 1.939438205166541e+00 + 1.913108500802827e+00 + 1.887088286918805e+00 + 1.861373929947225e+00 + 1.835961834345823e+00 + 1.810848447621857e+00 + 1.786030255213551e+00 + 1.761503779943143e+00 + 1.737265586321442e+00 + 1.713312275115213e+00 + 1.689640483715834e+00 + 1.666246890168493e+00 + 1.643128207113473e+00 + 1.620281182788458e+00 + 1.597702604935450e+00 + 1.575389294699183e+00 + 1.553338108095691e+00 + 1.531545938928566e+00 + 1.510009713407906e+00 + 1.488726392261371e+00 + 1.467692971937850e+00 + 1.446906480207705e+00 + 1.426363979153670e+00 + 1.406062564959602e+00 + 1.385999363940103e+00 + 1.366171536058216e+00 + 1.346576274130787e+00 + 1.327210800124654e+00 + 1.308072368828510e+00 + 1.289158266486426e+00 + 1.270465807665891e+00 + 1.251992339187792e+00 + 1.233735237889272e+00 + 1.215691907984316e+00 + 1.197859785096899e+00 + 1.180236333919823e+00 + 1.162819045897185e+00 + 1.145605442741095e+00 + 1.128593073973588e+00 + 1.111779515284567e+00 + 1.095162371880828e+00 + 1.078739275475473e+00 + 1.062507883334588e+00 + 1.046465881646835e+00 + 1.030610981874106e+00 + 1.014940920295882e+00 + 9.994534611152829e-01 + 9.841463930086627e-01 + 9.690175291593270e-01 + 9.540647093618980e-01 + 9.392857969051628e-01 + 9.246786792729085e-01 + 9.102412696338000e-01 + 8.959715035501604e-01 + 8.818673402027871e-01 + 8.679267639726700e-01 + 8.541477806828601e-01 + 8.405284189410931e-01 + 8.270667316623297e-01 + 8.137607925445277e-01 + 8.006086976874616e-01 + 7.876085666023992e-01 + 7.747585387876328e-01 + 7.620567754307064e-01 + 7.495014602973783e-01 + 7.370907966216457e-01 + 7.248230088485250e-01 + 7.126963429559050e-01 + 7.007090638413013e-01 + 6.888594572109231e-01 + 6.771458292779966e-01 + 6.655665045807759e-01 + 6.541198279036498e-01 + 6.428041637825399e-01 + 6.316178947210964e-01 + 6.205594229105623e-01 + 6.096271692194244e-01 + 5.988195719936049e-01 + 5.881350889221284e-01 + 5.775721954595900e-01 + 5.671293841176136e-01 + 5.568051664208875e-01 + 5.465980707194322e-01 + 5.365066418737608e-01 + 5.265294432641988e-01 + 5.166650544231339e-01 + 5.069120710261563e-01 + 4.972691065208522e-01 + 4.877347899800689e-01 + 4.783077664492044e-01 + 4.689866979017357e-01 + 4.597702612073233e-01 + 4.506571489021905e-01 + 4.416460700276524e-01 + 4.327357479310139e-01 + 4.239249212044141e-01 + 4.152123442913231e-01 + 4.065967853823845e-01 + 3.980770275573000e-01 + 3.896518691214699e-01 + 3.813201217302061e-01 + 3.730806117264635e-01 + 3.649321798861945e-01 + 3.568736798624809e-01 + 3.489039797704173e-01 + 3.410219613244947e-01 + 3.332265187013593e-01 + 3.255165605086018e-01 + 3.178910083898543e-01 + 3.103487958586359e-01 + 3.028888701360887e-01 + 2.955101913786422e-01 + 2.882117315106021e-01 + 2.809924755137371e-01 + 2.738514204626846e-01 + 2.667875748867065e-01 + 2.597999604522775e-01 + 2.528876103054370e-01 + 2.460495685714950e-01 + 2.392848919788540e-01 + 2.325926484518769e-01 + 2.259719167857841e-01 + 2.194217877354759e-01 + 2.129413627175208e-01 + 2.065297538274627e-01 + 2.001860848470379e-01 + 1.939094897086202e-01 + 1.876991127051828e-01 + 1.815541094417414e-01 + 1.754736452756007e-01 + 1.694568956974621e-01 + 1.635030470498157e-01 + 1.576112950508609e-01 + 1.517808453564031e-01 + 1.460109140364644e-01 + 1.403007261766342e-01 + 1.346495165784953e-01 + 1.290565300180611e-01 + 1.235210199704221e-01 + 1.180422494287421e-01 + 1.126194909047226e-01 + 1.072520253105725e-01 + 1.019391428882170e-01 + 9.668014295417104e-02 + 9.147433293860986e-02 + 8.632102939670111e-02 + 8.121955756354367e-02 + 7.616925052308399e-02 + 7.116945022739722e-02 + 6.621950693894064e-02 + 6.131877854698166e-02 + 5.646663155698649e-02 + 5.166244039926220e-02 + 4.690558691983024e-02 + 4.219546135291571e-02 + 3.753146148005052e-02 + 3.291299228995687e-02 + 2.833946692296159e-02 + 2.381030573044091e-02 + 1.932493608562982e-02 + 1.488279323951757e-02 + 1.048331933378767e-02 + 6.125963364112487e-03 + 1.810181928441051e-03 + -2.464561780353438e-03 + -6.698797872311066e-03 + -1.089304914227914e-02 + -1.504783206514415e-02 + -1.916365656771068e-02 + -2.324102555134537e-02 + -2.728043585421590e-02 + -3.128237791440187e-02 + -3.524733543518016e-02 + -3.917578628623598e-02 + -4.306820207473552e-02 + -4.692504798332473e-02 + -5.074678357205704e-02 + -5.453386225593208e-02 + -5.828673132987599e-02 + -6.200583267980977e-02 + -6.569160218738558e-02 + -6.934446987831328e-02 + -7.296486055963237e-02 + -7.655319319650564e-02 + -8.010988114185758e-02 + -8.363533270338043e-02 + -8.712995053780122e-02 + -9.059413193193132e-02 + -9.402826926020177e-02 + -9.743274938790861e-02 + -1.008079541064505e-01 + -1.041542604608216e-01 + -1.074720401131623e-01 + -1.107616598682374e-01 + -1.140234819193764e-01 + -1.172578632830942e-01 + -1.204651563472630e-01 + -1.236457089966172e-01 + -1.267998640735528e-01 + -1.299279600346222e-01 + -1.330303309819349e-01 + -1.361073061299021e-01 + -1.391592104828469e-01 + -1.421863647988072e-01 + -1.451890851855495e-01 + -1.481676836992942e-01 + -1.511224682100277e-01 + -1.540537421399553e-01 + -1.569618050599837e-01 + -1.598469524764272e-01 + -1.627094755828420e-01 + -1.655496618778017e-01 + -1.683677949236447e-01 + -1.711641541517163e-01 + -1.739390154343296e-01 + -1.766926508045859e-01 + -1.794253283544359e-01 + -1.821373127460387e-01 + -1.848288648981078e-01 + -1.875002419786971e-01 + -1.901516978443653e-01 + -1.927834826976386e-01 + -1.953958431846420e-01 + -1.979890227559544e-01 + -2.005632612952037e-01 + -2.031187952951354e-01 + -2.056558581586019e-01 + -2.081746798356756e-01 + -2.106754870586183e-01 + -2.131585035595214e-01 + -2.156239496932611e-01 + -2.180720427808593e-01 + -2.205029972492180e-01 + -2.229170242226580e-01 + -2.253143319488933e-01 + -2.276951258807381e-01 + -2.300596082676421e-01 + -2.324079786235529e-01 + -2.347404337552020e-01 + -2.370571673694448e-01 + -2.393583705568364e-01 + -2.416442317867498e-01 + -2.439149365965413e-01 + -2.461706679522762e-01 + -2.484116061949506e-01 + -2.506379289887288e-01 + -2.528498115997045e-01 + -2.550474267213886e-01 + -2.572309443123204e-01 + -2.594005320626763e-01 + -2.615563552809436e-01 + -2.636985766193694e-01 + -2.658273564814242e-01 + -2.679428529152917e-01 + -2.700452216727706e-01 + -2.721346163277996e-01 + -2.742111879420810e-01 + -2.762750854486131e-01 + -2.783264557785196e-01 + -2.803654434689624e-01 + -2.823921908087395e-01 + -2.844068381756654e-01 + -2.864095239109061e-01 + -2.884003839950212e-01 + -2.903795524955394e-01 + -2.923471617904532e-01 + -2.943033420191942e-01 + -2.962482212763220e-01 + -2.981819256807660e-01 + -3.001045795717013e-01 + -3.020163055275332e-01 + -3.039172239885258e-01 + -3.058074538261057e-01 + -3.076871122705734e-01 + -3.095563141516905e-01 + -3.114151727559051e-01 + -3.132637999546681e-01 + -3.151023057635463e-01 + -3.169307984585886e-01 + -3.187493845723606e-01 + -3.205581689117248e-01 + -3.223572548745565e-01 + -3.241467443749414e-01 + -3.259267374142799e-01 + -3.276973325691862e-01 + -3.294586270053045e-01 + -3.312107160890871e-01 + -3.329536937962613e-01 + -3.346876527428858e-01 + -3.364126840777460e-01 + -3.381288775295205e-01 + -3.398363212268976e-01 + -3.415351019703767e-01 + -3.432253052941175e-01 + -3.449070152459444e-01 + -3.465803145779997e-01 + -3.482452848007116e-01 + -3.499020059501182e-01 + -3.515505568826250e-01 + -3.531910152859590e-01 + -3.548234573752725e-01 + -3.564479582224311e-01 + -3.580645917875246e-01 + -3.596734306319520e-01 + -3.612745463110025e-01 + -3.628680093075062e-01 + -3.644538885605036e-01 + -3.660322520458910e-01 + -3.676031668332329e-01 + -3.691666987516095e-01 + -3.707229124779602e-01 + -3.722718716670178e-01 + -3.738136390608672e-01 + -3.753482761467254e-01 + -3.768758432749062e-01 + -3.783964002465877e-01 + -3.799100056353694e-01 + -3.814167166997948e-01 + -3.829165900177774e-01 + -3.844096813827866e-01 + -3.858960456354959e-01 + -3.873757364088854e-01 + -3.888488064600710e-01 + -3.903153077375676e-01 + -3.917752913452379e-01 + -3.932288075353338e-01 + -3.946759055987133e-01 + -3.961166339473677e-01 + -3.975510402779339e-01 + -3.989791715122148e-01 + -4.004010735521537e-01 + -4.018167915706618e-01 + -4.032263700983349e-01 + -4.046298528561717e-01 + -4.060272827301947e-01 + -4.074187016384220e-01 + -4.088041511045764e-01 + -4.101836719923956e-01 + -4.115573039941210e-01 + -4.129250863351419e-01 + -4.142870577221169e-01 + -4.156432559911067e-01 + -4.169937183587125e-01 + -4.183384813924751e-01 + -4.196775807600287e-01 + -4.210110517890521e-01 + -4.223389293056927e-01 + -4.236612469859990e-01 + -4.249780381358575e-01 + -4.262893357128523e-01 + -4.275951718561576e-01 + -4.288955781548201e-01 + -4.301905856512787e-01 + -4.314802245948859e-01 + -4.327645248125306e-01 + -4.340435157401011e-01 + -4.353172261319986e-01 + -4.365856843065750e-01 + -4.378489181140523e-01 + -4.391069545111200e-01 + -4.403598201489721e-01 + -4.416075414900063e-01 + -4.428501441617211e-01 + -4.440876533447125e-01 + -4.453200938994337e-01 + -4.465474900663732e-01 + -4.477698656375739e-01 + -4.489872440132049e-01 + -4.501996482091757e-01 + -4.514071007293397e-01 + -4.526096234423685e-01 + -4.538072381973440e-01 + -4.549999664346564e-01 + -4.561878287239103e-01 + -4.573708454564815e-01 + -4.585490367840940e-01 + -4.597224223838171e-01 + -4.608910215483196e-01 + -4.620548532229073e-01 + -4.632139359470852e-01 + -4.643682878385313e-01 + -4.655179266829866e-01 + -4.666628700026897e-01 + -4.678031349656974e-01 + -4.689387384105899e-01 + -4.700696968538801e-01 + -4.711960263436956e-01 + -4.723177425902391e-01 + -4.734348612979066e-01 + -4.745473978039702e-01 + -4.756553669920032e-01 + -4.767587833133353e-01 + -4.778576611599107e-01 + -4.789520147944609e-01 + -4.800418577229591e-01 + -4.811272034289876e-01 + -4.822080654661853e-01 + -4.832844566828151e-01 + -4.843563896207272e-01 + -4.854238767122197e-01 + -4.864869303811440e-01 + -4.875455627166793e-01 + -4.885997853037442e-01 + -4.896496095260600e-01 + -4.906950466868944e-01 + -4.917361080077966e-01 + -4.927728043326091e-01 + -4.938051461993963e-01 + -4.948331438840640e-01 + -4.958568075736277e-01 + -4.968761473512240e-01 + -4.978911731225704e-01 + -4.989018945002663e-01 + -4.999083208482176e-01 + -5.009104613021941e-01 + -5.019083248972503e-01 + -5.029019205602130e-01 + -5.038912570358333e-01 + -5.048763428126108e-01 + -5.058571862004539e-01 + -5.068337954285524e-01 + -5.078061785541023e-01 + -5.087743434557772e-01 + -5.097382978215317e-01 + -5.106980492175714e-01 + -5.116536051046018e-01 + -5.126049728172039e-01 + -5.135521595357905e-01 + -5.144951722594635e-01 + -5.154340177423219e-01 + -5.163687027408718e-01 + -5.172992340140365e-01 + -5.182256180631845e-01 + -5.191478612397169e-01 + -5.200659697981208e-01 + -5.209799499565910e-01 + -5.218898077706294e-01 + -5.227955490891694e-01 + -5.236971796844924e-01 + -5.245947053253370e-01 + -5.254881317688337e-01 + -5.263774645512288e-01 + -5.272627090126604e-01 + -5.281438703531137e-01 + -5.290209539977515e-01 + -5.298939652703512e-01 + -5.307629091340913e-01 + -5.316277904778723e-01 + -5.324886142110474e-01 + -5.333453852986519e-01 + -5.341981085648844e-01 + -5.350467886587517e-01 + -5.358914300418783e-01 + -5.367320373706596e-01 + -5.375686152841432e-01 + -5.384011680734792e-01 + -5.392297001047128e-01 + -5.400542157871245e-01 + -5.408747194069761e-01 + -5.416912150785628e-01 + -5.425037068232484e-01 + -5.433121987186342e-01 + -5.441166948984431e-01 + -5.449171994793097e-01 + -5.457137163885653e-01 + -5.465062494211190e-01 + -5.472948023341125e-01 + -5.480793790518950e-01 + -5.488599834110802e-01 + -5.496366191273470e-01 + -5.504092899443518e-01 + -5.511779994928784e-01 + -5.519427513029639e-01 + -5.527035490710489e-01 + -5.534603964642170e-01 + -5.542132970155166e-01 + -5.549622540841348e-01 + -5.557072711574547e-01 + -5.564483519201564e-01 + -5.571854997347668e-01 + -5.579187179919134e-01 + -5.586480102698358e-01 + -5.593733798207278e-01 + -5.600948299343745e-01 + -5.608123641702291e-01 + -5.615259858006734e-01 + -5.622356980349319e-01 + -5.629415042452564e-01 + -5.636434078437447e-01 + -5.643414121719204e-01 + -5.650355204128609e-01 + -5.657257359067179e-01 + -5.664120620074502e-01 + -5.670945018655422e-01 + -5.677730587569899e-01 + -5.684477360210265e-01 + -5.691185368946378e-01 + -5.697854646230838e-01 + -5.704485224954847e-01 + -5.711077138615709e-01 + -5.717630418949810e-01 + -5.724145097236072e-01 + -5.730621208149326e-01 + -5.737058785857441e-01 + -5.743457862811681e-01 + -5.749818469287452e-01 + -5.756140639044424e-01 + -5.762424408221788e-01 + -5.768669806578469e-01 + -5.774876866716016e-01 + -5.781045625480883e-01 + -5.787176116189466e-01 + -5.793268371095267e-01 + -5.799322422747063e-01 + -5.805338304985933e-01 + -5.811316053198213e-01 + -5.817255703521417e-01 + -5.823157287697720e-01 + -5.829020838792335e-01 + -5.834846393919914e-01 + -5.840633987477251e-01 + -5.846383653791880e-01 + -5.852095428966003e-01 + -5.857769347455314e-01 + -5.863405444497650e-01 + -5.869003757905312e-01 + -5.874564321893059e-01 + -5.880087171065591e-01 + -5.885572344676900e-01 + -5.891019879107314e-01 + -5.896429809482692e-01 + -5.901802172691951e-01 + -5.907137006344829e-01 + -5.912434348345698e-01 + -5.917694236578408e-01 + -5.922916709366750e-01 + -5.928101804947136e-01 + -5.933249560457028e-01 + -5.938360014505492e-01 + -5.943433207019595e-01 + -5.948469177828256e-01 + -5.953467966083078e-01 + -5.958429610716830e-01 + -5.963354151930935e-01 + -5.968241629967538e-01 + -5.973092085065336e-01 + -5.977905558475627e-01 + -5.982682091919279e-01 + -5.987421727060626e-01 + -5.992124504335189e-01 + -5.996790465513929e-01 + -6.001419654039183e-01 + -6.006012111567195e-01 + -6.010567880969444e-01 + -6.015087007096906e-01 + -6.019569532209035e-01 + -6.024015499278780e-01 + -6.028424953542454e-01 + -6.032797938560218e-01 + -6.037134498528102e-01 + -6.041434679778884e-01 + -6.045698527611498e-01 + -6.049926086966888e-01 + -6.054117403213235e-01 + -6.058272522301629e-01 + -6.062391491063152e-01 + -6.066474357377917e-01 + -6.070521167750024e-01 + -6.074531969194140e-01 + -6.078506811508277e-01 + -6.082445741313003e-01 + -6.086348804777613e-01 + -6.090216052590219e-01 + -6.094047534940796e-01 + -6.097843300758768e-01 + -6.101603397998798e-01 + -6.105327877567767e-01 + -6.109016791586246e-01 + -6.112670188349342e-01 + -6.116288118244032e-01 + -6.119870634033990e-01 + -6.123417788015699e-01 + -6.126929631816058e-01 + -6.130406216857481e-01 + -6.133847595846266e-01 + -6.137253821544012e-01 + -6.140624946688724e-01 + -6.143961025253998e-01 + -6.147262111225434e-01 + -6.150528258526081e-01 + -6.153759522847098e-01 + -6.156955958775930e-01 + -6.160117619293205e-01 + -6.163244560848270e-01 + -6.166336840291214e-01 + -6.169394513190288e-01 + -6.172417635406579e-01 + -6.175406263458638e-01 + -6.178360454670117e-01 + -6.181280266570051e-01 + -6.184165756494230e-01 + -6.187016981615432e-01 + -6.189834001775048e-01 + -6.192616876391890e-01 + -6.195365661632112e-01 + -6.198080416901866e-01 + -6.200761202902517e-01 + -6.203408078682074e-01 + -6.206021103982113e-01 + -6.208600339819080e-01 + -6.211145848662764e-01 + -6.213657689929696e-01 + -6.216135922609831e-01 + -6.218580610691606e-01 + -6.220991816188346e-01 + -6.223369599996067e-01 + -6.225714026287344e-01 + -6.228025157602012e-01 + -6.230303055354752e-01 + -6.232547783843585e-01 + -6.234759406837302e-01 + -6.236937987287140e-01 + -6.239083589372102e-01 + -6.241196278170006e-01 + -6.243276119301496e-01 + -6.245323178052680e-01 + -6.247337518435782e-01 + -6.249319204108740e-01 + -6.251268303788342e-01 + -6.253184884524475e-01 + -6.255069009766967e-01 + -6.256920746682559e-01 + -6.258740163701917e-01 + -6.260527328726909e-01 + -6.262282307801417e-01 + -6.264005167589047e-01 + -6.265695976839164e-01 + -6.267354804070123e-01 + -6.268981717782514e-01 + -6.270576786775590e-01 + -6.272140080071444e-01 + -6.273671666729116e-01 + -6.275171615794166e-01 + -6.276639997579259e-01 + -6.278076882572314e-01 + -6.279482340166599e-01 + -6.280856440870369e-01 + -6.282199255881125e-01 + -6.283510856176625e-01 + -6.284791313265480e-01 + -6.286040698559974e-01 + -6.287259082244099e-01 + -6.288446536460758e-01 + -6.289603134448759e-01 + -6.290728947784819e-01 + -6.291824049507972e-01 + -6.292888513285516e-01 + -6.293922409846477e-01 + -6.294925812787570e-01 + -6.295898797737625e-01 + -6.296841435561055e-01 + -6.297753800606519e-01 + -6.298635970427606e-01 + -6.299488015475858e-01 + -6.300310009429952e-01 + -6.301102030719377e-01 + -6.301864151575183e-01 + -6.302596446383568e-01 + -6.303298993873763e-01 + -6.303971867419816e-01 + -6.304615141097702e-01 + -6.305228892569528e-01 + -6.305813198299395e-01 + -6.306368133767261e-01 + -6.306893774095105e-01 + -6.307390196514921e-01 + -6.307857478830511e-01 + -6.308295697989852e-01 + -6.308704929504492e-01 + -6.309085250312678e-01 + -6.309436740902202e-01 + -6.309759477467396e-01 + -6.310053535533821e-01 + -6.310318994644450e-01 + -6.310555933584764e-01 + -6.310764430051494e-01 + -6.310944561130111e-01 + -6.311096405770181e-01 + -6.311220043955681e-01 + -6.311315554778594e-01 + -6.311383016234806e-01 + -6.311422506666780e-01 + -6.311434107412031e-01 + -6.311417897164122e-01 + -6.311373953205278e-01 + -6.311302357049288e-01 + -6.311203189243589e-01 + -6.311076528624781e-01 + -6.310922454230109e-01 + -6.310741047155799e-01 + -6.310532389686786e-01 + -6.310296560496979e-01 + -6.310033639455465e-01 + -6.309743708604478e-01 + -6.309426848479184e-01 + -6.309083139781310e-01 + -6.308712663782245e-01 + -6.308315500367087e-01 + -6.307891730465176e-01 + -6.307441436976022e-01 + -6.306964701860184e-01 + -6.306461606431758e-01 + -6.305932231701467e-01 + -6.305376659121696e-01 + -6.304794970062023e-01 + -6.304187245700347e-01 + -6.303553569805046e-01 + -6.302894025962669e-01 + -6.302208694979176e-01 + -6.301497657748981e-01 + -6.300760996409697e-01 + -6.299998795100692e-01 + -6.299211136514613e-01 + -6.298398102240708e-01 + -6.297559773915322e-01 + -6.296696236381925e-01 + -6.295807574013027e-01 + -6.294893864698483e-01 + -6.293955192818200e-01 + -6.292991645521353e-01 + -6.292003301703828e-01 + -6.290990243894826e-01 + -6.289952557946986e-01 + -6.288890326523825e-01 + -6.287803632353285e-01 + -6.286692558944412e-01 + -6.285557190083507e-01 + -6.284397608763040e-01 + -6.283213897612988e-01 + -6.282006141317734e-01 + -6.280774423035318e-01 + -6.279518824496017e-01 + -6.278239431715447e-01 + -6.276936329659792e-01 + -6.275609600561086e-01 + -6.274259327786519e-01 + -6.272885594754783e-01 + -6.271488484695774e-01 + -6.270068083043618e-01 + -6.268624473656752e-01 + -6.267157737459064e-01 + -6.265667960682418e-01 + -6.264155229504741e-01 + -6.262619625962428e-01 + -6.261061232036680e-01 + -6.259480131935671e-01 + -6.257876413121770e-01 + -6.256250157840939e-01 + -6.254601447340036e-01 + -6.252930366972961e-01 + -6.251237002207461e-01 + -6.249521437614608e-01 + -6.247783756272571e-01 + -6.246024042025771e-01 + -6.244242378628240e-01 + -6.242438848129136e-01 + -6.240613535991787e-01 + -6.238766528797192e-01 + -6.236897908519111e-01 + -6.235007757894835e-01 + -6.233096161276966e-01 + -6.231163203935827e-01 + -6.229208970207127e-01 + -6.227233543185018e-01 + -6.225237005131427e-01 + -6.223219439896936e-01 + -6.221180932821515e-01 + -6.219121568092019e-01 + -6.217041429518334e-01 + -6.214940600654550e-01 + -6.212819164164759e-01 + -6.210677203231969e-01 + -6.208514802076882e-01 + -6.206332045396580e-01 + -6.204129016762088e-01 + -6.201905798335827e-01 + -6.199662474146647e-01 + -6.197399127976954e-01 + -6.195115842446862e-01 + -6.192812702436963e-01 + -6.190489791767655e-01 + -6.188147191241502e-01 + -6.185784984898128e-01 + -6.183403256742428e-01 + -6.181002088023494e-01 + -6.178581563548247e-01 + -6.176141767575239e-01 + -6.173682779492388e-01 + -6.171204683326561e-01 + -6.168707564823446e-01 + -6.166191506410910e-01 + -6.163656588303806e-01 + -6.161102891648960e-01 + -6.158530502595377e-01 + -6.155939503803246e-01 + -6.153329975648972e-01 + -6.150702000825419e-01 + -6.148055662334609e-01 + -6.145391042808102e-01 + -6.142708224010276e-01 + -6.140007288452335e-01 + -6.137288318637003e-01 + -6.134551394181680e-01 + -6.131796597624336e-01 + -6.129024013716257e-01 + -6.126233721919256e-01 + -6.123425803085120e-01 + -6.120600340492245e-01 + -6.117757414641359e-01 + -6.114897106263320e-01 + -6.112019497520672e-01 + -6.109124671249474e-01 + -6.106212708396386e-01 + -6.103283687668304e-01 + -6.100337690977098e-01 + -6.097374799911853e-01 + -6.094395093910078e-01 + -6.091398653642833e-01 + -6.088385561024087e-01 + -6.085355898356886e-01 + -6.082309742898534e-01 + -6.079247173286892e-01 + -6.076168274335947e-01 + -6.073073125569597e-01 + -6.069961804304546e-01 + -6.066834389911383e-01 + -6.063690964791788e-01 + -6.060531610171422e-01 + -6.057356401425160e-01 + -6.054165419100932e-01 + -6.050958745176729e-01 + -6.047736455801236e-01 + -6.044498630713394e-01 + -6.041245351487873e-01 + -6.037976695799242e-01 + -6.034692741523835e-01 + -6.031393567367498e-01 + -6.028079252312161e-01 + -6.024749874110954e-01 + -6.021405510415967e-01 + -6.018046242357692e-01 + -6.014672148046522e-01 + -6.011283302871711e-01 + -6.007879786053585e-01 + -6.004461675660895e-01 + -6.001029047914931e-01 + -5.997581981158361e-01 + -5.994120553308485e-01 + -5.990644841299763e-01 + -5.987154923482985e-01 + -5.983650876334137e-01 + -5.980132773866208e-01 + -5.976600693793478e-01 + -5.973054714798371e-01 + -5.969494914365839e-01 + -5.965921366499265e-01 + -5.962334146660923e-01 + -5.958733334161985e-01 + -5.955119002909084e-01 + -5.951491227180785e-01 + -5.947850085981863e-01 + -5.944195652847630e-01 + -5.940528001366523e-01 + -5.936847210858313e-01 + -5.933153356463837e-01 + -5.929446511281147e-01 + -5.925726749957585e-01 + -5.921994146776245e-01 + -5.918248776317639e-01 + -5.914490714685531e-01 + -5.910720036664838e-01 + -5.906936815737919e-01 + -5.903141124838686e-01 + -5.899333037986367e-01 + -5.895512629817115e-01 + -5.891679973989596e-01 + -5.887835143677140e-01 + -5.883978211708401e-01 + -5.880109250637769e-01 + -5.876228334774148e-01 + -5.872335538851382e-01 + -5.868430932520374e-01 + -5.864514587624869e-01 + -5.860586578978172e-01 + -5.856646978273519e-01 + -5.852695857497323e-01 + -5.848733289413443e-01 + -5.844759343592291e-01 + -5.840774091773326e-01 + -5.836777608879199e-01 + -5.832769964405170e-01 + -5.828751228494087e-01 + -5.824721474461222e-01 + -5.820680771414760e-01 + -5.816629188869962e-01 + -5.812566799399100e-01 + -5.808493672421570e-01 + -5.804409877894702e-01 + -5.800315488931496e-01 + -5.796210574409676e-01 + -5.792095201878247e-01 + -5.787969440874288e-01 + -5.783833362006023e-01 + -5.779687035684036e-01 + -5.775530530727403e-01 + -5.771363914738108e-01 + -5.767187255974381e-01 + -5.763000625573806e-01 + -5.758804091701334e-01 + -5.754597721176316e-01 + -5.750381583420940e-01 + -5.746155745735496e-01 + -5.741920274506414e-01 + -5.737675239291555e-01 + -5.733420708457294e-01 + -5.729156748671344e-01 + -5.724883426083383e-01 + -5.720600807791615e-01 + -5.716308961653045e-01 + -5.712007954599810e-01 + -5.707697852850102e-01 + -5.703378722272431e-01 + -5.699050629355835e-01 + -5.694713639828742e-01 + -5.690367818720566e-01 + -5.686013232951076e-01 + -5.681649948578290e-01 + -5.677278029894381e-01 + -5.672897541841998e-01 + -5.668508549514105e-01 + -5.664111117923113e-01 + -5.659705312703284e-01 + -5.655291198708162e-01 + -5.650868839171556e-01 + -5.646438297396282e-01 + -5.641999637601537e-01 + -5.637552925169101e-01 + -5.633098223391846e-01 + -5.628635595080524e-01 + -5.624165104013904e-01 + -5.619686812129524e-01 + -5.615200782186582e-01 + -5.610707080464411e-01 + -5.606205767194389e-01 + -5.601696901727680e-01 + -5.597180550627928e-01 + -5.592656777039943e-01 + -5.588125640834879e-01 + -5.583587201544192e-01 + -5.579041522213387e-01 + -5.574488666805684e-01 + -5.569928694014434e-01 + -5.565361664370690e-01 + -5.560787640439252e-01 + -5.556206683493946e-01 + -5.551618853326125e-01 + -5.547024208857518e-01 + -5.542422809960397e-01 + -5.537814718214656e-01 + -5.533199995563840e-01 + -5.528578699065892e-01 + -5.523950887012330e-01 + -5.519316620139326e-01 + -5.514675956847120e-01 + -5.510028956792654e-01 + -5.505375681122012e-01 + -5.500716185256428e-01 + -5.496050526609592e-01 + -5.491378767346244e-01 + -5.486700964364043e-01 + -5.482017173715276e-01 + -5.477327453596050e-01 + -5.472631862648399e-01 + -5.467930458019109e-01 + -5.463223294669347e-01 + -5.458510432287255e-01 + -5.453791929736209e-01 + -5.449067840000759e-01 + -5.444338218966174e-01 + -5.439603123952597e-01 + -5.434862611282417e-01 + -5.430116738240560e-01 + -5.425365561436022e-01 + -5.420609134355956e-01 + -5.415847511352451e-01 + -5.411080748136494e-01 + -5.406308901525899e-01 + -5.401532026526268e-01 + -5.396750177177082e-01 + -5.391963408895222e-01 + -5.387171774649071e-01 + -5.382375326730170e-01 + -5.377574122460893e-01 + -5.372768215524759e-01 + -5.367957656717409e-01 + -5.363142502106338e-01 + -5.358322806031541e-01 + -5.353498619928253e-01 + -5.348669995911720e-01 + -5.343836987381033e-01 + -5.338999648334884e-01 + -5.334158029976483e-01 + -5.329312184607295e-01 + -5.324462166033185e-01 + -5.319608023124877e-01 + -5.314749807283198e-01 + -5.309887574568507e-01 + -5.305021373834015e-01 + -5.300151254489873e-01 + -5.295277270550899e-01 + -5.290399471460621e-01 + -5.285517906644440e-01 + -5.280632628772480e-01 + -5.275743687930865e-01 + -5.270851133485194e-01 + -5.265955015863814e-01 + -5.261055383771222e-01 + -5.256152286457003e-01 + -5.251245776074701e-01 + -5.246335901741112e-01 + -5.241422711040373e-01 + -5.236506252667474e-01 + -5.231586574906982e-01 + -5.226663726352896e-01 + -5.221737757253558e-01 + -5.216808715970676e-01 + -5.211876649797802e-01 + -5.206941607382560e-01 + -5.202003635069621e-01 + -5.197062778404161e-01 + -5.192119087096243e-01 + -5.187172609937223e-01 + -5.182223393579939e-01 + -5.177271482807863e-01 + -5.172316923981860e-01 + -5.167359765148800e-01 + -5.162400053033168e-01 + -5.157437833544083e-01 + -5.152473152222008e-01 + -5.147506055161414e-01 + -5.142536587718484e-01 + -5.137564794429633e-01 + -5.132590721404230e-01 + -5.127614414610688e-01 + -5.122635918987255e-01 + -5.117655278912282e-01 + -5.112672538418732e-01 + -5.107687741570007e-01 + -5.102700934318621e-01 + -5.097712161511054e-01 + -5.092721464952894e-01 + -5.087728887586368e-01 + -5.082734473837237e-01 + -5.077738269224511e-01 + -5.072740315773641e-01 + -5.067740655070005e-01 + -5.062739331645890e-01 + -5.057736389339202e-01 + -5.052731870571522e-01 + -5.047725816027976e-01 + -5.042718267556677e-01 + -5.037709267723895e-01 + -5.032698858620083e-01 + -5.027687082997551e-01 + -5.022673983228978e-01 + -5.017659599138892e-01 + -5.012643972224838e-01 + -5.007627144504877e-01 + -5.002609154317549e-01 + -4.997590043686060e-01 + -4.992569856498065e-01 + -4.987548629163329e-01 + -4.982526401066572e-01 + -4.977503215591422e-01 + -4.972479113357710e-01 + -4.967454132608579e-01 + -4.962428310876830e-01 + -4.957401690448965e-01 + -4.952374311333114e-01 + -4.947346209581790e-01 + -4.942317423902789e-01 + -4.937287995168010e-01 + -4.932257964768885e-01 + -4.927227367709692e-01 + -4.922196239985463e-01 + -4.917164622446805e-01 + -4.912132554897460e-01 + -4.907100074652099e-01 + -4.902067216434476e-01 + -4.897034018910860e-01 + -4.892000521396699e-01 + -4.886966760659724e-01 + -4.881932772071136e-01 + -4.876898592264887e-01 + -4.871864261030248e-01 + -4.866829812293292e-01 + -4.861795280147214e-01 + -4.856760706607693e-01 + -4.851726126266216e-01 + -4.846691571189501e-01 + -4.841657080652781e-01 + -4.836622690168214e-01 + -4.831588433295234e-01 + -4.826554347812969e-01 + -4.821520468729375e-01 + -4.816486828820545e-01 + -4.811453462713459e-01 + -4.806420406166251e-01 + -4.801387695127853e-01 + -4.796355363816449e-01 + -4.791323444815042e-01 + -4.786291970695780e-01 + -4.781260978727254e-01 + -4.776230503252043e-01 + -4.771200574973888e-01 + -4.766171228253449e-01 + -4.761142497486388e-01 + -4.756114415594656e-01 + -4.751087014547341e-01 + -4.746060327035595e-01 + -4.741034386828772e-01 + -4.736009226286348e-01 + -4.730984877717556e-01 + -4.725961373913972e-01 + -4.720938746127822e-01 + -4.715917026117559e-01 + -4.710896247171748e-01 + -4.705876440070008e-01 + -4.700857635192827e-01 + -4.695839864443119e-01 + -4.690823160197067e-01 + -4.685807553687465e-01 + -4.680793073815264e-01 + -4.675779752270572e-01 + -4.670767621237579e-01 + -4.665756710130068e-01 + -4.660747048828000e-01 + -4.655738667469987e-01 + -4.650731595624170e-01 + -4.645725864569897e-01 + -4.640721505529091e-01 + -4.635718546060059e-01 + -4.630717014390969e-01 + -4.625716940204364e-01 + -4.620718354412200e-01 + -4.615721285948061e-01 + -4.610725762550177e-01 + -4.605731814392849e-01 + -4.600739469669758e-01 + -4.595748755034438e-01 + -4.590759700613133e-01 + -4.585772335293120e-01 + -4.580786685902926e-01 + -4.575802780165928e-01 + -4.570820645542124e-01 + -4.565840309327809e-01 + -4.560861801183225e-01 + -4.555885148408127e-01 + -4.550910375242214e-01 + -4.545937511574892e-01 + -4.540966585092119e-01 + -4.535997618400510e-01 + -4.531030641427475e-01 + -4.526065682230286e-01 + -4.521102761991050e-01 + -4.516141909960451e-01 + -4.511183155232459e-01 + -4.506226519976986e-01 + -4.501272029129367e-01 + -4.496319709853875e-01 + -4.491369589904647e-01 + -4.486421693813144e-01 + -4.481476045149588e-01 + -4.476532669406306e-01 + -4.471591591812127e-01 + -4.466652837454304e-01 + -4.461716431800687e-01 + -4.456782399407630e-01 + -4.451850764283579e-01 + -4.446921550770006e-01 + -4.441994783152980e-01 + -4.437070485743053e-01 + -4.432148683148817e-01 + -4.427229399164072e-01 + -4.422312656936392e-01 + -4.417398479943503e-01 + -4.412486891611525e-01 + -4.407577915365379e-01 + -4.402671575060104e-01 + -4.397767893532367e-01 + -4.392866892910770e-01 + -4.387968597340107e-01 + -4.383073029681190e-01 + -4.378180210879689e-01 + -4.373290163087382e-01 + -4.368402909924953e-01 + -4.363518475687816e-01 + -4.358636880897754e-01 + -4.353758146064535e-01 + -4.348882293452391e-01 + -4.344009344375495e-01 + -4.339139321041776e-01 + -4.334272247184532e-01 + -4.329408142509990e-01 + -4.324547026649819e-01 + -4.319688922210647e-01 + -4.314833850702666e-01 + -4.309981832317592e-01 + -4.305132886299618e-01 + -4.300287034407881e-01 + -4.295444298858639e-01 + -4.290604699557538e-01 + -4.285768254794737e-01 + -4.280934983815344e-01 + -4.276104909880938e-01 + -4.271278053251724e-01 + -4.266454431999359e-01 + -4.261634065006114e-01 + -4.256816972235624e-01 + -4.252003173934318e-01 + -4.247192689049444e-01 + -4.242385536499166e-01 + -4.237581735472888e-01 + -4.232781305399061e-01 + -4.227984264773803e-01 + -4.223190631457318e-01 + -4.218400424415704e-01 + -4.213613663598799e-01 + -4.208830368540804e-01 + -4.204050553934426e-01 + -4.199274237890383e-01 + -4.194501442515862e-01 + -4.189732183034799e-01 + -4.184966476302857e-01 + -4.180204343198091e-01 + -4.175445800025869e-01 + -4.170690862458790e-01 + -4.165939547960728e-01 + -4.161191875920802e-01 + -4.156447863485650e-01 + -4.151707524172177e-01 + -4.146970878143395e-01 + -4.142237944432959e-01 + -4.137508735370036e-01 + -4.132783267358912e-01 + -4.128061559031381e-01 + -4.123343628419361e-01 + -4.118629489726368e-01 + -4.113919157251955e-01 + -4.109212649651476e-01 + -4.104509983224866e-01 + -4.099811172669397e-01 + -4.095116233202941e-01 + -4.090425181490535e-01 + -4.085738034100841e-01 + -4.081054804950061e-01 + -4.076375508831032e-01 + -4.071700161671795e-01 + -4.067028779692610e-01 + -4.062361376606066e-01 + -4.057697965353959e-01 + -4.053038563335837e-01 + -4.048383186645677e-01 + -4.043731849141807e-01 + -4.039084564103662e-01 + -4.034441345521896e-01 + -4.029802208230617e-01 + -4.025167167043049e-01 + -4.020536236099443e-01 + -4.015909428779320e-01 + -4.011286758375757e-01 + -4.006668238996902e-01 + -4.002053885607212e-01 + -3.997443712200008e-01 + -3.992837731698375e-01 + -3.988235956310492e-01 + -3.983638399923307e-01 + -3.979045076012857e-01 + -3.974455996644362e-01 + -3.969871175556028e-01 + -3.965290626455606e-01 + -3.960714361699136e-01 + -3.956142393772600e-01 + -3.951574735515524e-01 + -3.947011399985758e-01 + -3.942452398354794e-01 + -3.937897742524872e-01 + -3.933347447660869e-01 + -3.928801524442024e-01 + -3.924259983101159e-01 + -3.919722839283850e-01 + -3.915190103656777e-01 + -3.910661784756895e-01 + -3.906137895655892e-01 + -3.901618450499643e-01 + -3.897103461604648e-01 + -3.892592935529919e-01 + -3.888086884121152e-01 + -3.883585322771145e-01 + -3.879088261485191e-01 + -3.874595710701922e-01 + -3.870107681556456e-01 + -3.865624182556531e-01 + -3.861145225228043e-01 + -3.856670823544472e-01 + -3.852200986473037e-01 + -3.847735723987576e-01 + -3.843275047747315e-01 + -3.838818965026468e-01 + -3.834367486164244e-01 + -3.829920625791138e-01 + -3.825478392535004e-01 + -3.821040794322394e-01 + -3.816607841213338e-01 + -3.812179544462010e-01 + -3.807755914230820e-01 + -3.803336958584798e-01 + -3.798922686597118e-01 + -3.794513108818827e-01 + -3.790108236907996e-01 + -3.785708078449803e-01 + -3.781312640889848e-01 + -3.776921934895628e-01 + -3.772535969260590e-01 + -3.768154752452403e-01 + -3.763778294637203e-01 + -3.759406605249527e-01 + -3.755039692439986e-01 + -3.750677562951532e-01 + -3.746320226741715e-01 + -3.741967693787305e-01 + -3.737619968727901e-01 + -3.733277061470673e-01 + -3.728938983949728e-01 + -3.724605741247380e-01 + -3.720277341192338e-01 + -3.715953793429491e-01 + -3.711635103186269e-01 + -3.707321278978636e-01 + -3.703012331728173e-01 + -3.698708267439747e-01 + -3.694409092657609e-01 + -3.690114815572269e-01 + -3.685825444178367e-01 + -3.681540985405766e-01 + -3.677261445488677e-01 + -3.672986832474148e-01 + -3.668717154245260e-01 + -3.664452417818624e-01 + -3.660192630178876e-01 + -3.655937798167319e-01 + -3.651687928442244e-01 + -3.647443027770453e-01 + -3.643203103057334e-01 + -3.638968161236534e-01 + -3.634738208501973e-01 + -3.630513251204181e-01 + -3.626293296340570e-01 + -3.622078349959966e-01 + -3.617868418102546e-01 + -3.613663507559756e-01 + -3.609463624258993e-01 + -3.605268773947004e-01 + -3.601078962972172e-01 + -3.596894197319259e-01 + -3.592714482724653e-01 + -3.588539824938156e-01 + -3.584370229705016e-01 + -3.580205702765974e-01 + -3.576046249835235e-01 + -3.571891876270012e-01 + -3.567742587321382e-01 + -3.563598388637925e-01 + -3.559459285557617e-01 + -3.555325283179373e-01 + -3.551196386794110e-01 + -3.547072601423791e-01 + -3.542953931967586e-01 + -3.538840383835105e-01 + -3.534731961849680e-01 + -3.530628670336623e-01 + -3.526530514374770e-01 + -3.522437498876279e-01 + -3.518349628366106e-01 + -3.514266907403692e-01 + -3.510189340474827e-01 + -3.506116931966672e-01 + -3.502049686305037e-01 + -3.497987607886671e-01 + -3.493930701015490e-01 + -3.489878969787067e-01 + -3.485832418243141e-01 + -3.481791050461803e-01 + -3.477754870557839e-01 + -3.473723882547367e-01 + -3.469698090253131e-01 + -3.465677497415546e-01 + -3.461662107762984e-01 + -3.457651925045610e-01 + -3.453646952920019e-01 + -3.449647194961470e-01 + -3.445652654674211e-01 + -3.441663335489293e-01 + -3.437679240760669e-01 + -3.433700373751777e-01 + -3.429726737709668e-01 + -3.425758335909973e-01 + -3.421795171685511e-01 + -3.417837248082679e-01 + -3.413884567949557e-01 + -3.409937134173290e-01 + -3.405994949798623e-01 + -3.402058017865278e-01 + -3.398126341027504e-01 + -3.394199921921239e-01 + -3.390278763247643e-01 + -3.386362867748747e-01 + -3.382452237948665e-01 + -3.378546876227794e-01 + -3.374646785249709e-01 + -3.370751967392704e-01 + -3.366862424749215e-01 + -3.362978159833941e-01 + -3.359099174981644e-01 + -3.355225472182573e-01 + -3.351357053498810e-01 + -3.347493921057998e-01 + -3.343636076973308e-01 + -3.339783523006465e-01 + -3.335936261089775e-01 + -3.332094293460974e-01 + -3.328257621547543e-01 + -3.324426246881435e-01 + -3.320600171619409e-01 + -3.316779397280902e-01 + -3.312963925265998e-01 + -3.309153757279716e-01 + -3.305348894653861e-01 + -3.301549338737799e-01 + -3.297755091265357e-01 + -3.293966153413448e-01 + -3.290182526215280e-01 + -3.286404211114115e-01 + -3.282631209285871e-01 + -3.278863521751846e-01 + -3.275101149642331e-01 + -3.271344093953196e-01 + -3.267592355617253e-01 + -3.263845935667566e-01 + -3.260104834866909e-01 + -3.256369053882101e-01 + -3.252638593756906e-01 + -3.248913455268053e-01 + -3.245193638907094e-01 + -3.241479145199382e-01 + -3.237769974872735e-01 + -3.234066128661293e-01 + -3.230367606622389e-01 + -3.226674409186036e-01 + -3.222986537192250e-01 + -3.219303990783447e-01 + -3.215626770146125e-01 + -3.211954875680534e-01 + -3.208288307426270e-01 + -3.204627065455645e-01 + -3.200971150001409e-01 + -3.197320561103191e-01 + -3.193675298795836e-01 + -3.190035363166080e-01 + -3.186400754009213e-01 + -3.182771471114187e-01 + -3.179147514432106e-01 + -3.175528883799994e-01 + -3.171915578921511e-01 + -3.168307599394173e-01 + -3.164704945026028e-01 + -3.161107615528752e-01 + -3.157515610223091e-01 + -3.153928928534279e-01 + -3.150347570035595e-01 + -3.146771534412893e-01 + -3.143200821026652e-01 + -3.139635429048980e-01 + -3.136075357713601e-01 + -3.132520606345795e-01 + -3.128971174301287e-01 + -3.125427060850263e-01 + -3.121888265003789e-01 + -3.118354785699559e-01 + -3.114826622209773e-01 + -3.111303773636806e-01 + -3.107786238863177e-01 + -3.104274016699458e-01 + -3.100767106006651e-01 + -3.097265505731036e-01 + -3.093769214901858e-01 + -3.090278232197667e-01 + -3.086792556057311e-01 + -3.083312185528201e-01 + -3.079837119302323e-01 + -3.076367355606988e-01 + -3.072902893338402e-01 + -3.069443731087555e-01 + -3.065989866918071e-01 + -3.062541299629242e-01 + -3.059098027895394e-01 + -3.055660049841727e-01 + -3.052227363624680e-01 + -3.048799967605036e-01 + -3.045377860349815e-01 + -3.041961039937703e-01 + -3.038549504461246e-01 + -3.035143252370935e-01 + -3.031742281687436e-01 + -3.028346590287212e-01 + -3.024956176228422e-01 + -3.021571037727226e-01 + -3.018191172891879e-01 + -3.014816579447884e-01 + -3.011447255196590e-01 + -3.008083198102758e-01 + -3.004724406299011e-01 + -3.001370877400161e-01 + -2.998022608844934e-01 + -2.994679598576119e-01 + -2.991341844426006e-01 + -2.988009344054493e-01 + -2.984682095047237e-01 + -2.981360094999961e-01 + -2.978043341494513e-01 + -2.974731831997393e-01 + -2.971425563888731e-01 + -2.968124534589884e-01 + -2.964828741880918e-01 + -2.961538183196636e-01 + -2.958252855581221e-01 + -2.954972756291568e-01 + -2.951697882690829e-01 + -2.948428232168150e-01 + -2.945163801997789e-01 + -2.941904589349781e-01 + -2.938650591314966e-01 + -2.935401805013078e-01 + -2.932158227514122e-01 + -2.928919855851135e-01 + -2.925686687341852e-01 + -2.922458719001491e-01 + -2.919235947316832e-01 + -2.916018369508684e-01 + -2.912805982685433e-01 + -2.909598783275820e-01 + -2.906396768390501e-01 + -2.903199935181080e-01 + -2.900008280188033e-01 + -2.896821799916231e-01 + -2.893640491090957e-01 + -2.890464350819901e-01 + -2.887293375718574e-01 + -2.884127562142981e-01 + -2.880966906629407e-01 + -2.877811406014185e-01 + -2.874661057081589e-01 + -2.871515856017081e-01 + -2.868375799224188e-01 + -2.865240883310016e-01 + -2.862111104788733e-01 + -2.858986460098822e-01 + -2.855866945593247e-01 + -2.852752557499685e-01 + -2.849643292069009e-01 + -2.846539145659956e-01 + -2.843440114851778e-01 + -2.840346195818731e-01 + -2.837257384382189e-01 + -2.834173676812886e-01 + -2.831095069534547e-01 + -2.828021558855677e-01 + -2.824953140398479e-01 + -2.821889810170294e-01 + -2.818831564761255e-01 + -2.815778399984394e-01 + -2.812730311718642e-01 + -2.809687296216254e-01 + -2.806649349155874e-01 + -2.803616466281044e-01 + -2.800588643758425e-01 + -2.797565877594176e-01 + -2.794548163592918e-01 + -2.791535497376408e-01 + -2.788527874670974e-01 + -2.785525291296920e-01 + -2.782527743138438e-01 + -2.779535226034689e-01 + -2.776547735631448e-01 + -2.773565267276963e-01 + -2.770587816777879e-01 + -2.767615379983304e-01 + -2.764647952191276e-01 + -2.761685529026718e-01 + -2.758728106207560e-01 + -2.755775679024411e-01 + -2.752828243050798e-01 + -2.749885793975552e-01 + -2.746948327088052e-01 + -2.744015837838452e-01 + -2.741088321749212e-01 + -2.738165773969221e-01 + -2.735248189792139e-01 + -2.732335564721672e-01 + -2.729427894286903e-01 + -2.726525173657923e-01 + -2.723627397775934e-01 + -2.720734562117226e-01 + -2.717846661993187e-01 + -2.714963692431568e-01 + -2.712085648712142e-01 + -2.709212526021956e-01 + -2.706344319386959e-01 + -2.703481024077038e-01 + -2.700622635195931e-01 + -2.697769147573882e-01 + -2.694920556414950e-01 + -2.692076856844960e-01 + -2.689238043678598e-01 + -2.686404111894522e-01 + -2.683575056539274e-01 + -2.680750872618156e-01 + -2.677931554956217e-01 + -2.675117098420171e-01 + -2.672307498062658e-01 + -2.669502748541435e-01 + -2.666702844564393e-01 + -2.663907781339158e-01 + -2.661117553567487e-01 + -2.658332155842312e-01 + -2.655551583220328e-01 + -2.652775830228958e-01 + -2.650004891310402e-01 + -2.647238761635896e-01 + -2.644477435849875e-01 + -2.641720908275026e-01 + -2.638969173638899e-01 + -2.636222226679477e-01 + -2.633480062028563e-01 + -2.630742674151484e-01 + -2.628010057675256e-01 + -2.625282207270022e-01 + -2.622559117157733e-01 + -2.619840781930577e-01 + -2.617127196438671e-01 + -2.614418354712815e-01 + -2.611714251171109e-01 + -2.609014880694051e-01 + -2.606320237300626e-01 + -2.603630315312080e-01 + -2.600945109576975e-01 + -2.598264614192900e-01 + -2.595588823272051e-01 + -2.592917731268140e-01 + -2.590251332515323e-01 + -2.587589621335352e-01 + -2.584932592080277e-01 + -2.582280238899204e-01 + -2.579632555996659e-01 + -2.576989537753064e-01 + -2.574351178135512e-01 + -2.571717471205571e-01 + -2.569088411498421e-01 + -2.566463993032694e-01 + -2.563844209736399e-01 + -2.561229055922761e-01 + -2.558618525664796e-01 + -2.556012613001373e-01 + -2.553411312228936e-01 + -2.550814617199114e-01 + -2.548222521633088e-01 + -2.545635019702024e-01 + -2.543052105636615e-01 + -2.540473773489130e-01 + -2.537900016856341e-01 + -2.535330829668063e-01 + -2.532766206092137e-01 + -2.530206140025920e-01 + -2.527650625373169e-01 + -2.525099656058339e-01 + -2.522553225882911e-01 + -2.520011328519368e-01 + -2.517473957689335e-01 + -2.514941107682424e-01 + -2.512412772287477e-01 + -2.509888944763962e-01 + -2.507369619085763e-01 + -2.504854789235316e-01 + -2.502344448942996e-01 + -2.499838591881081e-01 + -2.497337211675947e-01 + -2.494840301941274e-01 + -2.492347856449565e-01 + -2.489859868896926e-01 + -2.487376332816026e-01 + -2.484897241934407e-01 + -2.482422589923619e-01 + -2.479952370263670e-01 + -2.477486576676466e-01 + -2.475025202765806e-01 + -2.472568241776227e-01 + -2.470115687463755e-01 + -2.467667533575219e-01 + -2.465223773319763e-01 + -2.462784400223504e-01 + -2.460349407926405e-01 + -2.457918789841980e-01 + -2.455492539346369e-01 + -2.453070649885519e-01 + -2.450653115068263e-01 + -2.448239928212929e-01 + -2.445831082525867e-01 + -2.443426571517872e-01 + -2.441026388729581e-01 + -2.438630527595843e-01 + -2.436238981283992e-01 + -2.433851743000419e-01 + -2.431468806082007e-01 + -2.429090164035606e-01 + -2.426715810065056e-01 + -2.424345737214870e-01 + -2.421979939156839e-01 + -2.419618409122474e-01 + -2.417261139893861e-01 + -2.414908125047008e-01 + -2.412559357926221e-01 + -2.410214831428485e-01 + -2.407874538901436e-01 + -2.405538473594189e-01 + -2.403206628448788e-01 + -2.400878996494245e-01 + -2.398555571012901e-01 + -2.396236345489386e-01 + -2.393921312730178e-01 + -2.391610465589564e-01 + -2.389303797400675e-01 + -2.387001301206918e-01 + -2.384702970040557e-01 + -2.382408797142012e-01 + -2.380118775445049e-01 + -2.377832897784107e-01 + -2.375551157141706e-01 + -2.373273546676602e-01 + -2.371000059435911e-01 + -2.368730688080822e-01 + -2.366465425799626e-01 + -2.364204265795865e-01 + -2.361947200488833e-01 + -2.359694222928865e-01 + -2.357445326362645e-01 + -2.355200503199650e-01 + -2.352959746331166e-01 + -2.350723049000333e-01 + -2.348490404054174e-01 + -2.346261804119323e-01 + -2.344037241869703e-01 + -2.341816710532240e-01 + -2.339600202844186e-01 + -2.337387711161095e-01 + -2.335179228570654e-01 + -2.332974748051559e-01 + -2.330774262250223e-01 + -2.328577763695832e-01 + -2.326385245095027e-01 + -2.324196699339893e-01 + -2.322012119071908e-01 + -2.319831496985562e-01 + -2.317654825913850e-01 + -2.315482098501277e-01 + -2.313313307350805e-01 + -2.311148445095759e-01 + -2.308987504304352e-01 + -2.306830477605995e-01 + -2.304677357746768e-01 + -2.302528137313455e-01 + -2.300382808821649e-01 + -2.298241364802842e-01 + -2.296103797849810e-01 + -2.293970100645507e-01 + -2.291840265929282e-01 + -2.289714285853380e-01 + -2.287592152687787e-01 + -2.285473859575702e-01 + -2.283359398767679e-01 + -2.281248762354934e-01 + -2.279141943457151e-01 + -2.277038934347241e-01 + -2.274939727028900e-01 + -2.272844314557521e-01 + -2.270752689260884e-01 + -2.268664843063303e-01 + -2.266580768713468e-01 + -2.264500458690802e-01 + -2.262423905203003e-01 + -2.260351100742427e-01 + -2.258282037663114e-01 + -2.256216708189812e-01 + -2.254155104759721e-01 + -2.252097219699820e-01 + -2.250043045201664e-01 + -2.247992573593203e-01 + -2.245945797283117e-01 + -2.243902708632074e-01 + -2.241863299547213e-01 + -2.239827562297785e-01 + -2.237795489631809e-01 + -2.235767073479459e-01 + -2.233742305913306e-01 + -2.231721179508258e-01 + -2.229703686190215e-01 + -2.227689817967141e-01 + -2.225679567331226e-01 + -2.223672926389324e-01 + -2.221669887137792e-01 + -2.219670441735496e-01 + -2.217674582593449e-01 + -2.215682301863681e-01 + -2.213693591068690e-01 + -2.211708442449674e-01 + -2.209726848493868e-01 + -2.207748801280919e-01 + -2.205774292618204e-01 + -2.203803314334422e-01 + -2.201835858632719e-01 + -2.199871917828554e-01 + -2.197911484006530e-01 + -2.195954548584606e-01 + -2.194001103587555e-01 + -2.192051141421317e-01 + -2.190104654040938e-01 + -2.188161633272981e-01 + -2.186222070944807e-01 + -2.184285959003400e-01 + -2.182353289399314e-01 + -2.180424054040691e-01 + -2.178498244760091e-01 + -2.176575853577229e-01 + -2.174656872563868e-01 + -2.172741293153170e-01 + -2.170829107184905e-01 + -2.168920306960583e-01 + -2.167014883907840e-01 + -2.165112829716549e-01 + -2.163214136638050e-01 + -2.161318796380734e-01 + -2.159426800456078e-01 + -2.157538140457635e-01 + -2.155652808465100e-01 + -2.153770796378261e-01 + -2.151892095617934e-01 + -2.150016698096599e-01 + -2.148144595574814e-01 + -2.146275779264766e-01 + -2.144410241083473e-01 + -2.142547972973645e-01 + -2.140688966286696e-01 + -2.138833212713278e-01 + -2.136980704068839e-01 + -2.135131431962469e-01 + -2.133285387811797e-01 + -2.131442563097479e-01 + -2.129602949644751e-01 + -2.127766538888946e-01 + -2.125933322205115e-01 + -2.124103291538361e-01 + -2.122276438425608e-01 + -2.120452754131807e-01 + -2.118632230196053e-01 + -2.116814858091140e-01 + -2.115000629262062e-01 + -2.113189535366684e-01 + -2.111381567835468e-01 + -2.109576717897184e-01 + -2.107774976942802e-01 + -2.105976336477213e-01 + -2.104180788038142e-01 + -2.102388322993181e-01 + -2.100598932684242e-01 + -2.098812608470688e-01 + -2.097029341689293e-01 + -2.095249123662445e-01 + -2.093471945702010e-01 + -2.091697799107427e-01 + -2.089926675162990e-01 + -2.088158565139439e-01 + -2.086393460309292e-01 + -2.084631351904399e-01 + -2.082872231124364e-01 + -2.081116089361562e-01 + -2.079362917906518e-01 + -2.077612707758083e-01 + -2.075865450013143e-01 + -2.074121135936494e-01 + -2.072379756933602e-01 + -2.070641303864229e-01 + -2.068905767613353e-01 + -2.067173139701657e-01 + -2.065443411403108e-01 + -2.063716573678470e-01 + -2.061992617242399e-01 + -2.060271533191409e-01 + -2.058553312792487e-01 + -2.056837947057731e-01 + -2.055125426973447e-01 + -2.053415743575506e-01 + -2.051708887985432e-01 + -2.050004850901530e-01 + -2.048303622987561e-01 + -2.046605195888223e-01 + -2.044909560501713e-01 + -2.043216707137636e-01 + -2.041526627003059e-01 + -2.039839311053935e-01 + -2.038154749924674e-01 + -2.036472934779214e-01 + -2.034793856435669e-01 + -2.033117505334730e-01 + -2.031443872603400e-01 + -2.029772949053144e-01 + -2.028104725040109e-01 + -2.026439191805672e-01 + -2.024776340229360e-01 + -2.023116160442759e-01 + -2.021458643362796e-01 + -2.019803779931268e-01 + -2.018151560674621e-01 + -2.016501976152565e-01 + -2.014855017030598e-01 + -2.013210674074579e-01 + -2.011568937830132e-01 + -2.009929798869589e-01 + -2.008293247967324e-01 + -2.006659275530022e-01 + -2.005027871933542e-01 + -2.003399027889975e-01 + -2.001772733993469e-01 + -2.000148980648089e-01 + -1.998527758076494e-01 + -1.996909056937942e-01 + -1.995292867991256e-01 + -1.993679181495860e-01 + -1.992067987609143e-01 + -1.990459276660739e-01 + -1.988853039482369e-01 + -1.987249266385540e-01 + -1.985647947339026e-01 + -1.984049072747179e-01 + -1.982452632992336e-01 + -1.980858618372365e-01 + -1.979267019229292e-01 + -1.977677825816225e-01 + -1.976091028269504e-01 + -1.974506616652059e-01 + -1.972924581269546e-01 + -1.971344912595772e-01 + -1.969767600601777e-01 + -1.968192635197257e-01 + -1.966620006473419e-01 + -1.965049704914111e-01 + -1.963481720619957e-01 + -1.961916043159508e-01 + -1.960352662866058e-01 + -1.958791570023236e-01 + -1.957232754408347e-01 + -1.955676205736371e-01 + -1.954121914050787e-01 + -1.952569869811404e-01 + -1.951020062511957e-01 + -1.949472481743376e-01 + -1.947927118000400e-01 + -1.946383960967118e-01 + -1.944843000224015e-01 + -1.943304226010136e-01 + -1.941767627996486e-01 + -1.940233195719773e-01 + -1.938700919198527e-01 + -1.937170788011597e-01 + -1.935642791691659e-01 + -1.934116920404960e-01 + -1.932593163702813e-01 + -1.931071510855982e-01 + -1.929551951739442e-01 + -1.928034476111422e-01 + -1.926519073523142e-01 + -1.925005733427448e-01 + -1.923494445333268e-01 + -1.921985198850023e-01 + -1.920477983694047e-01 + -1.918972789099489e-01 + -1.917469604078139e-01 + -1.915968418647559e-01 + -1.914469222323946e-01 + -1.912972003977130e-01 + -1.911476753153419e-01 + -1.909983459354311e-01 + -1.908492111828614e-01 + -1.907002699990133e-01 + -1.905515213021572e-01 + -1.904029639829450e-01 + -1.902545969863778e-01 + -1.901064192518998e-01 + -1.899584296832387e-01 + -1.898106271909556e-01 + -1.896630106872466e-01 + -1.895155790805884e-01 + -1.893683312769490e-01 + -1.892212661839438e-01 + -1.890743827110565e-01 + -1.889276797429612e-01 + -1.887811561730439e-01 + -1.886348109311961e-01 + -1.884886428944042e-01 + -1.883426509295119e-01 + -1.881968339481026e-01 + -1.880511908244114e-01 + -1.879057204210790e-01 + -1.877604216379617e-01 + -1.876152933660684e-01 + -1.874703344759240e-01 + -1.873255438107255e-01 + -1.871809202288037e-01 + -1.870364626111082e-01 + -1.868921698587295e-01 + -1.867480408184902e-01 + -1.866040743068453e-01 + -1.864602692045282e-01 + -1.863166243684211e-01 + -1.861731386269271e-01 + -1.860298108368848e-01 + -1.858866398430710e-01 + -1.857436244746902e-01 + -1.856007635808946e-01 + -1.854580560069532e-01 + -1.853155005825300e-01 + -1.851730961153894e-01 + -1.850308414342497e-01 + -1.848887353926573e-01 + -1.847467767859671e-01 + -1.846049644137440e-01 + -1.844632971091011e-01 + -1.843217736893654e-01 + -1.841803929497542e-01 + -1.840391536710132e-01 + -1.838980546953601e-01 + -1.837570948428414e-01 + -1.836162728527662e-01 + -1.834755875142904e-01 + -1.833350376404768e-01 + -1.831946220322505e-01 + -1.830543394615971e-01 + -1.829141886951903e-01 + -1.827741685199023e-01 + -1.826342777082210e-01 + -1.824945150204152e-01 + -1.823548792146146e-01 + -1.822153690716325e-01 + -1.820759833652730e-01 + -1.819367208114371e-01 + -1.817975801678411e-01 + -1.816585602158104e-01 + -1.815196596908165e-01 + -1.813808773333636e-01 + -1.812422118861797e-01 + -1.811036620617022e-01 + -1.809652266051679e-01 + -1.808269042761388e-01 + -1.806886937510042e-01 + -1.805505937426575e-01 + -1.804126030135119e-01 + -1.802747202867512e-01 + -1.801369442562599e-01 + -1.799992736029428e-01 + -1.798617070428147e-01 + -1.797242432859927e-01 + -1.795868810229852e-01 + -1.794496189531687e-01 + -1.793124557648680e-01 + -1.791753901302353e-01 + -1.790384207459855e-01 + -1.789015462957270e-01 + -1.787647654308534e-01 + -1.786280768266134e-01 + -1.784914791546937e-01 + -1.783549710639760e-01 + -1.782185512276328e-01 + -1.780822183108111e-01 + -1.779459709418989e-01 + -1.778098077766785e-01 + -1.776737274641330e-01 + -1.775377286080068e-01 + -1.774018098685905e-01 + -1.772659699092733e-01 + -1.771302073149079e-01 + -1.769945207098710e-01 + -1.768589087303928e-01 + -1.767233699571559e-01 + -1.765879030229077e-01 + -1.764525065751527e-01 + -1.763171791703831e-01 + -1.761819193770276e-01 + -1.760467257991546e-01 + -1.759115970764966e-01 + -1.757765317826701e-01 + -1.756415284467932e-01 + -1.755065856823099e-01 + -1.753717020824757e-01 + -1.752368761936218e-01 + -1.751021065398607e-01 + -1.749673916979465e-01 + -1.748327302843573e-01 + -1.746981207950505e-01 + -1.745635617515470e-01 + -1.744290517410085e-01 + -1.742945892830799e-01 + -1.741601728915786e-01 + -1.740258011048612e-01 + -1.738914724532109e-01 + -1.737571854631234e-01 + -1.736229386541751e-01 + -1.734887304969607e-01 + -1.733545594900176e-01 + -1.732204242005106e-01 + -1.730863230847530e-01 + -1.729522545880760e-01 + -1.728182172369972e-01 + -1.726842095169024e-01 + -1.725502298924380e-01 + -1.724162768405715e-01 + -1.722823488297978e-01 + -1.721484443102564e-01 + -1.720145617082366e-01 + -1.718806995062888e-01 + -1.717468561876117e-01 + -1.716130301380271e-01 + -1.714792197950507e-01 + -1.713454236195253e-01 + -1.712116399980747e-01 + -1.710778673588311e-01 + -1.709441041583024e-01 + -1.708103487974718e-01 + -1.706765996579921e-01 + -1.705428551238503e-01 + -1.704091136122197e-01 + -1.702753735243250e-01 + -1.701416332337174e-01 + -1.700078910979654e-01 + -1.698741454939726e-01 + -1.697403948188084e-01 + -1.696066374424440e-01 + -1.694728717024124e-01 + -1.693390959182572e-01 + -1.692053084687103e-01 + -1.690715077127793e-01 + -1.689376919605925e-01 + -1.688038595533757e-01 + -1.686700088249354e-01 + -1.685361380772680e-01 + -1.684022456043037e-01 + -1.682683297240474e-01 + -1.681343887869522e-01 + -1.680004210524648e-01 + -1.678664247764668e-01 + -1.677323982879569e-01 + -1.675983398888628e-01 + -1.674642478421112e-01 + -1.673301203727337e-01 + -1.671959557470875e-01 + -1.670617522514179e-01 + -1.669275081523977e-01 + -1.667932216748827e-01 + -1.666588910302535e-01 + -1.665245144618999e-01 + -1.663900902261115e-01 + -1.662556165573334e-01 + -1.661210916150787e-01 + -1.659865136192500e-01 + -1.658518808205523e-01 + -1.657171913793817e-01 + -1.655824434869752e-01 + -1.654476353589082e-01 + -1.653127651402884e-01 + -1.651778310039089e-01 + -1.650428311475997e-01 + -1.649077636947325e-01 + -1.647726268054878e-01 + -1.646374186795111e-01 + -1.645021374143022e-01 + -1.643667811219276e-01 + -1.642313479645375e-01 + -1.640958360797542e-01 + -1.639602435821208e-01 + -1.638245685678920e-01 + -1.636888091342957e-01 + -1.635529633807896e-01 + -1.634170294056715e-01 + -1.632810052843662e-01 + -1.631448890987297e-01 + -1.630086789505001e-01 + -1.628723728914803e-01 + -1.627359689604526e-01 + -1.625994652175857e-01 + -1.624628597296842e-01 + -1.623261505411777e-01 + -1.621893356498993e-01 + -1.620524130844338e-01 + -1.619153808874262e-01 + -1.617782370829140e-01 + -1.616409796594323e-01 + -1.615036066042981e-01 + -1.613661159511956e-01 + -1.612285056749289e-01 + -1.610907737177844e-01 + -1.609529180644884e-01 + -1.608149367058714e-01 + -1.606768276166446e-01 + -1.605385887249777e-01 + -1.604002179666082e-01 + -1.602617132920370e-01 + -1.601230726507729e-01 + -1.599842939686116e-01 + -1.598453751511857e-01 + -1.597063141136520e-01 + -1.595671087574746e-01 + -1.594277569704320e-01 + -1.592882566576958e-01 + -1.591486057239582e-01 + -1.590088020577491e-01 + -1.588688434990599e-01 + -1.587287278927537e-01 + -1.585884531125515e-01 + -1.584480170419727e-01 + -1.583074175205078e-01 + -1.581666523320080e-01 + -1.580257193426469e-01 + -1.578846164090064e-01 + -1.577433413163997e-01 + -1.576018918624315e-01 + -1.574602658488709e-01 + -1.573184610699215e-01 + -1.571764753211041e-01 + -1.570343063929105e-01 + -1.568919520623428e-01 + -1.567494100904384e-01 + -1.566066782348808e-01 + -1.564637542612438e-01 + -1.563206358983480e-01 + -1.561773208679784e-01 + -1.560338069324230e-01 + -1.558900918260982e-01 + -1.557461732592160e-01 + -1.556020489404925e-01 + -1.554577165574562e-01 + -1.553131737930803e-01 + -1.551684183667380e-01 + -1.550234479797016e-01 + -1.548782603038531e-01 + -1.547328529851514e-01 + -1.545872236759721e-01 + -1.544413700402710e-01 + -1.542952897416497e-01 + -1.541489804232354e-01 + -1.540024397056934e-01 + -1.538556652055174e-01 + -1.537086545589509e-01 + -1.535614054107616e-01 + -1.534139153265976e-01 + -1.532661818894184e-01 + -1.531182027303390e-01 + -1.529699754364569e-01 + -1.528214975805129e-01 + -1.526727667337492e-01 + -1.525237804315604e-01 + -1.523745362382714e-01 + -1.522250317710374e-01 + -1.520752645246755e-01 + -1.519252320010430e-01 + -1.517749318043420e-01 + -1.516243614355856e-01 + -1.514735183783821e-01 + -1.513224001894709e-01 + -1.511710043669709e-01 + -1.510193283881065e-01 + -1.508673697644896e-01 + -1.507151259726486e-01 + -1.505625944726656e-01 + -1.504097727456887e-01 + -1.502566582737337e-01 + -1.501032485283334e-01 + -1.499495409558064e-01 + -1.497955329817901e-01 + -1.496412220312461e-01 + -1.494866055649385e-01 + -1.493316810218445e-01 + -1.491764458131949e-01 + -1.490208973395277e-01 + -1.488650330073115e-01 + -1.487088502321930e-01 + -1.485523464347014e-01 + -1.483955190026080e-01 + -1.482383652905619e-01 + -1.480808826638546e-01 + -1.479230685198502e-01 + -1.477649202713839e-01 + -1.476064352431592e-01 + -1.474476107543291e-01 + -1.472884441602761e-01 + -1.471289328556787e-01 + -1.469690741904771e-01 + -1.468088654425319e-01 + -1.466483039513407e-01 + -1.464873870643751e-01 + -1.463261120992439e-01 + -1.461644763685366e-01 + -1.460024771711030e-01 + -1.458401117873900e-01 + -1.456773775256328e-01 + -1.455142717039091e-01 + -1.453507916253095e-01 + -1.451869345373153e-01 + -1.450226976893540e-01 + -1.448580783981553e-01 + -1.446930739597145e-01 + -1.445276816274984e-01 + -1.443618986028185e-01 + -1.441957221635531e-01 + -1.440291496154044e-01 + -1.438621781832398e-01 + -1.436948050802729e-01 + -1.435270275400491e-01 + -1.433588428498931e-01 + -1.431902482406432e-01 + -1.430212409022274e-01 + -1.428518180756519e-01 + -1.426819769869911e-01 + -1.425117148424586e-01 + -1.423410288662751e-01 + -1.421699162875700e-01 + -1.419983743276740e-01 + -1.418264001727286e-01 + -1.416539910114460e-01 + -1.414811440470098e-01 + -1.413078564852965e-01 + -1.411341255423312e-01 + -1.409599484327010e-01 + -1.407853222965617e-01 + -1.406102443173973e-01 + -1.404347117561954e-01 + -1.402587217519092e-01 + -1.400822714659246e-01 + -1.399053581557436e-01 + -1.397279789611117e-01 + -1.395501310267636e-01 + -1.393718115937786e-01 + -1.391930178402762e-01 + -1.390137469173205e-01 + -1.388339959972602e-01 + -1.386537622953096e-01 + -1.384730430031857e-01 + -1.382918352262208e-01 + -1.381101361784074e-01 + -1.379279430940128e-01 + -1.377452530885734e-01 + -1.375620633453186e-01 + -1.373783710804598e-01 + -1.371941734475252e-01 + -1.370094676273606e-01 + -1.368242508215203e-01 + -1.366385202106050e-01 + -1.364522729969404e-01 + -1.362655063841710e-01 + -1.360782175127518e-01 + -1.358904035964053e-01 + -1.357020618930480e-01 + -1.355131895302153e-01 + -1.353237837054462e-01 + -1.351338416922692e-01 + -1.349433606493001e-01 + -1.347523377766847e-01 + -1.345607703467432e-01 + -1.343686555800670e-01 + -1.341759906744941e-01 + -1.339827728348106e-01 + -1.337889993407819e-01 + -1.335946674413715e-01 + -1.333997743261954e-01 + -1.332043172962129e-01 + -1.330082936252972e-01 + -1.328117004983506e-01 + -1.326145352418741e-01 + -1.324167951779929e-01 + -1.322184775207451e-01 + -1.320195795498869e-01 + -1.318200985870884e-01 + -1.316200319613715e-01 + -1.314193769770063e-01 + -1.312181309395709e-01 + -1.310162911860090e-01 + -1.308138550466099e-01 + -1.306108198680959e-01 + -1.304071830473609e-01 + -1.302029419358534e-01 + -1.299980938748928e-01 + -1.297926362720222e-01 + -1.295865665431496e-01 + -1.293798820977739e-01 + -1.291725803345192e-01 + -1.289646586840201e-01 + -1.287561146028010e-01 + -1.285469455426532e-01 + -1.283371489741077e-01 + -1.281267223855238e-01 + -1.279156632656401e-01 + -1.277039691180485e-01 + -1.274916374652222e-01 + -1.272786658468367e-01 + -1.270650518164191e-01 + -1.268507929437779e-01 + -1.266358868257778e-01 + -1.264203310530910e-01 + -1.262041232080785e-01 + -1.259872609212462e-01 + -1.257697418614985e-01 + -1.255515637216261e-01 + -1.253327241582721e-01 + -1.251132208452764e-01 + -1.248930515084944e-01 + -1.246722139029198e-01 + -1.244507057917270e-01 + -1.242285249387353e-01 + -1.240056691417677e-01 + -1.237821362222557e-01 + -1.235579240163024e-01 + -1.233330303641722e-01 + -1.231074531412701e-01 + -1.228811902909497e-01 + -1.226542397328350e-01 + -1.224265993926709e-01 + -1.221982672554952e-01 + -1.219692413178169e-01 + -1.217395195880366e-01 + -1.215091001037845e-01 + -1.212779809466277e-01 + -1.210461602256936e-01 + -1.208136360512357e-01 + -1.205804065871132e-01 + -1.203464700244819e-01 + -1.201118245085727e-01 + -1.198764682774020e-01 + -1.196403996473917e-01 + -1.194036168896808e-01 + -1.191661183059585e-01 + -1.189279022478277e-01 + -1.186889671039581e-01 + -1.184493112863093e-01 + -1.182089332317922e-01 + -1.179678314226111e-01 + -1.177260043675167e-01 + -1.174834505981953e-01 + -1.172401686820595e-01 + -1.169961572351099e-01 + -1.167514149230550e-01 + -1.165059404117411e-01 + -1.162597324012717e-01 + -1.160127896494247e-01 + -1.157651109471359e-01 + -1.155166951180050e-01 + -1.152675410245315e-01 + -1.150176475647009e-01 + -1.147670136740031e-01 + -1.145156383362403e-01 + -1.142635205942373e-01 + -1.140106595167702e-01 + -1.137570541746248e-01 + -1.135027037005393e-01 + -1.132476072885310e-01 + -1.129917641930528e-01 + -1.127351736954119e-01 + -1.124778350964029e-01 + -1.122197477241451e-01 + -1.119609110122396e-01 + -1.117013244465126e-01 + -1.114409874638995e-01 + -1.111798996306716e-01 + -1.109180606010562e-01 + -1.106554699641756e-01 + -1.103921273860412e-01 + -1.101280326261970e-01 + -1.098631854961093e-01 + -1.095975858385727e-01 + -1.093312335328026e-01 + -1.090641285282797e-01 + -1.087962708201439e-01 + -1.085276604456853e-01 + -1.082582975002112e-01 + -1.079881821487157e-01 + -1.077173146235393e-01 + -1.074456951780328e-01 + -1.071733241222459e-01 + -1.069002018428373e-01 + -1.066263287812705e-01 + -1.063517054311524e-01 + -1.060763323447030e-01 + -1.058002101349244e-01 + -1.055233394662503e-01 + -1.052457210591484e-01 + -1.049673557111256e-01 + -1.046882442747131e-01 + -1.044083876525769e-01 + -1.041277868396362e-01 + -1.038464428925467e-01 + -1.035643569070283e-01 + -1.032815300276830e-01 + -1.029979634724506e-01 + -1.027136585692167e-01 + -1.024286166734134e-01 + -1.021428391952729e-01 + -1.018563276648054e-01 + -1.015690836461885e-01 + -1.012811087487812e-01 + -1.009924046817547e-01 + -1.007029732152899e-01 + -1.004128161832720e-01 + -1.001219355157012e-01 + -9.983033318947014e-02 + -9.953801123789381e-02 + -9.924497181065381e-02 + -9.895121710675164e-02 + -9.865674937610269e-02 + -9.836157098451072e-02 + -9.806568435550449e-02 + -9.776909196067303e-02 + -9.747179635418697e-02 + -9.717380018935569e-02 + -9.687510621844479e-02 + -9.657571724434684e-02 + -9.627563612429342e-02 + -9.597486578738405e-02 + -9.567340928055243e-02 + -9.537126972794066e-02 + -9.506845031433311e-02 + -9.476495428226560e-02 + -9.446078497156121e-02 + -9.415594584893169e-02 + -9.385044042616277e-02 + -9.354427227181475e-02 + -9.323744504129654e-02 + -9.292996250412121e-02 + -9.262182850819620e-02 + -9.231304696021701e-02 + -9.200362185853247e-02 + -9.169355729461479e-02 + -9.138285746240042e-02 + -9.107152658866502e-02 + -9.075956898041553e-02 + -9.044698911371001e-02 + -9.013379148226228e-02 + -8.981998063663986e-02 + -8.950556130228651e-02 + -8.919053824746566e-02 + -8.887491629233073e-02 + -8.855870039130662e-02 + -8.824189556852820e-02 + -8.792450692262105e-02 + -8.760653968435946e-02 + -8.728799913331863e-02 + -8.696889060279582e-02 + -8.664921956673162e-02 + -8.632899159487925e-02 + -8.600821233560418e-02 + -8.568688750540578e-02 + -8.536502290181033e-02 + -8.504262441718942e-02 + -8.471969805751253e-02 + -8.439624991228166e-02 + -8.407228614692197e-02 + -8.374781300392073e-02 + -8.342283682562317e-02 + -8.309736406563741e-02 + -8.277140122021633e-02 + -8.244495489588025e-02 + -8.211803185128563e-02 + -8.179063884910803e-02 + -8.146278271646745e-02 + -8.113447041766966e-02 + -8.080570906396743e-02 + -8.047650581443898e-02 + -8.014686781276095e-02 + -7.981680240792462e-02 + -7.948631704884632e-02 + -7.915541916537581e-02 + -7.882411634928695e-02 + -7.849241630024972e-02 + -7.816032674199919e-02 + -7.782785552351333e-02 + -7.749501058768483e-02 + -7.716179991262426e-02 + -7.682823159405887e-02 + -7.649431382724470e-02 + -7.616005485767063e-02 + -7.582546305035900e-02 + -7.549054686796573e-02 + -7.515531479419155e-02 + -7.481977542018518e-02 + -7.448393744841550e-02 + -7.414780964186224e-02 + -7.381140084542565e-02 + -7.347471999131863e-02 + -7.313777608894645e-02 + -7.280057821330421e-02 + -7.246313551410177e-02 + -7.212545725749871e-02 + -7.178755277686714e-02 + -7.144943146088469e-02 + -7.111110278446059e-02 + -7.077257629326426e-02 + -7.043386160590276e-02 + -7.009496843792977e-02 + -6.975590657859251e-02 + -6.941668588038064e-02 + -6.907731623736572e-02 + -6.873780762436528e-02 + -6.839817013263073e-02 + -6.805841390025939e-02 + -6.771854911509186e-02 + -6.737858603607710e-02 + -6.703853501790490e-02 + -6.669840646610939e-02 + -6.635821080020089e-02 + -6.601795854049251e-02 + -6.567766029345612e-02 + -6.533732672933422e-02 + -6.499696856244534e-02 + -6.465659655384894e-02 + -6.431622152906083e-02 + -6.397585437117004e-02 + -6.363550601749234e-02 + -6.329518746487584e-02 + -6.295490977261127e-02 + -6.261468405232740e-02 + -6.227452144919842e-02 + -6.193443314782195e-02 + -6.159443038613483e-02 + -6.125452449175782e-02 + -6.091472682299209e-02 + -6.057504875634485e-02 + -6.023550170757411e-02 + -5.989609715178513e-02 + -5.955684662483948e-02 + -5.921776169220818e-02 + -5.887885392543541e-02 + -5.854013491232313e-02 + -5.820161634720029e-02 + -5.786330995026137e-02 + -5.752522743033200e-02 + -5.718738052319666e-02 + -5.684978102238282e-02 + -5.651244078415603e-02 + -5.617537160914179e-02 + -5.583858533460867e-02 + -5.550209389764803e-02 + -5.516590918758542e-02 + -5.483004310132478e-02 + -5.449450760284144e-02 + -5.415931467224975e-02 + -5.382447627285256e-02 + -5.349000433169809e-02 + -5.315591089010736e-02 + -5.282220800586091e-02 + -5.248890761304951e-02 + -5.215602173570349e-02 + -5.182356244333215e-02 + -5.149154173522678e-02 + -5.115997163165661e-02 + -5.082886416999516e-02 + -5.049823136495267e-02 + -5.016808523392447e-02 + -4.983843779429676e-02 + -4.950930104255546e-02 + -4.918068697018742e-02 + -4.885260756307724e-02 + -4.852507478656667e-02 + -4.819810059193260e-02 + -4.787169691654435e-02 + -4.754587567726797e-02 + -4.722064876717863e-02 + -4.689602805511645e-02 + -4.657202539266345e-02 + -4.624865260469157e-02 + -4.592592148337583e-02 + -4.560384379218360e-02 + -4.528243126408893e-02 + -4.496169559628109e-02 + -4.464164845065374e-02 + -4.432230145577399e-02 + -4.400366620090135e-02 + -4.368575422918912e-02 + -4.336857704568598e-02 + -4.305214611378011e-02 + -4.273647284209545e-02 + -4.242156859535529e-02 + -4.210744469338823e-02 + -4.179411239819886e-02 + -4.148158291999365e-02 + -4.116986741259326e-02 + -4.085897697584740e-02 + -4.054892265423421e-02 + -4.023971542131075e-02 + -3.993136619382251e-02 + -3.962388583123071e-02 + -3.931728511726146e-02 + -3.901157477126550e-02 + -3.870676544906659e-02 + -3.840286772630143e-02 + -3.809989211312446e-02 + -3.779784905219891e-02 + -3.749674889469364e-02 + -3.719660192057603e-02 + -3.689741833904486e-02 + -3.659920826728452e-02 + -3.630198174507707e-02 + -3.600574873447752e-02 + -3.571051910326670e-02 + -3.541630263528623e-02 + -3.512310902944081e-02 + -3.483094789226868e-02 + -3.453982873906083e-02 + -3.424976098893085e-02 + -3.396075396966352e-02 + -3.367281691959212e-02 + -3.338595897696143e-02 + -3.310018917253339e-02 + -3.281551644593574e-02 + -3.253194963945627e-02 + -3.224949747985180e-02 + -3.196816859917648e-02 + -3.168797153104005e-02 + -3.140891468854379e-02 + -3.113100638362195e-02 + -3.085425482400445e-02 + -3.057866810270934e-02 + -3.030425420554928e-02 + -3.003102099769914e-02 + -2.975897623664901e-02 + -2.948812757423692e-02 + -2.921848253324668e-02 + -2.895004852283206e-02 + -2.868283284398848e-02 + -2.841684266935389e-02 + -2.815208505554401e-02 + -2.788856694668657e-02 + -2.762629515277662e-02 + -2.736527636581735e-02 + -2.710551716378724e-02 + -2.684702399109470e-02 + -2.658980317107453e-02 + -2.633386090723251e-02 + -2.607920326420626e-02 + -2.582583618939347e-02 + -2.557376551170278e-02 + -2.532299691452088e-02 + -2.507353595904179e-02 + -2.482538808499682e-02 + -2.457855859507147e-02 + -2.433305266394129e-02 + -2.408887533435539e-02 + -2.384603151841623e-02 + -2.360452600145211e-02 + -2.336436343201402e-02 + -2.312554832609387e-02 + -2.288808507226996e-02 + -2.265197791904223e-02 + -2.241723098404469e-02 + -2.218384825955722e-02 + -2.195183359578144e-02 + -2.172119070564114e-02 + -2.149192317903017e-02 + -2.126403446983029e-02 + -2.103752788800763e-02 + -2.081240661670658e-02 + -2.058867370373213e-02 + -2.036633206151460e-02 + -2.014538447470562e-02 + -1.992583358220780e-02 + -1.970768189350454e-02 + -1.949093179354288e-02 + -1.927558551890179e-02 + -1.906164517346932e-02 + -1.884911273803516e-02 + -1.863799005561765e-02 + -1.842827883084014e-02 + -1.821998063919436e-02 + -1.801309692347624e-02 + -1.780762899421212e-02 + -1.760357803216378e-02 + -1.740094507853177e-02 + -1.719973104881908e-02 + -1.699993673354589e-02 + -1.680156278321953e-02 + -1.660460972117107e-02 + -1.640907794332531e-02 + -1.621496771146485e-02 + -1.602227916433384e-02 + -1.583101231371427e-02 + -1.564116703862817e-02 + -1.545274309883748e-02 + -1.526574012772238e-02 + -1.508015761914186e-02 + -1.489599495303444e-02 + -1.471325139301425e-02 + -1.453192606709004e-02 + -1.435201798658663e-02 + -1.417352604127694e-02 + -1.399644899327932e-02 + -1.382078549302728e-02 + -1.364653407164367e-02 + -1.347369313289915e-02 + -1.330226097046918e-02 + -1.313223576283522e-02 + -1.296361556501821e-02 + -1.279639832327015e-02 + -1.263058187022890e-02 + -1.246616392163480e-02 + -1.230314208589229e-02 + -1.214151385550256e-02 + -1.198127661271531e-02 + -1.182242763705999e-02 + -1.166496409538715e-02 + -1.150888304589847e-02 + -1.135418144553720e-02 + -1.120085614521435e-02 + -1.104890388968280e-02 + -1.089832132231912e-02 + -1.074910498405016e-02 + -1.060125131793449e-02 + -1.045475666967276e-02 + -1.030961728045836e-02 + -1.016582929759894e-02 + -1.002338877483435e-02 + -9.882291666582974e-03 + -9.742533838318660e-03 + -9.604111064390042e-03 + -9.467019020603698e-03 + -9.331253297647066e-03 + -9.196809399717920e-03 + -9.063682741005771e-03 + -8.931868652392608e-03 + -8.801362376326071e-03 + -8.672159070075210e-03 + -8.544253813557785e-03 + -8.417641603575709e-03 + -8.292317355716863e-03 + -8.168275911736944e-03 + -8.045512032694060e-03 + -7.924020397696789e-03 + -7.803795618017998e-03 + -7.684832236645861e-03 + -7.567124714929550e-03 + -7.450667444285828e-03 + -7.335454748033539e-03 + -7.221480883382279e-03 + -7.108740043647531e-03 + -6.997226348639058e-03 + -6.886933855315346e-03 + -6.777856561420043e-03 + -6.669988397445708e-03 + -6.563323236817328e-03 + -6.457854897474280e-03 + -6.353577131920331e-03 + -6.250483639666226e-03 + -6.148568068731946e-03 + -6.047824005897748e-03 + -5.948244987599076e-03 + -5.849824501347084e-03 + -5.752555978513681e-03 + -5.656432807264865e-03 + -5.561448332218353e-03 + -5.467595840626229e-03 + -5.374868578311233e-03 + -5.283259752176146e-03 + -5.192762521793490e-03 + -5.103370006626977e-03 + -5.015075286629148e-03 + -4.927871401782054e-03 + -4.841751355904269e-03 + -4.756708115524279e-03 + -4.672734611342882e-03 + -4.589823741835811e-03 + -4.507968371897898e-03 + -4.427161334111035e-03 + -4.347395432360992e-03 + -4.268663440821394e-03 + -4.190958105123732e-03 + -4.114272145237943e-03 + -4.038598254543495e-03 + -3.963929102325757e-03 + -3.890257335680627e-03 + -3.817575578228647e-03 + -3.745876433060482e-03 + -3.675152484340320e-03 + -3.605396296371936e-03 + -3.536600416225055e-03 + -3.468757375181376e-03 + -3.401859688514519e-03 + -3.335899857279008e-03 + -3.270870369624132e-03 + -3.206763701552144e-03 + -3.143572317880366e-03 + -3.081288673370167e-03 + -3.019905214426235e-03 + -2.959414379161780e-03 + -2.899808598243805e-03 + -2.841080297295089e-03 + -2.783221897187762e-03 + -2.726225814481816e-03 + -2.670084462721576e-03 + -2.614790253659214e-03 + -2.560335598217736e-03 + -2.506712907773368e-03 + -2.453914594305456e-03 + -2.401933071402304e-03 + -2.350760756079694e-03 + -2.300390068609935e-03 + -2.250813433830373e-03 + -2.202023282926124e-03 + -2.154012052729510e-03 + -2.106772187286276e-03 + -2.060296139752280e-03 + -2.014576371403607e-03 + -1.969605353508076e-03 + -1.925375568654703e-03 + -1.881879509874747e-03 + -1.839109683164610e-03 + -1.797058608044933e-03 + -1.755718816706420e-03 + -1.715082857166331e-03 + -1.675143293280333e-03 + -1.635892703443056e-03 + -1.597323683697497e-03 + -1.559428848712492e-03 + -1.522200830321536e-03 + -1.485632280183934e-03 + -1.449715870092584e-03 + -1.414444291041649e-03 + -1.379810256474448e-03 + -1.345806501791486e-03 + -1.312425783716085e-03 + -1.279660883530597e-03 + -1.247504606026809e-03 + -1.215949779407020e-03 + -1.184989258561609e-03 + -1.154615923413642e-03 + -1.124822679454977e-03 + -1.095602461024569e-03 + -1.066948228352973e-03 + -1.038852968864840e-03 + -1.011309701783447e-03 + -9.843114734261671e-04 + -9.578513585924095e-04 + -9.319224654553773e-04 + -9.065179311555053e-04 + -8.816309237062864e-04 + -8.572546450286300e-04 + -8.333823272254078e-04 + -8.100072355517002e-04 + -7.871226705798197e-04 + -7.647219641139264e-04 + -7.427984826268421e-04 + -7.213456295816899e-04 + -7.003568409622573e-04 + -6.798255887472549e-04 + -6.597453829223955e-04 + -6.401097674232178e-04 + -6.209123240167792e-04 + -6.021466729184949e-04 + -5.838064694784330e-04 + -5.658854088227745e-04 + -5.483772251938055e-04 + -5.312756891491387e-04 + -5.145746125261390e-04 + -4.982678466736966e-04 + -4.823492804447342e-04 + -4.668128456938979e-04 + -4.516525141842837e-04 + -4.368622962674093e-04 + -4.224362466667431e-04 + -4.083684605074392e-04 + -3.946530726767535e-04 + -3.812842634726816e-04 + -3.682562540846601e-04 + -3.555633067280622e-04 + -3.431997298953713e-04 + -3.311598733734528e-04 + -3.194381292167991e-04 + -3.080289365424697e-04 + -2.969267761950205e-04 + -2.861261725181284e-04 + -2.756216974519896e-04 + -2.654079650780054e-04 + -2.554796342192304e-04 + -2.458314117831762e-04 + -2.364580472261255e-04 + -2.273543358783364e-04 + -2.185151214599497e-04 + -2.099352907310207e-04 + -2.016097775283348e-04 + -1.935335642058985e-04 + -1.857016766078902e-04 + -1.781091887621722e-04 + -1.707512232636694e-04 + -1.636229467286454e-04 + -1.567195750846747e-04 + -1.500363727286114e-04 + -1.435686485603325e-04 + -1.373117617901810e-04 + -1.312611200233150e-04 + -1.254121759173047e-04 + -1.197604333131590e-04 + -1.143014443478538e-04 + -1.090308067994204e-04 + -1.039441703359566e-04 + -9.903723276260782e-05 + -9.430573818110019e-05 + -8.974548326970851e-05 + -8.535231263786576e-05 + -8.112211782233505e-05 + -7.705084341917009e-05 + -7.313448187442813e-05 + -6.936907333027644e-05 + -6.575071119249467e-05 + -6.227553658275769e-05 + -5.893973914900695e-05 + -5.573956203853484e-05 + -5.267129595161032e-05 + -4.973128084575291e-05 + -4.691591029739419e-05 + -4.422162539456759e-05 + -4.164491727590835e-05 + -3.918233061143914e-05 + -3.683045757249665e-05 + -3.458594118927672e-05 + -3.244547785270521e-05 + -3.040581149216269e-05 + -2.846373767254703e-05 + -2.661610506551807e-05 + -2.485980997915884e-05 + -2.319180110803831e-05 + -2.160907991664170e-05 + -2.010869565536285e-05 + -1.868775067119526e-05 + -1.734339969669114e-05 + -1.607284546059277e-05 + -1.487334444060605e-05 + -1.374220506933501e-05 + -1.267678404087566e-05 + -1.167449238525668e-05 + -1.073279264625808e-05 + -9.849195952118955e-06 + -9.021268250139948e-06 + -8.246626567968640e-06 + -7.522936912951058e-06 + -6.847920492700095e-06 + -6.219349168072015e-06 + -5.635044236568559e-06 + -5.092882470932048e-06 + -4.590790895044201e-06 + -4.126746487291455e-06 + -3.698781859286729e-06 + -3.304979520120749e-06 + -2.943472506918129e-06 + -2.612449510984778e-06 + -2.310148834845521e-06 + -2.034859929271865e-06 + -1.784927798002038e-06 + -1.558746837246077e-06 + -1.354763241038800e-06 + -1.171478551126456e-06 + -1.007443551831214e-06 + -8.612614901318561e-07 + -7.315906671893518e-07 + -6.171385461507109e-07 + -5.166657190198488e-07 + -4.289874671965282e-07 + -3.529682224298676e-07 + -2.875261969469119e-07 + -2.316338735426513e-07 + -1.843129441850586e-07 + -1.446395043479685e-07 + -1.117434618507405e-07 + -8.480406305106530e-08 + -6.305553831873507e-08 + -4.578544870958251e-08 + -3.233089380928512e-08 + -2.208448022054805e-08 + -1.449165750478591e-08 + -9.047683233008124e-09 + -5.303766739728543e-09 + -2.863504097386866e-09 + -1.380623394721897e-09 + -5.650641971230870e-10 + -1.790372475752245e-10 + -3.511990521548007e-11 + -8.192061809269277e-13 + 0.000000000000000e+00 diff --git a/examples/SPIN/test_problems/validation_nve/in.spin.iron-nve b/examples/SPIN/test_problems/validation_nve/in.spin.iron-nve new file mode 100644 index 0000000000000000000000000000000000000000..f00fb1ec12500b61e3517fd307bf2c7c5dcf7cea --- /dev/null +++ b/examples/SPIN/test_problems/validation_nve/in.spin.iron-nve @@ -0,0 +1,49 @@ +# bcc iron in a 3d periodic box + +clear +units metal +atom_style spin + +dimension 3 +boundary p p p + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice bcc 2.8665 +region box block 0.0 3.0 0.0 3.0 0.0 3.0 +create_box 1 box +create_atoms 1 box + +# setting mass, mag. moments, and interactions for bcc iron + +mass 1 55.845 +set group all spin/random 31 2.2 +velocity all create 100 4928459 rot yes dist gaussian + +pair_style hybrid/overlay eam/alloy spin/exchange 3.5 +pair_coeff * * eam/alloy Fe_Mishin2006.eam.alloy Fe +pair_coeff * * spin/exchange exchange 3.4 0.02726 0.2171 1.841 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 +fix 2 all langevin/spin 0.0 0.00 21 +fix 3 all nve/spin lattice moving +timestep 0.0001 + +# compute and output options + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_tmag temp v_emag ke pe etotal +thermo 200 + +run 100000 diff --git a/examples/SPIN/test_problems/validation_nve/plot_nve.py b/examples/SPIN/test_problems/validation_nve/plot_nve.py new file mode 100755 index 0000000000000000000000000000000000000000..ffa7defaeda2e9599af8da0a80e4413280b470ee --- /dev/null +++ b/examples/SPIN/test_problems/validation_nve/plot_nve.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 + +import numpy as np, pylab, tkinter +import matplotlib.pyplot as plt +from scipy.optimize import curve_fit +from decimal import * +import sys, string, os + + +argv = sys.argv +if len(argv) != 3: + print("Syntax: ./plot_precession.py res_lammps.dat") + sys.exit() + +dirname = os.path.join(os.getcwd(), "Feb_07") +lammps_file = sys.argv[1] +llg_file = sys.argv[2] + +t,tmag,temp,e_mag,e_kin,e_pot,e_tot = np.loadtxt(lammps_file,skiprows=0, usecols=(1,2,3,4,5,6,7),unpack=True) + +fig = plt.figure(figsize=(8,8)) +ax1 = plt.subplot(3,1,1) +ax2 = plt.subplot(3,1,2) +ax3 = plt.subplot(3,1,3) + +ax1.plot(t, e_tot, 'k-', label='Total energy') +ax1.plot(t, e_pot, 'r-', label='Potential energy') +ax1.set_ylabel("E (eV)") +ax1.legend(loc=3) + +ax2.plot(t, e_kin, 'b-', label='Kinetic energy') +ax2.plot(t, e_mag, 'g-', label='Magnetic energy') +ax2.set_ylabel("E (eV)") +ax2.legend(loc=3) + +ax3.plot(t, temp, 'b--', label='Latt. temperature') +ax3.plot(t, tmag, 'r--', label='Spin temperature') +ax3.set_ylabel("T (K)") +ax3.legend(loc=3) + +plt.xlabel('Time (in ps)') +plt.legend() +plt.show() + +fig.savefig(os.path.join(os.getcwd(), "nve_spin_lattice.pdf"), bbox_inches="tight") +plt.close(fig) diff --git a/examples/SPIN/test_problems/validation_nve/run-test-nve.sh b/examples/SPIN/test_problems/validation_nve/run-test-nve.sh new file mode 100755 index 0000000000000000000000000000000000000000..441e7cf46d7c3ceb27c02a8f4bc18afeb0760331 --- /dev/null +++ b/examples/SPIN/test_problems/validation_nve/run-test-nve.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# clean old res +rm res_*.dat + +# compute Lammps +./../../../../src/lmp_serial \ + -in in.spin.iron-nve +in="$(grep -n Step log.lammps | awk -F ':' '{print $1}')" +en="$(grep -n Loop log.lammps | awk -F ':' '{print $1}')" +in="$(echo "$in+1" | bc -l)" +en="$(echo "$en-$in" | bc -l)" +tail -n +$in log.lammps | head -n $en > res_lammps.dat + +# plot results +python3 -m plot_nve.py res_lammps.dat res_llg.dat diff --git a/examples/SPIN/test_problems/validation_nvt/Fe_Mishin2006.eam.alloy b/examples/SPIN/test_problems/validation_nvt/Fe_Mishin2006.eam.alloy new file mode 100644 index 0000000000000000000000000000000000000000..69231bb7eeed5677009482896fe520f4445413dd --- /dev/null +++ b/examples/SPIN/test_problems/validation_nvt/Fe_Mishin2006.eam.alloy @@ -0,0 +1,15009 @@ +comment 1 +comment 2 +Converted by Ganga P Purja Pun using C++ code on Mon Nov 3 10:48:17 2014 +1 Fe +5001 2.400000000000000e-03 5001 1.134673400048920e-03 5.673367000244601e+00 +26 5.584700000000000e+01 2.855300000000000e+00 BCC + 0.000000000000000e+00 + -2.338738741480766e-02 + -4.628214468925276e-02 + -6.912258036387915e-02 + -9.175603963501618e-02 + -1.141497214000000e-01 + -1.363388222824136e-01 + -1.583226111166723e-01 + -1.800965752913192e-01 + -2.016645989796093e-01 + -2.230289901000000e-01 + -2.441907391820846e-01 + -2.651515164004249e-01 + -2.859130554412839e-01 + -3.064769176965011e-01 + -3.268446844000000e-01 + -3.470179531091855e-01 + -3.669982968636285e-01 + -3.867872784635140e-01 + -4.063864535839295e-01 + -4.257973671999999e-01 + -4.450215551034667e-01 + -4.640605423684923e-01 + -4.829158454463851e-01 + -5.015889707095635e-01 + -5.200814146000000e-01 + -5.383946650297390e-01 + -5.565301991603658e-01 + -5.744894857268537e-01 + -5.922739837155686e-01 + -6.098851423000000e-01 + -6.273244022645037e-01 + -6.445931939756846e-01 + -6.616929394780281e-01 + -6.786250511352716e-01 + -6.953909318999999e-01 + -7.119919765952718e-01 + -7.284295696432279e-01 + -7.447050873484842e-01 + -7.608198966551055e-01 + -7.767753551999997e-01 + -7.925728126189590e-01 + -8.082136084644670e-01 + -8.236990743647939e-01 + -8.390305327768360e-01 + -8.542092970000001e-01 + -8.692366724924486e-01 + -8.841139549244775e-01 + -8.988424321942350e-01 + -9.134233831702263e-01 + -9.278580778000000e-01 + -9.421477783833053e-01 + -9.562937376266863e-01 + -9.702972006149804e-01 + -9.841594035897939e-01 + -9.978815741999999e-01 + -1.011464932626877e+00 + -1.024910689374227e+00 + -1.038220047492379e+00 + -1.051394201530088e+00 + -1.064434338000000e+00 + -1.077341636148692e+00 + -1.090117264995124e+00 + -1.102762386491716e+00 + -1.115278154872840e+00 + -1.127665716000000e+00 + -1.139926208397942e+00 + -1.152060761127338e+00 + -1.164070496370367e+00 + -1.175956528594255e+00 + -1.187719964000000e+00 + -1.199361901614307e+00 + -1.210883431375302e+00 + -1.222285636497208e+00 + -1.233569592577916e+00 + -1.244736367000000e+00 + -1.255787020138112e+00 + -1.266722603828963e+00 + -1.277544163022812e+00 + -1.288252734706093e+00 + -1.298849349000000e+00 + -1.309335029515858e+00 + -1.319710789766606e+00 + -1.329977637188551e+00 + -1.340136572985217e+00 + -1.350188590000000e+00 + -1.360134674013246e+00 + -1.369975802739515e+00 + -1.379712947700534e+00 + -1.389347073262624e+00 + -1.398879136000000e+00 + -1.408310086019347e+00 + -1.417640866058629e+00 + -1.426872412303706e+00 + -1.436005653062347e+00 + -1.445041510000000e+00 + -1.453980898788701e+00 + -1.462824726507764e+00 + -1.471573894410746e+00 + -1.480229297407323e+00 + -1.488791823000000e+00 + -1.497262352264216e+00 + -1.505641758273629e+00 + -1.513930908641532e+00 + -1.522130664940848e+00 + -1.530241881000000e+00 + -1.538265404312065e+00 + -1.546202075663511e+00 + -1.554052729949587e+00 + -1.561818194964572e+00 + -1.569499292000000e+00 + -1.577096836740392e+00 + -1.584611637631811e+00 + -1.592044497451960e+00 + -1.599396212413425e+00 + -1.606667572000000e+00 + -1.613859360067553e+00 + -1.620972353850301e+00 + -1.628007324820493e+00 + -1.634965037585305e+00 + -1.641846251000000e+00 + -1.648651718746855e+00 + -1.655382187120556e+00 + -1.662038397074428e+00 + -1.668621083574689e+00 + -1.675130975000000e+00 + -1.681568794320005e+00 + -1.687935258528783e+00 + -1.694231079461267e+00 + -1.700456962682860e+00 + -1.706613607000000e+00 + -1.712701705964896e+00 + -1.718721948142722e+00 + -1.724675016499694e+00 + -1.730561586828580e+00 + -1.736382330000000e+00 + -1.742137912457411e+00 + -1.747828994017656e+00 + -1.753456229026646e+00 + -1.759020265526974e+00 + -1.764521747000000e+00 + -1.769961312537754e+00 + -1.775339594027171e+00 + -1.780657218386706e+00 + -1.785914807192779e+00 + -1.791112977000000e+00 + -1.796252339747382e+00 + -1.801333500673453e+00 + -1.806357060360666e+00 + -1.811323614328498e+00 + -1.816233753000000e+00 + -1.821088062150410e+00 + -1.825887120985276e+00 + -1.830631504346673e+00 + -1.835321782397329e+00 + -1.839958520000000e+00 + -1.844542277375300e+00 + -1.849073608812209e+00 + -1.853553064572349e+00 + -1.857981190434394e+00 + -1.862358526000000e+00 + -1.866685606181999e+00 + -1.870962962067843e+00 + -1.875191119963197e+00 + -1.879370599902195e+00 + -1.883501918000000e+00 + -1.887585586736770e+00 + -1.891622112755978e+00 + -1.895611998485404e+00 + -1.899555741651499e+00 + -1.903453835000000e+00 + -1.907306767129976e+00 + -1.911115021853308e+00 + -1.914879078883785e+00 + -1.918599413067872e+00 + -1.922276495000000e+00 + -1.925910791456493e+00 + -1.929502763824010e+00 + -1.933052869645757e+00 + -1.936561562187136e+00 + -1.940029290000000e+00 + -1.943456497684402e+00 + -1.946843625242175e+00 + -1.950191109032846e+00 + -1.953499381145042e+00 + -1.956768869000000e+00 + -1.959999996258605e+00 + -1.963193182622893e+00 + -1.966348843806269e+00 + -1.969467390558660e+00 + -1.972549230000000e+00 + -1.975594766012145e+00 + -1.978604397775412e+00 + -1.981578520793180e+00 + -1.984517526364655e+00 + -1.987421802000000e+00 + -1.990291731861874e+00 + -1.993127695559656e+00 + -1.995930069364364e+00 + -1.998699225767272e+00 + -2.001435533000000e+00 + -2.004139355858378e+00 + -2.006811055538481e+00 + -2.009450989763890e+00 + -2.012059511961222e+00 + -2.014636972000000e+00 + -2.017183716742461e+00 + -2.019700089231340e+00 + -2.022186429001382e+00 + -2.024643071393187e+00 + -2.027070349000000e+00 + -2.029468591629637e+00 + -2.031838124095820e+00 + -2.034179268100215e+00 + -2.036492342201070e+00 + -2.038777662000000e+00 + -2.041035540156417e+00 + -2.043266284490031e+00 + -2.045470200083453e+00 + -2.047647589335177e+00 + -2.049798751000000e+00 + -2.051923980684690e+00 + -2.054023570214561e+00 + -2.056097808968247e+00 + -2.058146983649728e+00 + -2.060171377000000e+00 + -2.062171268551674e+00 + -2.064146934640815e+00 + -2.066098649159696e+00 + -2.068026683134947e+00 + -2.069931304000000e+00 + -2.071812776276246e+00 + -2.073671361471072e+00 + -2.075507318519566e+00 + -2.077320903301032e+00 + -2.079112369000000e+00 + -2.080881966139512e+00 + -2.082629940879466e+00 + -2.084356537486872e+00 + -2.086061998635378e+00 + -2.087746563000000e+00 + -2.089410466229110e+00 + -2.091053941828707e+00 + -2.092677220771137e+00 + -2.094280530723314e+00 + -2.095864097000000e+00 + -2.097428142794787e+00 + -2.098972888139794e+00 + -2.100498550462023e+00 + -2.102005344216306e+00 + -2.103493482000000e+00 + -2.104963174393948e+00 + -2.106414628045970e+00 + -2.107848047779690e+00 + -2.109263636820768e+00 + -2.110661595000000e+00 + -2.112042119471926e+00 + -2.113405405101743e+00 + -2.114751645122076e+00 + -2.116081030815763e+00 + -2.117393750000000e+00 + -2.118689987937912e+00 + -2.119969928191390e+00 + -2.121233752504371e+00 + -2.122481640223532e+00 + -2.123713768000000e+00 + -2.124930310308645e+00 + -2.126131439345980e+00 + -2.127317325650607e+00 + -2.128488137848567e+00 + -2.129644042000000e+00 + -2.130785202020548e+00 + -2.131911779449632e+00 + -2.133023934312029e+00 + -2.134121824962966e+00 + -2.135205607000000e+00 + -2.136275433910872e+00 + -2.137331457489147e+00 + -2.138373827747866e+00 + -2.139402692469194e+00 + -2.140418197999999e+00 + -2.141420489142081e+00 + -2.142409707722756e+00 + -2.143385994030927e+00 + -2.144349486926974e+00 + -2.145300323000000e+00 + -2.146238637076898e+00 + -2.147164562479522e+00 + -2.148078231051947e+00 + -2.148979772720135e+00 + -2.149869315000000e+00 + -2.150746983835218e+00 + -2.151612904662093e+00 + -2.152467200947815e+00 + -2.153309993218032e+00 + -2.154141401000000e+00 + -2.154961542883784e+00 + -2.155770535619624e+00 + -2.156568494253713e+00 + -2.157355531798206e+00 + -2.158131760000000e+00 + -2.158897289576013e+00 + -2.159652229917454e+00 + -2.160396688379106e+00 + -2.161130769707277e+00 + -2.161854579000000e+00 + -2.162568220747540e+00 + -2.163271795520776e+00 + -2.163965403104795e+00 + -2.164649143391419e+00 + -2.165323114000000e+00 + -2.165987410839904e+00 + -2.166642128874188e+00 + -2.167287362019562e+00 + -2.167923202786971e+00 + -2.168549742000000e+00 + -2.169167069289456e+00 + -2.169775273557456e+00 + -2.170374442741976e+00 + -2.170964663390078e+00 + -2.171546020000000e+00 + -2.172118595836383e+00 + -2.172682474202094e+00 + -2.173237737093693e+00 + -2.173784464448991e+00 + -2.174322736000000e+00 + -2.174852630794634e+00 + -2.175374225221640e+00 + -2.175887595177988e+00 + -2.176392816678372e+00 + -2.176889964000000e+00 + -2.177379110090651e+00 + -2.177860327096639e+00 + -2.178333686466943e+00 + -2.178799258747679e+00 + -2.179257113000000e+00 + -2.179707317299531e+00 + -2.180149939409128e+00 + -2.180585046165010e+00 + -2.181012703042297e+00 + -2.181432975000000e+00 + -2.181845926509731e+00 + -2.182251621108717e+00 + -2.182650121032542e+00 + -2.183041486916871e+00 + -2.183425780000000e+00 + -2.183803061402426e+00 + -2.184173389857596e+00 + -2.184536823542827e+00 + -2.184893420600431e+00 + -2.185243238000000e+00 + -2.185586332004080e+00 + -2.185922759056265e+00 + -2.186252574500474e+00 + -2.186575831916728e+00 + -2.186892585000000e+00 + -2.187202887393400e+00 + -2.187506791832633e+00 + -2.187804350256584e+00 + -2.188095613705712e+00 + -2.188380633000000e+00 + -2.188659458643119e+00 + -2.188932140238827e+00 + -2.189198726845117e+00 + -2.189459267002233e+00 + -2.189713809000000e+00 + -2.189962400732430e+00 + -2.190205089044370e+00 + -2.190441920581333e+00 + -2.190672942002169e+00 + -2.190898199000000e+00 + -2.191117736662499e+00 + -2.191331600149922e+00 + -2.191539834109547e+00 + -2.191742482380937e+00 + -2.191939589000000e+00 + -2.192131197889608e+00 + -2.192317351831708e+00 + -2.192498093290244e+00 + -2.192673464653123e+00 + -2.192843508000000e+00 + -2.193008265149897e+00 + -2.193167777657277e+00 + -2.193322086821774e+00 + -2.193471233640668e+00 + -2.193615259000000e+00 + -2.193754203483761e+00 + -2.193888106693908e+00 + -2.194017008313546e+00 + -2.194140948496115e+00 + -2.194259967000000e+00 + -2.194374103230534e+00 + -2.194483396307522e+00 + -2.194587885225862e+00 + -2.194687608881331e+00 + -2.194782606000000e+00 + -2.194872915200472e+00 + -2.194958575082935e+00 + -2.195039623893670e+00 + -2.195116099394696e+00 + -2.195188040000000e+00 + -2.195255484352944e+00 + -2.195318470219860e+00 + -2.195377035223315e+00 + -2.195431217130485e+00 + -2.195481054000000e+00 + -2.195526583926446e+00 + -2.195567844428364e+00 + -2.195604873029958e+00 + -2.195637707516086e+00 + -2.195666386000000e+00 + -2.195690946583878e+00 + -2.195711426545218e+00 + -2.195727863658885e+00 + -2.195740296673043e+00 + -2.195748763000000e+00 + -2.195753299607926e+00 + -2.195753945356280e+00 + -2.195750739132009e+00 + -2.195743719093693e+00 + -2.195732923000000e+00 + -2.195718388771941e+00 + -2.195700155840199e+00 + -2.195678263244660e+00 + -2.195652748934943e+00 + -2.195623652000000e+00 + -2.195591012099144e+00 + -2.195554867993407e+00 + -2.195515258985693e+00 + -2.195472225489285e+00 + -2.195425807000000e+00 + -2.195376042760528e+00 + -2.195322973614639e+00 + -2.195266640267298e+00 + -2.195207082686241e+00 + -2.195144342000000e+00 + -2.195078459809379e+00 + -2.195009476453377e+00 + -2.194937433066797e+00 + -2.194862372466152e+00 + -2.194784337000000e+00 + -2.194703368615792e+00 + -2.194619509257747e+00 + -2.194532801771278e+00 + -2.194443290305841e+00 + -2.194351018000000e+00 + -2.194256027737437e+00 + -2.194158364279782e+00 + -2.194058072250755e+00 + -2.193955195446114e+00 + -2.193849779000000e+00 + -2.193741868885436e+00 + -2.193631510785602e+00 + -2.193518750557477e+00 + -2.193403634464252e+00 + -2.193286209000000e+00 + -2.193166521091908e+00 + -2.193044618884907e+00 + -2.192920550384463e+00 + -2.192794362988794e+00 + -2.192666105000000e+00 + -2.192535825071375e+00 + -2.192403570783236e+00 + -2.192269389306454e+00 + -2.192133327558461e+00 + -2.191995432000000e+00 + -2.191855748732582e+00 + -2.191714323538803e+00 + -2.191571201647935e+00 + -2.191426427571225e+00 + -2.191280046000000e+00 + -2.191132101498374e+00 + -2.190982637495000e+00 + -2.190831697104933e+00 + -2.190679323369808e+00 + -2.190525559000000e+00 + -2.190370446239990e+00 + -2.190214026313205e+00 + -2.190056340263544e+00 + -2.189897429222788e+00 + -2.189737334000000e+00 + -2.189576094894156e+00 + -2.189413751026334e+00 + -2.189250341662785e+00 + -2.189085906605225e+00 + -2.188920484000000e+00 + -2.188754111167896e+00 + -2.188586826517446e+00 + -2.188418667831656e+00 + -2.188249671505532e+00 + -2.188079874000000e+00 + -2.187909311655257e+00 + -2.187738019850499e+00 + -2.187566033827133e+00 + -2.187393388939138e+00 + -2.187220120000000e+00 + -2.187046261198758e+00 + -2.186871845656608e+00 + -2.186696906557450e+00 + -2.186521477499284e+00 + -2.186345591000000e+00 + -2.186169278929348e+00 + -2.185992573469870e+00 + -2.185815507881449e+00 + -2.185638116930091e+00 + -2.185460435000000e+00 + -2.185282496723147e+00 + -2.185104339046128e+00 + -2.184925999471230e+00 + -2.184747515559985e+00 + -2.184568926000000e+00 + -2.184390270419004e+00 + -2.184211589307456e+00 + -2.184032923638580e+00 + -2.183854314918697e+00 + -2.183675806000000e+00 + -2.183497440664645e+00 + -2.183319263062145e+00 + -2.183141318226586e+00 + -2.182963652427783e+00 + -2.182786312000000e+00 + -2.182609343803448e+00 + -2.182432796909005e+00 + -2.182256720850984e+00 + -2.182081165100584e+00 + -2.181906180000000e+00 + -2.181731816772078e+00 + -2.181558127901617e+00 + -2.181385166690271e+00 + -2.181212987245827e+00 + -2.181041644000000e+00 + -2.180871191888493e+00 + -2.180701687214938e+00 + -2.180533187286951e+00 + -2.180365750436408e+00 + -2.180199435000000e+00 + -2.180034299706266e+00 + -2.179870405073628e+00 + -2.179707812476292e+00 + -2.179546583956951e+00 + -2.179386782000000e+00 + -2.179228469569652e+00 + -2.179071710572415e+00 + -2.178916570209873e+00 + -2.178763115274171e+00 + -2.178611412000000e+00 + -2.178461526719817e+00 + -2.178313527960817e+00 + -2.178167485170301e+00 + -2.178023468394170e+00 + -2.177881548000000e+00 + -2.177741794911888e+00 + -2.177604281585152e+00 + -2.177469081134519e+00 + -2.177336267165003e+00 + -2.177205914000000e+00 + -2.177078096781969e+00 + -2.176952892179310e+00 + -2.176830377266735e+00 + -2.176710629289710e+00 + -2.176593727000000e+00 + -2.176479750089725e+00 + -2.176368778170715e+00 + -2.176260892035270e+00 + -2.176156174290115e+00 + -2.176054707000000e+00 + -2.175956572480315e+00 + -2.175861855847520e+00 + -2.175770642395214e+00 + -2.175683016800304e+00 + -2.175599066000000e+00 + -2.175518878125867e+00 + -2.175442540126260e+00 + -2.175370140365612e+00 + -2.175301769822078e+00 + -2.175237519000000e+00 + -2.175177478422601e+00 + -2.175121740498959e+00 + -2.175070398200953e+00 + -2.175023544771637e+00 + -2.174981275000000e+00 + -2.174943684570890e+00 + -2.174910868831887e+00 + -2.174882924320826e+00 + -2.174859949548785e+00 + -2.174842043000000e+00 + -2.174829303331104e+00 + -2.174821830410927e+00 + -2.174819725228089e+00 + -2.174823090027552e+00 + -2.174832027000000e+00 + -2.174846638699704e+00 + -2.174867029559900e+00 + -2.174893304738549e+00 + -2.174925569834485e+00 + -2.174963931000000e+00 + -2.175008495084293e+00 + -2.175059370394289e+00 + -2.175116665887380e+00 + -2.175180491027200e+00 + -2.175250956000000e+00 + -2.175328171805043e+00 + -2.175412250937077e+00 + -2.175503306357684e+00 + -2.175601451271103e+00 + -2.175706800000000e+00 + -2.175819467765500e+00 + -2.175939570549688e+00 + -2.176067225041086e+00 + -2.176202548787411e+00 + -2.176345660000000e+00 + -2.176496677644949e+00 + -2.176655722183164e+00 + -2.176822914601843e+00 + -2.176998376191533e+00 + -2.177182229000000e+00 + -2.177374595921169e+00 + -2.177575601341799e+00 + -2.177785370087074e+00 + -2.178004027196875e+00 + -2.178231699000000e+00 + -2.178468512862275e+00 + -2.178714597011325e+00 + -2.178970080062299e+00 + -2.179235091002944e+00 + -2.179509760000000e+00 + -2.179794218272507e+00 + -2.180088598277346e+00 + -2.180393032665419e+00 + -2.180707654074028e+00 + -2.181032597000000e+00 + -2.181367997183408e+00 + -2.181713990526695e+00 + -2.182070713375897e+00 + -2.182438302803596e+00 + -2.182816897000000e+00 + -2.183206635150288e+00 + -2.183607657686531e+00 + -2.184020105275964e+00 + -2.184444118610974e+00 + -2.184879840000000e+00 + -2.185327412889597e+00 + -2.185786981109685e+00 + -2.186258689137973e+00 + -2.186742682377725e+00 + -2.187239107000000e+00 + -2.187748109862624e+00 + -2.188269838801850e+00 + -2.188804442677439e+00 + -2.189352071523296e+00 + -2.189912875000000e+00 + -2.190487003093955e+00 + -2.191074608322116e+00 + -2.191675843787203e+00 + -2.192290862610885e+00 + -2.192919819000000e+00 + -2.193562867915730e+00 + -2.194220164552147e+00 + -2.194891865404003e+00 + -2.195578128862665e+00 + -2.196279113000000e+00 + -2.196994976008069e+00 + -2.197725877814248e+00 + -2.198471979076637e+00 + -2.199233441010543e+00 + -2.200010426000000e+00 + -2.200803097224683e+00 + -2.201611618154960e+00 + -2.202436152922896e+00 + -2.203276866629103e+00 + -2.204133924999999e+00 + -2.205007494076377e+00 + -2.205897739759803e+00 + -2.206804828380775e+00 + -2.207728927012060e+00 + -2.208670203000000e+00 + -2.209628824019662e+00 + -2.210604958519273e+00 + -2.211598775212799e+00 + -2.212610442959744e+00 + -2.213640131000000e+00 + -2.214688008997179e+00 + -2.215754247358006e+00 + -2.216839016713686e+00 + -2.217942487809252e+00 + -2.219064832000000e+00 + -2.220206221175054e+00 + -2.221366827800435e+00 + -2.222546824259503e+00 + -2.223746382727115e+00 + -2.224965677000000e+00 + -2.226204881636015e+00 + -2.227464170011377e+00 + -2.228743716360184e+00 + -2.230043696636770e+00 + -2.231364286000000e+00 + -2.232705659360148e+00 + -2.234067993079961e+00 + -2.235451464138931e+00 + -2.236856249856844e+00 + -2.238282527000000e+00 + -2.239730472524265e+00 + -2.241200265638654e+00 + -2.242692085287406e+00 + -2.244206109271373e+00 + -2.245742517000000e+00 + -2.247301488882056e+00 + -2.248883204910052e+00 + -2.250487845315945e+00 + -2.252115590911750e+00 + -2.253766623000000e+00 + -2.255441123360801e+00 + -2.257139274542631e+00 + -2.258861259219423e+00 + -2.260607260005651e+00 + -2.262377460000000e+00 + -2.264172042724291e+00 + -2.265991192137124e+00 + -2.267835092970031e+00 + -2.269703930922391e+00 + -2.271597891000000e+00 + -2.273517158096056e+00 + -2.275461918662951e+00 + -2.277432359442962e+00 + -2.279428667085979e+00 + -2.281451029000000e+00 + -2.283499633104788e+00 + -2.285574667389452e+00 + -2.287676320195930e+00 + -2.289804780398682e+00 + -2.291960237000000e+00 + -2.294142879454985e+00 + -2.296352898805551e+00 + -2.298590485768695e+00 + -2.300855830078550e+00 + -2.303149122999999e+00 + -2.305470556795997e+00 + -2.307820323545887e+00 + -2.310198615647936e+00 + -2.312605626084681e+00 + -2.315041548000000e+00 + -2.317506574745174e+00 + -2.320000900224320e+00 + -2.322524719122656e+00 + -2.325078227061991e+00 + -2.327661619000000e+00 + -2.330275089774933e+00 + -2.332918835677479e+00 + -2.335593053485966e+00 + -2.338297940187764e+00 + -2.341033693000000e+00 + -2.343800509406319e+00 + -2.346598587410773e+00 + -2.349428125303093e+00 + -2.352289321686241e+00 + -2.355182376000000e+00 + -2.358107488272472e+00 + -2.361064858786305e+00 + -2.364054687769198e+00 + -2.367077175375671e+00 + -2.370132523000000e+00 + -2.373220932737141e+00 + -2.376342606269360e+00 + -2.379497745810629e+00 + -2.382686554578887e+00 + -2.385909236000000e+00 + -2.389165993580208e+00 + -2.392457030846979e+00 + -2.395782552198058e+00 + -2.399142763298191e+00 + -2.402537869000000e+00 + -2.405968074028050e+00 + -2.409433584975734e+00 + -2.412934608682329e+00 + -2.416471351683178e+00 + -2.420044020999999e+00 + -2.423652824137149e+00 + -2.427297969206864e+00 + -2.430979664728804e+00 + -2.434698119632543e+00 + -2.438453543000000e+00 + -2.442246144168148e+00 + -2.446076133195298e+00 + -2.449943720571852e+00 + -2.453849117184546e+00 + -2.457792534000000e+00 + -2.461774182279110e+00 + -2.465794274341647e+00 + -2.469853022790889e+00 + -2.473950640299041e+00 + -2.478087340000000e+00 + -2.482263335430129e+00 + -2.486478840559030e+00 + -2.490734069661931e+00 + -2.495029237377275e+00 + -2.499364559000000e+00 + -2.503740250274207e+00 + -2.508156527124766e+00 + -2.512613605692700e+00 + -2.517111702474631e+00 + -2.521651035000000e+00 + -2.526231821279088e+00 + -2.530854278665131e+00 + -2.535518625205626e+00 + -2.540225080261870e+00 + -2.544973863000000e+00 + -2.549765192573217e+00 + -2.554599288888442e+00 + -2.559476372494764e+00 + -2.564396664604125e+00 + -2.569360386000000e+00 + -2.574367757513646e+00 + -2.579419001423863e+00 + -2.584514340338642e+00 + -2.589653996851589e+00 + -2.594838193999999e+00 + -2.600067155253186e+00 + -2.605341104662645e+00 + -2.610660266783872e+00 + -2.616024866689215e+00 + -2.621435129000000e+00 + -2.626891278432673e+00 + -2.632393541366548e+00 + -2.637942144571998e+00 + -2.643537314797249e+00 + -2.649179279000000e+00 + -2.654868264471787e+00 + -2.660604499292826e+00 + -2.666388211776204e+00 + -2.672219630341834e+00 + -2.678098984000000e+00 + -2.684026502306392e+00 + -2.690002415494505e+00 + -2.696026954017429e+00 + -2.702100348331385e+00 + -2.708222828000000e+00 + -2.714394620651795e+00 + -2.720615948464916e+00 + -2.726887030965477e+00 + -2.733208085290139e+00 + -2.739579324000000e+00 + -2.746000955816228e+00 + -2.752473185385865e+00 + -2.758996214641690e+00 + -2.765570242469367e+00 + -2.772195463000000e+00 + -2.778872066776275e+00 + -2.785600241574056e+00 + -2.792380171842594e+00 + -2.799212037606767e+00 + -2.806096015000000e+00 + -2.813032277129447e+00 + -2.820020994035244e+00 + -2.827062331911353e+00 + -2.834156451970738e+00 + -2.841303512999999e+00 + -2.848503671252426e+00 + -2.855757077985244e+00 + -2.863063881095023e+00 + -2.870424224890850e+00 + -2.877838249999999e+00 + -2.885306093928394e+00 + -2.892827890388094e+00 + -2.900403769651381e+00 + -2.908033857857018e+00 + -2.915718277999999e+00 + -2.923457150039019e+00 + -2.931250588977881e+00 + -2.939098706889091e+00 + -2.947001612872150e+00 + -2.954959411999998e+00 + -2.962972205810575e+00 + -2.971040091434060e+00 + -2.979163163083816e+00 + -2.987341511831155e+00 + -2.995575224999998e+00 + -3.003864386499012e+00 + -3.012209075546996e+00 + -3.020609368336417e+00 + -3.029065337835770e+00 + -3.037577052999999e+00 + -3.046144579520789e+00 + -3.054767979666142e+00 + -3.063447312238444e+00 + -3.072182631692820e+00 + -3.080973988999999e+00 + -3.089821432099915e+00 + -3.098725004829577e+00 + -3.107684748048404e+00 + -3.116700699185534e+00 + -3.125772890999999e+00 + -3.134901352715559e+00 + -3.144086110699870e+00 + -3.153327188100508e+00 + -3.162624603843408e+00 + -3.171978372999998e+00 + -3.181388507546933e+00 + -3.190855016073102e+00 + -3.200377903469057e+00 + -3.209957169965227e+00 + -3.219592812999998e+00 + -3.229284827363896e+00 + -3.239033203402719e+00 + -3.248837927724427e+00 + -3.258698982652799e+00 + -3.268616347999998e+00 + -3.278590000914418e+00 + -3.288619913401403e+00 + -3.298706054004673e+00 + -3.308848387625167e+00 + -3.319046875999998e+00 + -3.329301477934322e+00 + -3.339612147844228e+00 + -3.349978836592516e+00 + -3.360401490990458e+00 + -3.370880054999998e+00 + -3.381414469596773e+00 + -3.392004670266620e+00 + -3.402650589976440e+00 + -3.413352159446950e+00 + -3.424109303999998e+00 + -3.434921945057392e+00 + -3.445790001867386e+00 + -3.456713390096515e+00 + -3.467692020464062e+00 + -3.478725800999998e+00 + -3.489814636890736e+00 + -3.500958427789171e+00 + -3.512157070764700e+00 + -3.523410460589800e+00 + -3.534718486999998e+00 + -3.546081035880271e+00 + -3.557497990199610e+00 + -3.568969229634993e+00 + -3.580494629586285e+00 + -3.592074061999998e+00 + -3.603707395759104e+00 + -3.615394495446970e+00 + -3.627135222559569e+00 + -3.638929435115233e+00 + -3.650776987000000e+00 + -3.662677728742245e+00 + -3.674631507325805e+00 + -3.686638166579447e+00 + -3.698697546469576e+00 + -3.710809483000000e+00 + -3.722973808828209e+00 + -3.735190352595157e+00 + -3.747458940125227e+00 + -3.759779394010754e+00 + -3.772151532000000e+00 + -3.784575168342916e+00 + -3.797050115062026e+00 + -3.809576180292756e+00 + -3.822153166803174e+00 + -3.834780875000000e+00 + -3.847459102820189e+00 + -3.860187643203489e+00 + -3.872966285667494e+00 + -3.885794816080159e+00 + -3.898673017000000e+00 + -3.911600667911320e+00 + -3.924577543724543e+00 + -3.937603416269590e+00 + -3.950678053996523e+00 + -3.963801221000000e+00 + -3.976972678103186e+00 + -3.990192183512261e+00 + -4.003459491521890e+00 + -4.016774351164608e+00 + -4.030136509000000e+00 + -4.043545709147031e+00 + -4.057001691149858e+00 + -4.070504190840202e+00 + -4.084052939835804e+00 + -4.097647667000000e+00 + -4.111288098419088e+00 + -4.124973955231095e+00 + -4.138704955371648e+00 + -4.152480813397504e+00 + -4.166301240000000e+00 + -4.180165942572286e+00 + -4.194074624537170e+00 + -4.208026986370140e+00 + -4.222022725101974e+00 + -4.236061533000000e+00 + -4.250143098846027e+00 + -4.264267108996074e+00 + -4.278433246261039e+00 + -4.292641188627767e+00 + -4.306890611000000e+00 + -4.321181185386233e+00 + -4.335512579090955e+00 + -4.349884456391911e+00 + -4.364296478350024e+00 + -4.378748302000000e+00 + -4.393239580901771e+00 + -4.407769964394226e+00 + -4.422339098999587e+00 + -4.436946628138934e+00 + -4.451592191000000e+00 + -4.466275423153566e+00 + -4.480995955945297e+00 + -4.495753418053649e+00 + -4.510547435209278e+00 + -4.525377628000000e+00 + -4.540243613376725e+00 + -4.555145006346341e+00 + -4.570081418154680e+00 + -4.585052454714511e+00 + -4.600057719000000e+00 + -4.615096811360685e+00 + -4.630169328028963e+00 + -4.645274861878902e+00 + -4.660413001844071e+00 + -4.675583333000000e+00 + -4.690785437259527e+00 + -4.706018892955582e+00 + -4.721283275134317e+00 + -4.736578154793736e+00 + -4.751903099000000e+00 + -4.767257671719769e+00 + -4.782641433752779e+00 + -4.798053942304498e+00 + -4.813494749953581e+00 + -4.828963406000000e+00 + -4.844459456896403e+00 + -4.859982445123970e+00 + -4.875531909767719e+00 + -4.891107385884028e+00 + -4.906708405000000e+00 + -4.922334495555203e+00 + -4.937985181840751e+00 + -4.953659984814029e+00 + -4.969358421588884e+00 + -4.985080006000000e+00 + -5.000824248840914e+00 + -5.016590656340991e+00 + -5.032378731598668e+00 + -5.048187974277996e+00 + -5.064017880000000e+00 + -5.079867940973151e+00 + -5.095737645459726e+00 + -5.111626480823102e+00 + -5.127533934923204e+00 + -5.143459499999997e+00 + -5.159402672483111e+00 + -5.175362954369220e+00 + -5.191339851303958e+00 + -5.207332872891304e+00 + -5.223341533999998e+00 + -5.239365353688981e+00 + -5.255403855076862e+00 + -5.271456565155025e+00 + -5.287523015792912e+00 + -5.303602743999997e+00 + -5.319695290809813e+00 + -5.335800201177900e+00 + -5.351917023850607e+00 + -5.368045312413205e+00 + -5.384184625999997e+00 + -5.400334527792659e+00 + -5.416494583936292e+00 + -5.432664365069262e+00 + -5.448843447960337e+00 + -5.465031412999998e+00 + -5.481227843963791e+00 + -5.497432330198948e+00 + -5.513644465003656e+00 + -5.529863846001168e+00 + -5.546090074999998e+00 + -5.562322757794160e+00 + -5.578561506278419e+00 + -5.594805935740466e+00 + -5.611055664858841e+00 + -5.627310317999997e+00 + -5.643569523867812e+00 + -5.659832914603254e+00 + -5.676100126691991e+00 + -5.692370802379017e+00 + -5.708644587999997e+00 + -5.724921133351554e+00 + -5.741200092816319e+00 + -5.757481124931912e+00 + -5.773763893288205e+00 + -5.790048065999996e+00 + -5.806333315035348e+00 + -5.822619317227983e+00 + -5.838905753168464e+00 + -5.855192307810226e+00 + -5.871478670999998e+00 + -5.887764536642293e+00 + -5.904049603224381e+00 + -5.920333573291820e+00 + -5.936616154283948e+00 + -5.952897057999997e+00 + -5.969175999997380e+00 + -5.985452700712029e+00 + -6.001726884408804e+00 + -6.017998279848357e+00 + -6.034266620999998e+00 + -6.050531645917840e+00 + -6.066793096565948e+00 + -6.083050719154321e+00 + -6.099304265265417e+00 + -6.115553489999997e+00 + -6.131798152082848e+00 + -6.148038016850530e+00 + -6.164272852974367e+00 + -6.180502432221914e+00 + -6.196726531999996e+00 + -6.212944934149927e+00 + -6.229157424398349e+00 + -6.245363792906498e+00 + -6.261563835474968e+00 + -6.277757350999996e+00 + -6.293944141712862e+00 + -6.310124016459128e+00 + -6.326296787703612e+00 + -6.342462271423720e+00 + -6.358620288999997e+00 + -6.374770666045148e+00 + -6.390913231950829e+00 + -6.407047820441565e+00 + -6.423174270811367e+00 + -6.439292425999997e+00 + -6.455402132459349e+00 + -6.471503242529590e+00 + -6.487595612302881e+00 + -6.503679101824962e+00 + -6.519753575999998e+00 + -6.535818903908587e+00 + -6.551874959678351e+00 + -6.567921621040533e+00 + -6.583958769810407e+00 + -6.599986292999997e+00 + -6.616004081814032e+00 + -6.632012031732002e+00 + -6.648010042213294e+00 + -6.663998017633650e+00 + -6.679975866999997e+00 + -6.695943503153600e+00 + -6.711900843425355e+00 + -6.727847809100131e+00 + -6.743784326279565e+00 + -6.759710325999998e+00 + -6.775625743222601e+00 + -6.791530516957345e+00 + -6.807424590371784e+00 + -6.823307911868753e+00 + -6.839180434000000e+00 + -6.855042113095231e+00 + -6.870892911056689e+00 + -6.886732793343159e+00 + -6.902561729209182e+00 + -6.918379693000000e+00 + -6.934186663377454e+00 + -6.949982623974822e+00 + -6.965767562024019e+00 + -6.981541468861679e+00 + -6.997304341000000e+00 + -7.013056179149240e+00 + -7.028796988372566e+00 + -7.044526777567921e+00 + -7.060245560311449e+00 + -7.075953355000000e+00 + -7.091650184045992e+00 + -7.107336074531737e+00 + -7.123011057484204e+00 + -7.138675168614964e+00 + -7.154328448000000e+00 + -7.169970939523060e+00 + -7.185602692089082e+00 + -7.201223758621499e+00 + -7.216834196750369e+00 + -7.232434069000000e+00 + -7.248023441650831e+00 + -7.263602384451917e+00 + -7.279170972021925e+00 + -7.294729285389790e+00 + -7.310277408000000e+00 + -7.325815426345584e+00 + -7.341343434315181e+00 + -7.356861529294433e+00 + -7.372369811668458e+00 + -7.387868387000000e+00 + -7.403357365322682e+00 + -7.418836861854650e+00 + -7.434306995371617e+00 + -7.449767888578398e+00 + -7.465219669000000e+00 + -7.480662468362842e+00 + -7.496096423635060e+00 + -7.511521675509539e+00 + -7.526938368822738e+00 + -7.542346653000000e+00 + -7.557746681513402e+00 + -7.573138613106206e+00 + -7.588522610348394e+00 + -7.603898840111169e+00 + -7.619267474000000e+00 + -7.634628687556104e+00 + -7.649982660818226e+00 + -7.665329578157865e+00 + -7.680669629212444e+00 + -7.696003007000000e+00 + -7.711329908041110e+00 + -7.726650535322031e+00 + -7.741965095546557e+00 + -7.757273799081691e+00 + -7.772576861000000e+00 + -7.787874500482072e+00 + -7.803166941820082e+00 + -7.818454413357075e+00 + -7.833737148083599e+00 + -7.849015383000000e+00 + -7.864289358883056e+00 + -7.879559322305754e+00 + -7.894825523493435e+00 + -7.910088216491122e+00 + -7.925347660000000e+00 + -7.940604117030949e+00 + -7.955857856656654e+00 + -7.971109151088187e+00 + -7.986358275553978e+00 + -8.001605510999999e+00 + -8.016851143003514e+00 + -8.032095461580266e+00 + -8.047338760488806e+00 + -8.062581337981985e+00 + -8.077823497000001e+00 + -8.093065544394179e+00 + -8.108307791577566e+00 + -8.123550554205780e+00 + -8.138794153086785e+00 + -8.154038912999997e+00 + -8.169285162395173e+00 + -8.184533235306290e+00 + -8.199783469410583e+00 + -8.215036206329255e+00 + -8.230291792999997e+00 + -8.245550580528674e+00 + -8.260812923863309e+00 + -8.276079182235739e+00 + -8.291349720375887e+00 + -8.306624906999998e+00 + -8.321905114401627e+00 + -8.337190720129563e+00 + -8.352482105562956e+00 + -8.367779656388556e+00 + -8.383083762999997e+00 + -8.398394819954635e+00 + -8.413713227271362e+00 + -8.429039388338369e+00 + -8.444373710123081e+00 + -8.459716604999997e+00 + -8.475068489773328e+00 + -8.490429785807564e+00 + -8.505800918193746e+00 + -8.521182316459129e+00 + -8.536574414999997e+00 + -8.551977652264261e+00 + -8.567392471335332e+00 + -8.582819319266729e+00 + -8.598258647834163e+00 + -8.613710912999998e+00 + -8.629176574613023e+00 + -8.644656098315155e+00 + -8.660149953296175e+00 + -8.675658612460374e+00 + -8.691182553999997e+00 + -8.706722260346437e+00 + -8.722278218076596e+00 + -8.737850917931995e+00 + -8.753440855866357e+00 + -8.769048531999996e+00 + -8.784674450012169e+00 + -8.800319118245726e+00 + -8.815983049456063e+00 + -8.831666761752983e+00 + -8.847370776999997e+00 + -8.863095620572640e+00 + -8.878841823409489e+00 + -8.894609920288579e+00 + -8.910400450202564e+00 + -8.926213956999996e+00 + -8.942050988642462e+00 + -8.957912097948702e+00 + -8.973797841249739e+00 + -8.989708778929735e+00 + -9.005645476999996e+00 + -9.021608505891232e+00 + -9.037598440089864e+00 + -9.053615857732957e+00 + -9.069661341483132e+00 + -9.085735478999997e+00 + -9.101838862125330e+00 + -9.117972087538210e+00 + -9.134135755396374e+00 + -9.150330469883286e+00 + -9.166556840999997e+00 + -9.182815483079517e+00 + -9.199107013675079e+00 + -9.215432054583626e+00 + -9.231791233265957e+00 + -9.248185180999997e+00 + -9.264614532683632e+00 + -9.281079929109534e+00 + -9.297582014702897e+00 + -9.314121437654787e+00 + -9.330698850999996e+00 + -9.347314912003528e+00 + -9.363970283188971e+00 + -9.380665630745685e+00 + -9.397401624940878e+00 + -9.414178940999998e+00 + -9.430998258244742e+00 + -9.447860260503859e+00 + -9.464765635761319e+00 + -9.481715077019253e+00 + -9.498709280999996e+00 + -9.515748948037771e+00 + -9.532834784442173e+00 + -9.549967500513571e+00 + -9.567147810785697e+00 + -9.584376433999996e+00 + -9.601654092679016e+00 + -9.618981514566920e+00 + -9.636359431588877e+00 + -9.653788580476306e+00 + -9.671269701999998e+00 + -9.688803540653204e+00 + -9.706390846469759e+00 + -9.724032373152683e+00 + -9.741728878390186e+00 + -9.759481124999997e+00 + -9.777289880050315e+00 + -9.795155915261526e+00 + -9.813080006095444e+00 + -9.831062932433847e+00 + -9.849105478999997e+00 + -9.867208434586416e+00 + -9.885372592754736e+00 + -9.903598750780114e+00 + -9.921887710289466e+00 + -9.940240277999996e+00 + -9.958657264807959e+00 + -9.977139486156000e+00 + -9.995687760994512e+00 + -1.001430291248964e+01 + -1.003298577000000e+01 + -1.005173716741557e+01 + -1.007055794170109e+01 + -1.008944893313717e+01 + -1.010841098632109e+01 + -1.012744495000000e+01 + -1.014655167843811e+01 + -1.016573203809090e+01 + -1.018498689280856e+01 + -1.020431709913334e+01 + -1.022372353000000e+01 + -1.024320706932260e+01 + -1.026276860085199e+01 + -1.028240900753776e+01 + -1.030212917226508e+01 + -1.032192999000000e+01 + -1.034181236270672e+01 + -1.036177718920571e+01 + -1.038182537682789e+01 + -1.040195784652257e+01 + -1.042217551000000e+01 + -1.044247927800797e+01 + -1.046287008429113e+01 + -1.048334886197221e+01 + -1.050391653528098e+01 + -1.052457404000000e+01 + -1.054532232135862e+01 + -1.056616233045517e+01 + -1.058709501821704e+01 + -1.060812133414642e+01 + -1.062924224000000e+01 + -1.065045870478386e+01 + -1.067177169452023e+01 + -1.069318217826154e+01 + -1.071469113174534e+01 + -1.073629954000000e+01 + -1.075800839260036e+01 + -1.077981867489276e+01 + -1.080173138018267e+01 + -1.082374751517595e+01 + -1.084586808000000e+01 + -1.086809407471729e+01 + -1.089042651905600e+01 + -1.091286643223940e+01 + -1.093541482646511e+01 + -1.095807273000000e+01 + -1.098084118002027e+01 + -1.100372120627546e+01 + -1.102671384288809e+01 + -1.104982013415814e+01 + -1.107304113000000e+01 + -1.109637788366843e+01 + -1.111983144958023e+01 + -1.114340288661973e+01 + -1.116709326043168e+01 + -1.119090364000000e+01 + -1.121483509746141e+01 + -1.123888871051257e+01 + -1.126306556352589e+01 + -1.128736674843474e+01 + -1.131179335000000e+01 + -1.133634645465832e+01 + -1.136102717560643e+01 + -1.138583662099362e+01 + -1.141077588304154e+01 + -1.143584608000000e+01 + -1.146104834278586e+01 + -1.148638378227322e+01 + -1.151185351768562e+01 + -1.153745868860526e+01 + -1.156320043000000e+01 + -1.158907987623694e+01 + -1.161509817570602e+01 + -1.164125647703402e+01 + -1.166755592508495e+01 + -1.169399768000000e+01 + -1.172058291041322e+01 + -1.174731277861368e+01 + -1.177418844974534e+01 + -1.180121109657743e+01 + -1.182838189999999e+01 + -1.185570204579555e+01 + -1.188317272006268e+01 + -1.191079511332836e+01 + -1.193857042312863e+01 + -1.196649984999999e+01 + -1.199458459773775e+01 + -1.202282587700695e+01 + -1.205122490269986e+01 + -1.207978289369385e+01 + -1.210850107000000e+01 + -1.213738065440104e+01 + -1.216642287877588e+01 + -1.219562898067898e+01 + -1.222500020303083e+01 + -1.225453779000000e+01 + -1.228424298812801e+01 + -1.231411705144751e+01 + -1.234416123799292e+01 + -1.237437680934280e+01 + -1.240476502999999e+01 + -1.243532716862228e+01 + -1.246606450371358e+01 + -1.249697831672881e+01 + -1.252806989018327e+01 + -1.255934050999999e+01 + -1.259079146663310e+01 + -1.262242406008652e+01 + -1.265423959346900e+01 + -1.268623937120698e+01 + -1.271842469999999e+01 + -1.275079689157634e+01 + -1.278335727210730e+01 + -1.281610716656313e+01 + -1.284904789393346e+01 + -1.288218078999999e+01 + -1.291550719883721e+01 + -1.294902845305343e+01 + -1.298274589252790e+01 + -1.301666087289315e+01 + -1.305077474999999e+01 + -1.308508888000826e+01 + -1.311960462335401e+01 + -1.315432334734565e+01 + -1.318924642810263e+01 + -1.322437523999999e+01 + -1.325971115825294e+01 + -1.329525556797775e+01 + -1.333100986045822e+01 + -1.336697543256989e+01 + -1.340315367999999e+01 + -1.343954599979837e+01 + -1.347615379884262e+01 + -1.351297848978655e+01 + -1.355002149050285e+01 + -1.358728421999999e+01 + -1.362476809975143e+01 + -1.366247455932581e+01 + -1.370040503112116e+01 + -1.373856094968169e+01 + -1.377694375999999e+01 + -1.381555491216690e+01 + -1.385439585021465e+01 + -1.389346802391279e+01 + -1.393277289476898e+01 + -1.397231192999999e+01 + -1.401208659972945e+01 + -1.405209837359219e+01 + -1.409234872413346e+01 + -1.413283912961124e+01 + -1.417357107999999e+01 + -1.421454607102127e+01 + -1.425576559243092e+01 + -1.429723113890712e+01 + -1.433894421536596e+01 + -1.438090632999999e+01 + -1.442311899332400e+01 + -1.446558371903957e+01 + -1.450830202765583e+01 + -1.455127544879836e+01 + -1.459450550999999e+01 + -1.463799373973040e+01 + -1.468174167767078e+01 + -1.472575086722947e+01 + -1.477002285383887e+01 + -1.481455918999999e+01 + -1.485936143285468e+01 + -1.490443114031971e+01 + -1.494976987417127e+01 + -1.499537920245961e+01 + -1.504126069999999e+01 + -1.508741594534576e+01 + -1.513384651602358e+01 + -1.518055399754435e+01 + -1.522753998778076e+01 + -1.527480607999999e+01 + -1.532235386668537e+01 + -1.537018495189360e+01 + -1.541830094448208e+01 + -1.546670345694024e+01 + -1.551539410999999e+01 + -1.556437452805486e+01 + -1.561364632974880e+01 + -1.566321114034704e+01 + -1.571307059764629e+01 + -1.576322633999999e+01 + -1.581368000886133e+01 + -1.586443325932826e+01 + -1.591548774371123e+01 + -1.596684510633198e+01 + -1.601850700999999e+01 + -1.607047512763592e+01 + -1.612275112310131e+01 + -1.617533666634843e+01 + -1.622823344044972e+01 + -1.628144312999999e+01 + -1.633496741991010e+01 + -1.638880799548294e+01 + -1.644296655089035e+01 + -1.649744479372194e+01 + -1.655224442999999e+01 + -1.660736716535794e+01 + -1.666281471098534e+01 + -1.671858878335293e+01 + -1.677469110504862e+01 + -1.683112339999999e+01 + -1.688788739643611e+01 + -1.694498483759857e+01 + -1.700241746523662e+01 + -1.706018701398879e+01 + -1.711829522999999e+01 + -1.717674386844329e+01 + -1.723553468924849e+01 + -1.729466945458232e+01 + -1.735414992865347e+01 + -1.741397788000000e+01 + -1.747415508099178e+01 + -1.753468330867943e+01 + -1.759556434768274e+01 + -1.765679999203036e+01 + -1.771839203000000e+01 + -1.778034225038387e+01 + -1.784265246094084e+01 + -1.790532446959567e+01 + -1.796836007816754e+01 + -1.803176110000000e+01 + -1.809552935720682e+01 + -1.815966667575656e+01 + -1.822417488078262e+01 + -1.828905579597735e+01 + -1.835431126000000e+01 + -1.841994311958100e+01 + -1.848595321475806e+01 + -1.855234338946741e+01 + -1.861911549709020e+01 + -1.868627140000000e+01 + -1.875381296458075e+01 + -1.882174205218422e+01 + -1.889006052915166e+01 + -1.895877027181235e+01 + -1.902787316000000e+01 + -1.909737107797965e+01 + -1.916726592078253e+01 + -1.923755957952593e+01 + -1.930825393714747e+01 + -1.937935090000000e+01 + -1.945085238647674e+01 + -1.952276030080435e+01 + -1.959507655061933e+01 + -1.966780305499929e+01 + -1.974094174000000e+01 + -1.981449453618971e+01 + -1.988846337667131e+01 + -1.996285019538828e+01 + -2.003765692760933e+01 + -2.011288552000000e+01 + -2.018853792621815e+01 + -2.026461609876727e+01 + -2.034112199510926e+01 + -2.041805758075517e+01 + -2.049542482000000e+01 + -2.057322568006621e+01 + -2.065146214471047e+01 + -2.073013619484967e+01 + -2.080924980240138e+01 + -2.088880495999999e+01 + -2.096880367169853e+01 + -2.104924793164823e+01 + -2.113013973534757e+01 + -2.121148108557274e+01 + -2.129327399999999e+01 + -2.137552050224973e+01 + -2.145822260294134e+01 + -2.154138232011573e+01 + -2.162500168821614e+01 + -2.170908273999999e+01 + -2.179362750931509e+01 + -2.187863804236205e+01 + -2.196411638687395e+01 + -2.205006458870765e+01 + -2.213648469999998e+01 + -2.222337877888525e+01 + -2.231074889087196e+01 + -2.239859710699802e+01 + -2.248692550346576e+01 + -2.257573614999999e+01 + -2.266503111825121e+01 + -2.275481250533621e+01 + -2.284508240538831e+01 + -2.293584289979252e+01 + -2.302709608999998e+01 + -2.311884408797131e+01 + -2.321108899308483e+01 + -2.330383291176923e+01 + -2.339707796636268e+01 + -2.349082627999998e+01 + -2.358507997580010e+01 + -2.367984117841246e+01 + -2.377511201965127e+01 + -2.387089464198777e+01 + -2.396719118999998e+01 + -2.406400380890314e+01 + -2.416133464354705e+01 + -2.425918584868402e+01 + -2.435755959379285e+01 + -2.445645803999998e+01 + -2.455588334675619e+01 + -2.465583769169985e+01 + -2.475632325621871e+01 + -2.485734222078360e+01 + -2.495889676999998e+01 + -2.506098909313465e+01 + -2.516362138717989e+01 + -2.526679585042222e+01 + -2.537051468121954e+01 + -2.547478008999998e+01 + -2.557959429416147e+01 + -2.568495950782481e+01 + -2.579087794894506e+01 + -2.589735184307986e+01 + -2.600438342000000e+01 + -2.611197491414786e+01 + -2.622012856937007e+01 + -2.632884662779481e+01 + -2.643813132670684e+01 + -2.654798492000000e+01 + -2.665840967182968e+01 + -2.676940784333127e+01 + -2.688098169572292e+01 + -2.699313349264397e+01 + -2.710586551000000e+01 + -2.721918002998715e+01 + -2.733307932892397e+01 + -2.744756569229334e+01 + -2.756264142140602e+01 + -2.767830881000000e+01 + -2.779457014947893e+01 + -2.791142774510221e+01 + -2.802888390934165e+01 + -2.814694096000261e+01 + -2.826560121000000e+01 + -2.838486697334714e+01 + -2.850474058224100e+01 + -2.862522436986088e+01 + -2.874632066501767e+01 + -2.886803181000000e+01 + -2.899036015446983e+01 + -2.911330804180568e+01 + -2.923687781925054e+01 + -2.936107184331833e+01 + -2.948589248000000e+01 + -2.961134209910011e+01 + -2.973742306338333e+01 + -2.986413774439124e+01 + -2.999148852946843e+01 + -3.011947780000000e+01 + -3.024810793640462e+01 + -3.037738133418102e+01 + -3.050730039248281e+01 + -3.063786751048302e+01 + -3.076908508999995e+01 + -3.090095553801825e+01 + -3.103348127528379e+01 + -3.116666472242296e+01 + -3.130050829541340e+01 + -3.143501442000000e+01 + -3.157018552904646e+01 + -3.170602405762165e+01 + -3.184253244456109e+01 + -3.197971313442012e+01 + -3.211756857999995e+01 + -3.225610123724459e+01 + -3.239531355479052e+01 + -3.253520799120049e+01 + -3.267578702273956e+01 + -3.281705312000000e+01 + -3.295900875199519e+01 + -3.310165640007095e+01 + -3.324499854981137e+01 + -3.338903768896250e+01 + -3.353377630999994e+01 + -3.367921690900354e+01 + -3.382536198480008e+01 + -3.397221404252836e+01 + -3.411977559608952e+01 + -3.426804916000000e+01 + -3.441703725062087e+01 + -3.456674239208418e+01 + -3.471716711255165e+01 + -3.486831394366235e+01 + -3.502018541999994e+01 + -3.517278408030963e+01 + -3.532611247314257e+01 + -3.548017315006599e+01 + -3.563496866378715e+01 + -3.579050157000000e+01 + -3.594677442911821e+01 + -3.610378981290209e+01 + -3.626155029281482e+01 + -3.642005843692588e+01 + -3.657931682999995e+01 + -3.673932806450152e+01 + -3.690009471994529e+01 + -3.706161938636306e+01 + -3.722390467398576e+01 + -3.738695318000000e+01 + -3.755076749872251e+01 + -3.771535025114206e+01 + -3.788070405790332e+01 + -3.804683152981476e+01 + -3.821373528999994e+01 + -3.838141797088579e+01 + -3.854988220802454e+01 + -3.871913063690455e+01 + -3.888916589260551e+01 + -3.905999062000000e+01 + -3.923160747217621e+01 + -3.940401910958111e+01 + -3.957722819105772e+01 + -3.975123737141870e+01 + -3.992604931999993e+01 + -4.010166671506960e+01 + -4.027809223215413e+01 + -4.045532854769397e+01 + -4.063337834194039e+01 + -4.081224431000000e+01 + -4.099192915380458e+01 + -4.117243556512365e+01 + -4.135376624213356e+01 + -4.153592389680070e+01 + -4.171891123999993e+01 + -4.190273098404973e+01 + -4.208738585293229e+01 + -4.227287857304115e+01 + -4.245921187066173e+01 + -4.264638848000000e+01 + -4.283441114000928e+01 + -4.302328258823673e+01 + -4.321300556988346e+01 + -4.340358284230594e+01 + -4.359501715999993e+01 + -4.378731127802528e+01 + -4.398046796379431e+01 + -4.417448998677844e+01 + -4.436938011574802e+01 + -4.456514113000000e+01 + -4.476177581459606e+01 + -4.495928695037842e+01 + -4.515767732491430e+01 + -4.535694973791337e+01 + -4.555710698999992e+01 + -4.575815188265746e+01 + -4.596008722124420e+01 + -4.616291581815827e+01 + -4.636664049491562e+01 + -4.657126407000000e+01 + -4.677678936329625e+01 + -4.698321921012249e+01 + -4.719055644645270e+01 + -4.739880390447049e+01 + -4.760796442999992e+01 + -4.781804087601476e+01 + -4.802903608806237e+01 + -4.824095291782189e+01 + -4.845379422957746e+01 + -4.866756289000000e+01 + -4.888226176793614e+01 + -4.909789373759357e+01 + -4.931446167624577e+01 + -4.953196846445736e+01 + -4.975041698999991e+01 + -4.996981014490353e+01 + -5.019015082042799e+01 + -5.041144191483806e+01 + -5.063368633767777e+01 + -5.085688700000000e+01 + -5.108104681363623e+01 + -5.130616869222998e+01 + -5.153225555529192e+01 + -5.175931033093434e+01 + -5.198733594999992e+01 + -5.221633534632925e+01 + -5.244631146064819e+01 + -5.267726723814258e+01 + -5.290920562821528e+01 + -5.314212958000000e+01 + -5.337604204603855e+01 + -5.361094599406351e+01 + -5.384684439089011e+01 + -5.408374019738633e+01 + -5.432163638999991e+01 + -5.456053595406303e+01 + -5.480044186892415e+01 + -5.504135712069162e+01 + -5.528328470761400e+01 + -5.552622762000000e+01 + -5.577018884907901e+01 + -5.601517141284037e+01 + -5.626117832498171e+01 + -5.650821258404100e+01 + -5.675627720999991e+01 + -5.700537523542541e+01 + -5.725550968441112e+01 + -5.750668358361924e+01 + -5.775889996777224e+01 + -5.801216188000000e+01 + -5.826647236807500e+01 + -5.852183447842080e+01 + -5.877825126017432e+01 + -5.903572576827786e+01 + -5.929426106999990e+01 + -5.955386023803142e+01 + -5.981452633626228e+01 + -6.007626243395679e+01 + -6.033907161265666e+01 + -6.060295696000000e+01 + -6.086792156620618e+01 + -6.113396851889991e+01 + -6.140110091270849e+01 + -6.166932185413050e+01 + -6.193863444999990e+01 + -6.220904180836374e+01 + -6.248054704389698e+01 + -6.275315327303672e+01 + -6.302686361375484e+01 + -6.330168120000000e+01 + -6.357760917255472e+01 + -6.385465065869710e+01 + -6.413280879313753e+01 + -6.441208672717147e+01 + -6.469248761000000e+01 + -6.497401459207148e+01 + -6.525667083785252e+01 + -6.554045951020694e+01 + -6.582538376572784e+01 + -6.611144677999999e+01 + -6.639865173804421e+01 + -6.668700181225154e+01 + -6.697650018322166e+01 + -6.726715004865791e+01 + -6.755895460000001e+01 + -6.785191702754956e+01 + -6.814604053706709e+01 + -6.844132833746227e+01 + -6.873778363702108e+01 + -6.903540965000001e+01 + -6.933420959649558e+01 + -6.963418670433917e+01 + -6.993534420146086e+01 + -7.023768531400407e+01 + -7.054121327999999e+01 + -7.084593134500037e+01 + -7.115184275353549e+01 + -7.145895075432828e+01 + -7.176725860332733e+01 + -7.207676955999989e+01 + -7.238748688676215e+01 + -7.269941385037480e+01 + -7.301255372170849e+01 + -7.332690977697268e+01 + -7.364248530000000e+01 + -7.395928357816831e+01 + -7.427730789465394e+01 + -7.459656154154558e+01 + -7.491704782593285e+01 + -7.523877004999987e+01 + -7.556173151464178e+01 + -7.588593553169946e+01 + -7.621138541944168e+01 + -7.653808450163682e+01 + -7.686603610000000e+01 + -7.719524353805981e+01 + -7.752571015313293e+01 + -7.785743928572181e+01 + -7.819043427631229e+01 + -7.852469846999988e+01 + -7.886023521608357e+01 + -7.919704786882774e+01 + -7.953513978918475e+01 + -7.987451434652237e+01 + -8.021517491000000e+01 + -8.055712484895521e+01 + -8.090036753598372e+01 + -8.124490635250547e+01 + -8.159074469172391e+01 + -8.193788593999987e+01 + -8.228633348416081e+01 + -8.263609073313938e+01 + -8.298716109541151e+01 + -8.333954797103145e+01 + -8.369325477000000e+01 + -8.404828491028280e+01 + -8.440464181412719e+01 + -8.476232890793086e+01 + -8.512134962287996e+01 + -8.548170738999985e+01 + -8.584340564352934e+01 + -8.620644783199920e+01 + -8.657083740482393e+01 + -8.693657780824788e+01 + -8.730367250000000e+01 + -8.767212494371194e+01 + -8.804193859639375e+01 + -8.841311692593891e+01 + -8.878566341870939e+01 + -8.915958154999986e+01 + -8.953487479168578e+01 + -8.991154663471849e+01 + -9.028960057395298e+01 + -9.066904010318096e+01 + -9.104986872000001e+01 + -9.143208992686800e+01 + -9.181570723560402e+01 + -9.220072416114067e+01 + -9.258714421980083e+01 + -9.297497092999986e+01 + -9.336420781525349e+01 + -9.375485841426487e+01 + -9.414692626313253e+01 + -9.454041488965315e+01 + -9.493532784000000e+01 + -9.533166867059026e+01 + -9.572944092950100e+01 + -9.612864816948191e+01 + -9.652929395407540e+01 + -9.693138184999984e+01 + -9.733491542620831e+01 + -9.773989825487583e+01 + -9.814633391500892e+01 + -9.855422599474231e+01 + -9.896357808000000e+01 + -9.937439375553778e+01 + -9.978667660951464e+01 + -1.002004302498873e+02 + -1.006156583113964e+02 + -1.010323643999998e+02 + -1.014505521033422e+02 + -1.018702250167552e+02 + -1.022913867648504e+02 + -1.027140410123223e+02 + -1.031381914000000e+02 + -1.035638415489902e+02 + -1.039909950728156e+02 + -1.044196556182856e+02 + -1.048498268814360e+02 + -1.052815124999998e+02 + -1.057147160851634e+02 + -1.061494413040264e+02 + -1.065856918490972e+02 + -1.070234714266025e+02 + -1.074627837000000e+02 + -1.079036323073108e+02 + -1.083460208994117e+02 + -1.087899531790557e+02 + -1.092354329168266e+02 + -1.096824637999998e+02 + -1.101310494754678e+02 + -1.105811936593542e+02 + -1.110329000832090e+02 + -1.114861724712110e+02 + -1.119410145000000e+02 + -1.123974298499644e+02 + -1.128554223365672e+02 + -1.133149957276709e+02 + -1.137761536721920e+02 + -1.142388998999998e+02 + -1.147032381951767e+02 + -1.151691723223398e+02 + -1.156367060267053e+02 + -1.161058430340008e+02 + -1.165765871000000e+02 + -1.170489420136372e+02 + -1.175229116129598e+02 + -1.179984996644491e+02 + -1.184757098197581e+02 + -1.189545458999998e+02 + -1.194350118004852e+02 + -1.199171112501401e+02 + -1.204008480035419e+02 + -1.208862259161727e+02 + -1.213732488000000e+02 + -1.218619204419225e+02 + -1.223522446637547e+02 + -1.228442252993658e+02 + -1.233378661832514e+02 + -1.238331710999998e+02 + -1.243301438371392e+02 + -1.248287883218059e+02 + -1.253291084262349e+02 + -1.258311078947497e+02 + -1.263347906000000e+02 + -1.268401604692280e+02 + -1.273472212887372e+02 + -1.278559768940106e+02 + -1.283664312433523e+02 + -1.288785881999998e+02 + -1.293924515790011e+02 + -1.299080252758494e+02 + -1.304253131892150e+02 + -1.309443191941241e+02 + -1.314650472000000e+02 + -1.319875011267033e+02 + -1.325116848422910e+02 + -1.330376022476014e+02 + -1.335652573069860e+02 + -1.340946538999998e+02 + -1.346257958892211e+02 + -1.351586873010075e+02 + -1.356933320835031e+02 + -1.362297340185993e+02 + -1.367678971000000e+02 + -1.373078254089903e+02 + -1.378495227933144e+02 + -1.383929931342302e+02 + -1.389382404506380e+02 + -1.394852686999998e+02 + -1.400340818144768e+02 + -1.405846838138316e+02 + -1.411370786853171e+02 + -1.416912703410207e+02 + -1.422472628000000e+02 + -1.428050601118040e+02 + -1.433646661562381e+02 + -1.439260848884736e+02 + -1.444893204354731e+02 + -1.450543767999998e+02 + -1.456212579173106e+02 + -1.461899678121767e+02 + -1.467605105156440e+02 + -1.473328900351724e+02 + -1.479071104000000e+02 + -1.484831756560318e+02 + -1.490610898537698e+02 + -1.496408570188282e+02 + -1.502224811435522e+02 + -1.508059663000000e+02 + -1.513913165938295e+02 + -1.519785360517816e+02 + -1.525676287092765e+02 + -1.531585986433134e+02 + -1.537514498999998e+02 + -1.543461865304275e+02 + -1.549428126935839e+02 + -1.555413324987964e+02 + -1.561417499433451e+02 + -1.567440691000000e+02 + -1.573482940967384e+02 + -1.579544290618548e+02 + -1.585624781173004e+02 + -1.591724453723425e+02 + -1.597843348999998e+02 + -1.603981507727703e+02 + -1.610138971515657e+02 + -1.616315781879854e+02 + -1.622511979884414e+02 + -1.628727607000000e+02 + -1.634962704824068e+02 + -1.641217314319286e+02 + -1.647491476962519e+02 + -1.653785235159056e+02 + -1.660098629999998e+02 + -1.666431702132707e+02 + -1.672784494025457e+02 + -1.679157047805161e+02 + -1.685549404452619e+02 + -1.691961606000000e+02 + -1.698393694891200e+02 + -1.704845712252935e+02 + -1.711317699673001e+02 + -1.717809699904770e+02 + -1.724321754999998e+02 + -1.730853906636835e+02 + -1.737406197069920e+02 + -1.743978668390772e+02 + -1.750571362267288e+02 + -1.757184321000000e+02 + -1.763817587329360e+02 + -1.770471204015179e+02 + -1.777145213342737e+02 + -1.783839656916257e+02 + -1.790554576999997e+02 + -1.797290016348361e+02 + -1.804046017812607e+02 + -1.810822624074922e+02 + -1.817619877551305e+02 + -1.824437821000000e+02 + -1.831276497291853e+02 + -1.838135948809234e+02 + -1.845016218201234e+02 + -1.851917348679873e+02 + -1.858839382999998e+02 + -1.865782363787204e+02 + -1.872746334451072e+02 + -1.879731338116912e+02 + -1.886737417231232e+02 + -1.893764615000000e+02 + -1.900812975069574e+02 + -1.907882540726982e+02 + -1.914973354967214e+02 + -1.922085460544087e+02 + -1.929218900999998e+02 + -1.936373720194894e+02 + -1.943549961169706e+02 + -1.950747667123967e+02 + -1.957966881820183e+02 + -1.965207649000000e+02 + -1.972470012275908e+02 + -1.979754014928308e+02 + -1.987059700466389e+02 + -1.994387112870401e+02 + -2.001736295999997e+02 + -2.009107293657797e+02 + -2.016500149830146e+02 + -2.023914908231823e+02 + -2.031351612174878e+02 + -2.038810306000000e+02 + -2.046291034502658e+02 + -2.053793841544099e+02 + -2.061318770718700e+02 + -2.068865865659776e+02 + -2.076435170999997e+02 + -2.084026731731890e+02 + -2.091640591716557e+02 + -2.099276794995817e+02 + -2.106935386300690e+02 + -2.114616410000000e+02 + -2.122319910255027e+02 + -2.130045931490896e+02 + -2.137794518362718e+02 + -2.145565715736208e+02 + -2.153359567999997e+02 + -2.161176119510612e+02 + -2.169015415764713e+02 + -2.176877501734888e+02 + -2.184762421224458e+02 + -2.192670219000000e+02 + -2.200600940505141e+02 + -2.208554631128035e+02 + -2.216531335679009e+02 + -2.224531098207831e+02 + -2.232553963999997e+02 + -2.240599978997985e+02 + -2.248669188401317e+02 + -2.256761637092654e+02 + -2.264877369816284e+02 + -2.273016432000000e+02 + -2.281178869415660e+02 + -2.289364727411434e+02 + -2.297574051254879e+02 + -2.305806886273285e+02 + -2.314063277999997e+02 + -2.322343272067604e+02 + -2.330646913987978e+02 + -2.338974249389632e+02 + -2.347325324124834e+02 + -2.355700184000000e+02 + -2.364098874788788e+02 + -2.372521442301691e+02 + -2.380967932316097e+02 + -2.389438390563173e+02 + -2.397932862999997e+02 + -2.406451395699314e+02 + -2.414994034603918e+02 + -2.423560825773335e+02 + -2.432151815494090e+02 + -2.440767050000000e+02 + -2.449406575494712e+02 + -2.458070438246558e+02 + -2.466758684438366e+02 + -2.475471360143982e+02 + -2.484208511999996e+02 + -2.492970186823550e+02 + -2.501756430678903e+02 + -2.510567289807632e+02 + -2.519402811010791e+02 + -2.528263041000000e+02 + -2.537148026392507e+02 + -2.546057813786489e+02 + -2.554992449908618e+02 + -2.563951981686694e+02 + -2.572936455999996e+02 + -2.581945919701986e+02 + -2.590980419711499e+02 + -2.600040002856254e+02 + -2.609124715849065e+02 + -2.618234606000000e+02 + -2.627369720784054e+02 + -2.636530106775519e+02 + -2.645715811006028e+02 + -2.654926881465747e+02 + -2.664163364999997e+02 + -2.673425308135112e+02 + -2.682712759281322e+02 + -2.692025766270102e+02 + -2.701364375396030e+02 + -2.710728634000000e+02 + -2.720118590149941e+02 + -2.729534291763713e+02 + -2.738975786300543e+02 + -2.748443120676544e+02 + -2.757936342999997e+02 + -2.767455501787848e+02 + -2.777000643997414e+02 + -2.786571816989448e+02 + -2.796169069311608e+02 + -2.805792449000000e+02 + -2.815442003833780e+02 + -2.825117782146733e+02 + -2.834819832031981e+02 + -2.844548201023671e+02 + -2.854302936999997e+02 + -2.864084088239409e+02 + -2.873891703596309e+02 + -2.883725831483384e+02 + -2.893586519481404e+02 + -2.903473816000000e+02 + -2.913387769892773e+02 + -2.923328429461174e+02 + -2.933295843055886e+02 + -2.943290059327143e+02 + -2.953311126999996e+02 + -2.963359094688269e+02 + -2.973434010446480e+02 + -2.983535922946355e+02 + -2.993664881931061e+02 + -3.003820936000000e+02 + -3.014004133189056e+02 + -3.024214522481863e+02 + -3.034452153213495e+02 + -3.044717074825456e+02 + -3.055009335999997e+02 + -3.065328985120693e+02 + -3.075676071368449e+02 + -3.086050644279766e+02 + -3.096452753553112e+02 + -3.106882447999996e+02 + -3.117339776182681e+02 + -3.127824787990806e+02 + -3.138337533166528e+02 + -3.148878060732492e+02 + -3.159446419999996e+02 + -3.170042660531784e+02 + -3.180666831989414e+02 + -3.191318984100575e+02 + -3.201999166629245e+02 + -3.212707428999996e+02 + -3.223443820580361e+02 + -3.234208391391681e+02 + -3.245001191331626e+02 + -3.255822269890704e+02 + -3.266671677000000e+02 + -3.277549462905472e+02 + -3.288455677875165e+02 + -3.299390371822755e+02 + -3.310353594166140e+02 + -3.321345394999996e+02 + -3.332365824854958e+02 + -3.343414934139734e+02 + -3.354492773045120e+02 + -3.365599391517927e+02 + -3.376734840000000e+02 + -3.387899169180687e+02 + -3.399092429402671e+02 + -3.410314670886827e+02 + -3.421565943845668e+02 + -3.432846298999996e+02 + -3.444155787253185e+02 + -3.455494458929433e+02 + -3.466862364711880e+02 + -3.478259555971263e+02 + -3.489686083000000e+02 + -3.501141995805723e+02 + -3.512627346186799e+02 + -3.524142185495274e+02 + -3.535686563776662e+02 + -3.547260531999996e+02 + -3.558864141649047e+02 + -3.570497443588797e+02 + -3.582160488781844e+02 + -3.593853328562213e+02 + -3.605576014000000e+02 + -3.617328596203877e+02 + -3.629111127187760e+02 + -3.640923658296630e+02 + -3.652766239636252e+02 + -3.664638922999996e+02 + -3.676541760949210e+02 + -3.688474804479435e+02 + -3.700438104641370e+02 + -3.712431713192604e+02 + -3.724455682000000e+02 + -3.736510062917217e+02 + -3.748594907617601e+02 + -3.760710267615897e+02 + -3.772856194324323e+02 + -3.785032739999995e+02 + -3.797239957155368e+02 + -3.809477897125092e+02 + -3.821746611611994e+02 + -3.834046153278488e+02 + -3.846376574000000e+02 + -3.858737925371631e+02 + -3.871130260109560e+02 + -3.883553630688227e+02 + -3.896008088834330e+02 + -3.908493686999996e+02 + -3.921010477895665e+02 + -3.933558513233559e+02 + -3.946137845264227e+02 + -3.958748527376295e+02 + -3.971390612000000e+02 + -3.984064151109821e+02 + -3.996769197585321e+02 + -4.009505804131749e+02 + -4.022274022910074e+02 + -4.035073906999995e+02 + -4.047905509809469e+02 + -4.060768883569506e+02 + -4.073664080804837e+02 + -4.086591154923789e+02 + -4.099550159000000e+02 + -4.112541145837781e+02 + -4.125564168232350e+02 + -4.138619279264533e+02 + -4.151706532436620e+02 + -4.164825980999995e+02 + -4.177977678009782e+02 + -4.191161676484082e+02 + -4.204378029703449e+02 + -4.217626791355866e+02 + -4.230908015000000e+02 + -4.244221753941545e+02 + -4.257568060915354e+02 + -4.270946989435893e+02 + -4.284358594322071e+02 + -4.297802928999995e+02 + -4.311280046181774e+02 + -4.324789999621266e+02 + -4.338332843406102e+02 + -4.351908631680312e+02 + -4.365517418000000e+02 + -4.379159255766193e+02 + -4.392834199290754e+02 + -4.406542302894571e+02 + -4.420283620541414e+02 + -4.434058205999995e+02 + -4.447866113008026e+02 + -4.461707395611302e+02 + -4.475582108324218e+02 + -4.489490306166049e+02 + -4.503432043000000e+02 + -4.517407372272212e+02 + -4.531416348857516e+02 + -4.545459027469901e+02 + -4.559535462055746e+02 + -4.573645706999994e+02 + -4.587789817092769e+02 + -4.601967847429616e+02 + -4.616179852648949e+02 + -4.630425886631084e+02 + -4.644706004000000e+02 + -4.659020259847504e+02 + -4.673368709042424e+02 + -4.687751406384571e+02 + -4.702168406679589e+02 + -4.716619764999995e+02 + -4.731105536500806e+02 + -4.745625775980045e+02 + -4.760180538297375e+02 + -4.774769878558138e+02 + -4.789393852000000e+02 + -4.804052513950263e+02 + -4.818745919803221e+02 + -4.833474124743740e+02 + -4.848237183656923e+02 + -4.863035151999994e+02 + -4.877868085453354e+02 + -4.892736039052198e+02 + -4.907639068268139e+02 + -4.922577229425711e+02 + -4.937550578000000e+02 + -4.952559169027076e+02 + -4.967603058178435e+02 + -4.982682301416788e+02 + -4.997796954853789e+02 + -5.012947073999994e+02 + -5.028132714194502e+02 + -5.043353931676896e+02 + -5.058610782730644e+02 + -5.073903323321115e+02 + -5.089231609000000e+02 + -5.104595695386481e+02 + -5.119995639358939e+02 + -5.135431497453484e+02 + -5.150903325247945e+02 + -5.166411178999995e+02 + -5.181955115300345e+02 + -5.197535190105982e+02 + -5.213151459679927e+02 + -5.228803980937674e+02 + -5.244492810000000e+02 + -5.260218002787250e+02 + -5.275979616609960e+02 + -5.291777708470047e+02 + -5.307612334400695e+02 + -5.323483550999995e+02 + -5.339391415258532e+02 + -5.355335984055753e+02 + -5.371317314077364e+02 + -5.387335461786030e+02 + -5.403390484000000e+02 + -5.419482437848817e+02 + -5.435611380730882e+02 + -5.451777369746786e+02 + -5.467980461477442e+02 + -5.484220712999994e+02 + -5.500498181706811e+02 + -5.516812924850077e+02 + -5.533164999617454e+02 + -5.549554463178829e+02 + -5.565981372999994e+02 + -5.582445786620732e+02 + -5.598947761113077e+02 + -5.615487353817318e+02 + -5.632064622631164e+02 + -5.648679624999994e+02 + -5.665332418238921e+02 + -5.682023060429622e+02 + -5.698751609401672e+02 + -5.715518122344557e+02 + -5.732322656999993e+02 + -5.749165271427032e+02 + -5.766046023386582e+02 + -5.782964970855437e+02 + -5.799922172195480e+02 + -5.816917684999994e+02 + -5.833951566715182e+02 + -5.851023876259811e+02 + -5.868134672043320e+02 + -5.885284011210334e+02 + -5.902471951999993e+02 + -5.919698553287222e+02 + -5.936963873375214e+02 + -5.954267970459595e+02 + -5.971610902818754e+02 + -5.988992728999993e+02 + -6.006413507613062e+02 + -6.023873296844711e+02 + -6.041372155081436e+02 + -6.058910141182311e+02 + -6.076487313999993e+02 + -6.094103732273231e+02 + -6.111759454411804e+02 + -6.129454539033432e+02 + -6.147189045204622e+02 + -6.164963032000001e+02 + -6.182776558378988e+02 + -6.200629682925465e+02 + -6.218522464515877e+02 + -6.236454962577363e+02 + -6.254427235999993e+02 + -6.272439343587159e+02 + -6.290491345295636e+02 + -6.308583300588987e+02 + -6.326715267788347e+02 + -6.344887306000001e+02 + -6.363099474872770e+02 + -6.381351833941451e+02 + -6.399644442896572e+02 + -6.417977361628546e+02 + -6.436350648999993e+02 + -6.454764363684590e+02 + -6.473218566292518e+02 + -6.491713317125350e+02 + -6.510248675271151e+02 + -6.528824700000000e+02 + -6.547441451008573e+02 + -6.566098988943567e+02 + -6.584797373808955e+02 + -6.603536664383599e+02 + -6.622316920999993e+02 + -6.641138204605409e+02 + -6.660000574329696e+02 + -6.678904089961051e+02 + -6.697848812885560e+02 + -6.716834803000000e+02 + -6.735862119462199e+02 + -6.754930822688247e+02 + -6.774040973560088e+02 + -6.793192633079277e+02 + -6.812385860999992e+02 + -6.831620716800010e+02 + -6.850897262089175e+02 + -6.870215557906422e+02 + -6.889575663679858e+02 + -6.908977640000001e+02 + -6.928421548145254e+02 + -6.947907448778678e+02 + -6.967435402481692e+02 + -6.987005469959722e+02 + -7.006617711999993e+02 + -7.026272189534540e+02 + -7.045968963883171e+02 + -7.065708096021641e+02 + -7.085489646318440e+02 + -7.105313676000000e+02 + -7.125180246666655e+02 + -7.145089419064375e+02 + -7.165041254104643e+02 + -7.185035813280965e+02 + -7.205073157999993e+02 + -7.225153349567163e+02 + -7.245276449240899e+02 + -7.265442518464891e+02 + -7.285651618951177e+02 + -7.305903812000000e+02 + -7.326199158832851e+02 + -7.346537721472801e+02 + -7.366919561576724e+02 + -7.387344740033407e+02 + -7.407813318999993e+02 + -7.428325361060332e+02 + -7.448880927069067e+02 + -7.469480078569493e+02 + -7.490122878703525e+02 + -7.510809389000000e+02 + -7.531539670309830e+02 + -7.552313785295614e+02 + -7.573131796529314e+02 + -7.593993765758833e+02 + -7.614899754999992e+02 + -7.635849826556215e+02 + -7.656844042998581e+02 + -7.677882466626152e+02 + -7.698965159244972e+02 + -7.720092183000000e+02 + -7.741263600422619e+02 + -7.762479474594095e+02 + -7.783739868214470e+02 + -7.805044843222228e+02 + -7.826394461999992e+02 + -7.847788787331309e+02 + -7.869227882276110e+02 + -7.890711809493379e+02 + -7.912240631015031e+02 + -7.933814410000000e+02 + -7.955433210068886e+02 + -7.977097093640062e+02 + -7.998806123337483e+02 + -8.020560362523712e+02 + -8.042359873999992e+02 + -8.064204720446323e+02 + -8.086094965664213e+02 + -8.108030673053524e+02 + -8.130011905039750e+02 + -8.152038725000000e+02 + -8.174111196729889e+02 + -8.196229383028856e+02 + -8.218393347070877e+02 + -8.240603152936806e+02 + -8.262858863999992e+02 + -8.285160543327190e+02 + -8.307508254786889e+02 + -8.329902062108882e+02 + -8.352342028550189e+02 + -8.374828218000000e+02 + -8.397360694583805e+02 + -8.419939521636865e+02 + -8.442564762682848e+02 + -8.465236481828920e+02 + -8.487954742999991e+02 + -8.510719610066617e+02 + -8.533531147279726e+02 + -8.556389418758610e+02 + -8.579294488297842e+02 + -8.602246420000000e+02 + -8.625245278111681e+02 + -8.648291126593311e+02 + -8.671384029773127e+02 + -8.694524052578884e+02 + -8.717711258999991e+02 + -8.740945712693658e+02 + -8.764227478547659e+02 + -8.787556621558366e+02 + -8.810933206360036e+02 + -8.834357297000000e+02 + -8.857828957473700e+02 + -8.881348253000285e+02 + -8.904915248639498e+02 + -8.928530008754832e+02 + -8.952192597999992e+02 + -8.975903081273602e+02 + -8.999661523549036e+02 + -9.023467989915350e+02 + -9.047322545556517e+02 + -9.071225255000001e+02 + -9.095176182642568e+02 + -9.119175394073789e+02 + -9.143222954857910e+02 + -9.167318930073794e+02 + -9.191463384999990e+02 + -9.215656384936522e+02 + -9.239897994653498e+02 + -9.264188279352956e+02 + -9.288527305046009e+02 + -9.312915136999991e+02 + -9.337351840217814e+02 + -9.361837480744202e+02 + -9.386372124507991e+02 + -9.410955836845210e+02 + -9.435588682999991e+02 + -9.460270728408829e+02 + -9.485002039375468e+02 + -9.509782681943145e+02 + -9.534612721474879e+02 + -9.559492223999991e+02 + -9.584421255818636e+02 + -9.609399882443452e+02 + -9.634428169536077e+02 + -9.659506183300578e+02 + -9.684633989999990e+02 + -9.709811655949806e+02 + -9.735039247636544e+02 + -9.760316831188370e+02 + -9.785644472182881e+02 + -9.811022236999991e+02 + -9.836450192407020e+02 + -9.861928404528198e+02 + -9.887456939756197e+02 + -9.913035865101050e+02 + -9.938665246999991e+02 + -9.964345151588749e+02 + -9.990075645416573e+02 + -1.001585679508894e+03 + -1.004168866725833e+03 + -1.006757132999999e+03 + -1.009350485088950e+03 + -1.011948929193798e+03 + -1.014552472017065e+03 + -1.017161121149714e+03 + -1.019774882999999e+03 + -1.022393763477281e+03 + -1.025017769745607e+03 + -1.027646908706880e+03 + -1.030281186449000e+03 + -1.032920609999999e+03 + -1.035565186772393e+03 + -1.038214923106640e+03 + -1.040869825393126e+03 + -1.043529900494233e+03 + -1.046195154999999e+03 + -1.048865595518282e+03 + -1.051541229525064e+03 + -1.054222063914517e+03 + -1.056908104442841e+03 + -1.059599357999999e+03 + -1.062295832060889e+03 + -1.064997533253958e+03 + -1.067704468147946e+03 + -1.070416643533965e+03 + -1.073134066000000e+03 + -1.075856742179453e+03 + -1.078584679453085e+03 + -1.081317884808236e+03 + -1.084056364390976e+03 + -1.086800124999999e+03 + -1.089549173751501e+03 + -1.092303517161580e+03 + -1.095063162007773e+03 + -1.097828115663242e+03 + -1.100598385000000e+03 + -1.103373976558467e+03 + -1.106154896984741e+03 + -1.108941153067517e+03 + -1.111732751768433e+03 + -1.114529699999999e+03 + -1.117332004656191e+03 + -1.120139672719820e+03 + -1.122952711033347e+03 + -1.125771126202256e+03 + -1.128594925000000e+03 + -1.131424114391635e+03 + -1.134258701616623e+03 + -1.137098693726371e+03 + -1.139944097362279e+03 + -1.142794918999999e+03 + -1.145651165298781e+03 + -1.148512843975744e+03 + -1.151379962155369e+03 + -1.154252525751212e+03 + -1.157130542000000e+03 + -1.160014018647746e+03 + -1.162902961796701e+03 + -1.165797378002409e+03 + -1.168697275086654e+03 + -1.171602659999999e+03 + -1.174513539162380e+03 + -1.177429919411751e+03 + -1.180351807863387e+03 + -1.183279211838743e+03 + -1.186212138000000e+03 + -1.189150592852472e+03 + -1.192094584016226e+03 + -1.195044118757429e+03 + -1.197999203418656e+03 + -1.200959844999999e+03 + -1.203926050837031e+03 + -1.206897827704460e+03 + -1.209875182550288e+03 + -1.212858122751144e+03 + -1.215846655000000e+03 + -1.218840785789916e+03 + -1.221840522671824e+03 + -1.224845872987532e+03 + -1.227856843385819e+03 + -1.230873440999999e+03 + -1.233895673111333e+03 + -1.236923546192640e+03 + -1.239957067026041e+03 + -1.242996243170224e+03 + -1.246041082000000e+03 + -1.249091590572761e+03 + -1.252147775334870e+03 + -1.255209643154299e+03 + -1.258277201722480e+03 + -1.261350457999999e+03 + -1.264429418673061e+03 + -1.267514091394071e+03 + -1.270604483350415e+03 + -1.273700600740823e+03 + -1.276802451000000e+03 + -1.279910042060014e+03 + -1.283023380429426e+03 + -1.286142472688095e+03 + -1.289267326090673e+03 + -1.292397947999999e+03 + -1.295534345774196e+03 + -1.298676526619691e+03 + -1.301824497524618e+03 + -1.304978265240164e+03 + -1.308137837000000e+03 + -1.311303220289759e+03 + -1.314474422298625e+03 + -1.317651449999705e+03 + -1.320834310180745e+03 + -1.324023009999999e+03 + -1.327217556910311e+03 + -1.330417958531748e+03 + -1.333624221890711e+03 + -1.336836353161044e+03 + -1.340054360000000e+03 + -1.343278250622039e+03 + -1.346508031450468e+03 + -1.349743709083262e+03 + -1.352985291084887e+03 + -1.356232784999999e+03 + -1.359486198143600e+03 + -1.362745537187869e+03 + -1.366010809195683e+03 + -1.369282022032803e+03 + -1.372559183000000e+03 + -1.375842299046690e+03 + -1.379131377356324e+03 + -1.382426425045151e+03 + -1.385727449081319e+03 + -1.389034456999999e+03 + -1.392347456524483e+03 + -1.395666454635196e+03 + -1.398991458464414e+03 + -1.402322475610563e+03 + -1.405659513000000e+03 + -1.409002577344138e+03 + -1.412351676333446e+03 + -1.415706817605260e+03 + -1.419068008353351e+03 + -1.422435255999999e+03 + -1.425808567964313e+03 + -1.429187950969556e+03 + -1.432573411998037e+03 + -1.435964958719179e+03 + -1.439362599000000e+03 + -1.442766340439504e+03 + -1.446176189225631e+03 + -1.449592152360744e+03 + -1.453014238529275e+03 + -1.456442454999999e+03 + -1.459876808245435e+03 + -1.463317305593544e+03 + -1.466763954627232e+03 + -1.470216762966463e+03 + -1.473675738000000e+03 + -1.477140887005819e+03 + -1.480612217422037e+03 + -1.484089736511155e+03 + -1.487573451238554e+03 + -1.491063368999999e+03 + -1.494559497483912e+03 + -1.498061844364458e+03 + -1.501570417048641e+03 + -1.505085222548716e+03 + -1.508606267999999e+03 + -1.512133560790714e+03 + -1.515667108906144e+03 + -1.519206919999463e+03 + -1.522753000996140e+03 + -1.526305358999999e+03 + -1.529864001422905e+03 + -1.533428936276007e+03 + -1.537000171179230e+03 + -1.540577712963951e+03 + -1.544161568999999e+03 + -1.547751747018138e+03 + -1.551348254621497e+03 + -1.554951099280540e+03 + -1.558560288310355e+03 + -1.562175828999999e+03 + -1.565797728689906e+03 + -1.569425994958102e+03 + -1.573060635400619e+03 + -1.576701657553543e+03 + -1.580349068999999e+03 + -1.584002877314684e+03 + -1.587663089901193e+03 + -1.591329713983250e+03 + -1.595002756657312e+03 + -1.598682225999999e+03 + -1.602368130282464e+03 + -1.606060476003810e+03 + -1.609759270166310e+03 + -1.613464521194204e+03 + -1.617176236999999e+03 + -1.620894424965112e+03 + -1.624619091996676e+03 + -1.628350245392540e+03 + -1.632087893195996e+03 + -1.635832042999999e+03 + -1.639582702219653e+03 + -1.643339878880967e+03 + -1.647103580589531e+03 + -1.650873814125992e+03 + -1.654650586999999e+03 + -1.658433907212865e+03 + -1.662223782656940e+03 + -1.666020220927786e+03 + -1.669823229228366e+03 + -1.673632814999999e+03 + -1.677448985879331e+03 + -1.681271749582079e+03 + -1.685101113981527e+03 + -1.688937087086922e+03 + -1.692779675999999e+03 + -1.696628887589857e+03 + -1.700484730162859e+03 + -1.704347211664743e+03 + -1.708216339003609e+03 + -1.712092119999999e+03 + -1.715974562928047e+03 + -1.719863675285195e+03 + -1.723759464515762e+03 + -1.727661938269359e+03 + -1.731571103999999e+03 + -1.735486969207614e+03 + -1.739409542118944e+03 + -1.743338830603560e+03 + -1.747274841733617e+03 + -1.751217582999999e+03 + -1.755167062315964e+03 + -1.759123288009624e+03 + -1.763086267739076e+03 + -1.767056008086542e+03 + -1.771032516999999e+03 + -1.775015803098183e+03 + -1.779005873908525e+03 + -1.783002736756524e+03 + -1.787006399126180e+03 + -1.791016869000000e+03 + -1.795034154506543e+03 + -1.799058263107071e+03 + -1.803089202321619e+03 + -1.807126980015989e+03 + -1.811171603999998e+03 + -1.815223082025504e+03 + -1.819281421835726e+03 + -1.823346631247284e+03 + -1.827418718154363e+03 + -1.831497690000000e+03 + -1.835583554215247e+03 + -1.839676319355180e+03 + -1.843775993459518e+03 + -1.847882583413063e+03 + -1.851996096999998e+03 + -1.856116542493618e+03 + -1.860243927542824e+03 + -1.864378259929794e+03 + -1.868519547846306e+03 + -1.872667799000000e+03 + -1.876823020804590e+03 + -1.880985220849939e+03 + -1.885154407199114e+03 + -1.889330588466962e+03 + -1.893513771999998e+03 + -1.897703964771626e+03 + -1.901901175652285e+03 + -1.906105412964281e+03 + -1.910316683519642e+03 + -1.914534995000000e+03 + -1.918760355678869e+03 + -1.922992773607248e+03 + -1.927232256642219e+03 + -1.931478812455566e+03 + -1.935732448999998e+03 + -1.939993174351127e+03 + -1.944260996306766e+03 + -1.948535922544343e+03 + -1.952817960710870e+03 + -1.957107119000000e+03 + -1.961403405774432e+03 + -1.965706828656891e+03 + -1.970017395435863e+03 + -1.974335114379128e+03 + -1.978659992999998e+03 + -1.982992038634364e+03 + -1.987331259958815e+03 + -1.991677665406797e+03 + -1.996031262526477e+03 + -2.000392059000000e+03 + -2.004760062662544e+03 + -2.009135281420714e+03 + -2.013517723394651e+03 + -2.017907396963966e+03 + -2.022304309999998e+03 + -2.026708470129302e+03 + -2.031119885371980e+03 + -2.035538563783372e+03 + -2.039964513323241e+03 + -2.044397742000000e+03 + -2.048838257821666e+03 + -2.053286068652342e+03 + -2.057741182613919e+03 + -2.062203608203423e+03 + -2.066673352999998e+03 + -2.071150424296029e+03 + -2.075634830690161e+03 + -2.080126580708826e+03 + -2.084625682242698e+03 + -2.089132143000000e+03 + -2.093645970720743e+03 + -2.098167173603271e+03 + -2.102695759983049e+03 + -2.107231738184183e+03 + -2.111775115999998e+03 + -2.116325901057278e+03 + -2.120884101683680e+03 + -2.125449726108530e+03 + -2.130022782163747e+03 + -2.134603278000000e+03 + -2.139191221924691e+03 + -2.143786621957349e+03 + -2.148389485981318e+03 + -2.152999821836896e+03 + -2.157617637999998e+03 + -2.162242943074696e+03 + -2.166875744534069e+03 + -2.171516050262848e+03 + -2.176163869133622e+03 + -2.180819209000000e+03 + -2.185482077327040e+03 + -2.190152482871076e+03 + -2.194830433950524e+03 + -2.199515937814626e+03 + -2.204209002999998e+03 + -2.208909638535144e+03 + -2.213617851857877e+03 + -2.218333650781778e+03 + -2.223057044225780e+03 + -2.227788040000000e+03 + -2.232526645541351e+03 + -2.237272869895315e+03 + -2.242026721580949e+03 + -2.246788207788726e+03 + -2.251557336999999e+03 + -2.256334118250544e+03 + -2.261118559161676e+03 + -2.265910667352767e+03 + -2.270710451040453e+03 + -2.275517918999998e+03 + -2.280333080014267e+03 + -2.285155941593499e+03 + -2.289986511539142e+03 + -2.294824798563590e+03 + -2.299670810999998e+03 + -2.304524556927484e+03 + -2.309386044592275e+03 + -2.314255282195762e+03 + -2.319132277836808e+03 + -2.324017039999998e+03 + -2.328909577220459e+03 + -2.333809897224200e+03 + -2.338718008145149e+03 + -2.343633918994289e+03 + -2.348557637999998e+03 + -2.353489172912608e+03 + -2.358428531797252e+03 + -2.363375723128402e+03 + -2.368330755820120e+03 + -2.373293637999998e+03 + -2.378264377427878e+03 + -2.383242982526706e+03 + -2.388229461889857e+03 + -2.393223824063392e+03 + -2.398226076999998e+03 + -2.403236228525699e+03 + -2.408254287477974e+03 + -2.413280262494937e+03 + -2.418314161547432e+03 + -2.423355992999998e+03 + -2.428405765403690e+03 + -2.433463486890615e+03 + -2.438529165661329e+03 + -2.443602810190047e+03 + -2.448684428999998e+03 + -2.453774030522361e+03 + -2.458871622761939e+03 + -2.463977214083831e+03 + -2.469090813490053e+03 + -2.474212428999998e+03 + -2.479342068313477e+03 + -2.484479740528551e+03 + -2.489625454539379e+03 + -2.494779218375099e+03 + -2.499941039999998e+03 + -2.505110927624773e+03 + -2.510288890409285e+03 + -2.515474937052364e+03 + -2.520669075212765e+03 + -2.525871312999998e+03 + -2.531081658990217e+03 + -2.536300122220549e+03 + -2.541526711425658e+03 + -2.546761434689014e+03 + -2.552004299999998e+03 + -2.557255315555890e+03 + -2.562514490474060e+03 + -2.567781833501355e+03 + -2.573057352510476e+03 + -2.578341055999998e+03 + -2.583632952807378e+03 + -2.588933051312229e+03 + -2.594241359879810e+03 + -2.599557887014108e+03 + -2.604882640999998e+03 + -2.610215630196411e+03 + -2.615556863841929e+03 + -2.620906350516468e+03 + -2.626264097579904e+03 + -2.631630113999998e+03 + -2.637004409331177e+03 + -2.642386991059378e+03 + -2.647777867311749e+03 + -2.653177047878767e+03 + -2.658584540999998e+03 + -2.664000354190328e+03 + -2.669424496446296e+03 + -2.674856976816473e+03 + -2.680297803844182e+03 + -2.685746985999998e+03 + -2.691204531699670e+03 + -2.696670449221115e+03 + -2.702144747143579e+03 + -2.707627434527642e+03 + -2.713118519999998e+03 + -2.718618011850782e+03 + -2.724125918263655e+03 + -2.729642247949266e+03 + -2.735167010382318e+03 + -2.740700213999998e+03 + -2.746241866699575e+03 + -2.751791977076697e+03 + -2.757350554006517e+03 + -2.762917606479738e+03 + -2.768493143000000e+03 + -2.774077171851313e+03 + -2.779669701724049e+03 + -2.785270741295763e+03 + -2.790880299095185e+03 + -2.796498383999998e+03 + -2.802125004968091e+03 + -2.807760170344632e+03 + -2.813403888501788e+03 + -2.819056168106567e+03 + -2.824717018000000e+03 + -2.830386447079949e+03 + -2.836064464083062e+03 + -2.841751077483182e+03 + -2.847446295475182e+03 + -2.853150126999998e+03 + -2.858862581346520e+03 + -2.864583667200011e+03 + -2.870313392988722e+03 + -2.876051767015894e+03 + -2.881798798000000e+03 + -2.887554494917011e+03 + -2.893318866668325e+03 + -2.899091922031221e+03 + -2.904873669618204e+03 + -2.910664117999998e+03 + -2.916463275808826e+03 + -2.922271151990944e+03 + -2.928087755363771e+03 + -2.933913094445438e+03 + -2.939747178000000e+03 + -2.945590014959967e+03 + -2.951441614226611e+03 + -2.957301984588625e+03 + -2.963171134666888e+03 + -2.969049072999998e+03 + -2.974935808208847e+03 + -2.980831349417174e+03 + -2.986735705755709e+03 + -2.992648886132359e+03 + -2.998570899000000e+03 + -3.004501752659864e+03 + -3.010441455936329e+03 + -3.016390017858416e+03 + -3.022347447554316e+03 + -3.028313753999998e+03 + -3.034288946006182e+03 + -3.040273032141582e+03 + -3.046266021078663e+03 + -3.052267921739149e+03 + -3.058278743000000e+03 + -3.064298493698561e+03 + -3.070327182685398e+03 + -3.076364818827055e+03 + -3.082411411001809e+03 + -3.088466967999998e+03 + -3.094531498652569e+03 + -3.100605012177602e+03 + -3.106687517554494e+03 + -3.112779023262762e+03 + -3.118879538000000e+03 + -3.124989070695736e+03 + -3.131107630527278e+03 + -3.137235226491168e+03 + -3.143371867254447e+03 + -3.149517561999998e+03 + -3.155672319953097e+03 + -3.161836149115856e+03 + -3.168009058152651e+03 + -3.174191057129923e+03 + -3.180382155000000e+03 + -3.186582360055111e+03 + -3.192791681144167e+03 + -3.199010127274887e+03 + -3.205237707493432e+03 + -3.211474430999998e+03 + -3.217720306909038e+03 + -3.223975343611381e+03 + -3.230239549889151e+03 + -3.236512935355519e+03 + -3.242795509000000e+03 + -3.249087279482800e+03 + -3.255388255944051e+03 + -3.261698447489179e+03 + -3.268017862963467e+03 + -3.274346510999998e+03 + -3.280684400401313e+03 + -3.287031541121871e+03 + -3.293387942369432e+03 + -3.299753611888159e+03 + -3.306128559000000e+03 + -3.312512793688797e+03 + -3.318906324190636e+03 + -3.325309159228279e+03 + -3.331721308860119e+03 + -3.338142781999998e+03 + -3.344573586933190e+03 + -3.351013732692941e+03 + -3.357463228591956e+03 + -3.363922084041577e+03 + -3.370390307999997e+03 + -3.376867909193196e+03 + -3.383354896616435e+03 + -3.389851279509897e+03 + -3.396357067337695e+03 + -3.402872268999997e+03 + -3.409396893188545e+03 + -3.415930949268335e+03 + -3.422474446597307e+03 + -3.429027394238212e+03 + -3.435589800999997e+03 + -3.442161675674332e+03 + -3.448743027574159e+03 + -3.455333866086035e+03 + -3.461934200499129e+03 + -3.468544039999997e+03 + -3.475163393634110e+03 + -3.481792270131310e+03 + -3.488430678432703e+03 + -3.495078627927044e+03 + -3.501736127999997e+03 + -3.508403187927853e+03 + -3.515079816661691e+03 + -3.521766023349286e+03 + -3.528461817510494e+03 + -3.535167207999998e+03 + -3.541882203477436e+03 + -3.548606813620608e+03 + -3.555341047964333e+03 + -3.562084915431213e+03 + -3.568838424999997e+03 + -3.575601585773337e+03 + -3.582374407066678e+03 + -3.589156898185297e+03 + -3.595949068354316e+03 + -3.602750926999998e+03 + -3.609562483468940e+03 + -3.616383746264014e+03 + -3.623214724488114e+03 + -3.630055428392298e+03 + -3.636905866999997e+03 + -3.643766048793625e+03 + -3.650635983516456e+03 + -3.657515680680700e+03 + -3.664405149014039e+03 + -3.671304397999997e+03 + -3.678213437374473e+03 + -3.685132275760156e+03 + -3.692060922035369e+03 + -3.698999385901594e+03 + -3.705947676999997e+03 + -3.712905804797643e+03 + -3.719873778403308e+03 + -3.726851606752037e+03 + -3.733839298718909e+03 + -3.740836863999997e+03 + -3.747844312547997e+03 + -3.754861653211674e+03 + -3.761888895074585e+03 + -3.768926047955085e+03 + -3.775973120999997e+03 + -3.783030123074376e+03 + -3.790097063813473e+03 + -3.797173952745452e+03 + -3.804260798968512e+03 + -3.811357611999997e+03 + -3.818464401472386e+03 + -3.825581176303607e+03 + -3.832707945585785e+03 + -3.839844718954238e+03 + -3.846991505999998e+03 + -3.854148316201428e+03 + -3.861315158822798e+03 + -3.868492043129635e+03 + -3.875678978458987e+03 + -3.882875973999997e+03 + -3.890083038997269e+03 + -3.897300183310783e+03 + -3.904527416633095e+03 + -3.911764748154242e+03 + -3.919012186999997e+03 + -3.926269742452437e+03 + -3.933537424464352e+03 + -3.940815242621297e+03 + -3.948103205720294e+03 + -3.955401322999997e+03 + -3.962709604107895e+03 + -3.970028058998795e+03 + -3.977356697163350e+03 + -3.984695527326294e+03 + -3.992044558999997e+03 + -3.999403802144854e+03 + -4.006773266302151e+03 + -4.014152960820400e+03 + -4.021542894928240e+03 + -4.028943077999997e+03 + -4.036353519583599e+03 + -4.043774229513202e+03 + -4.051205217431756e+03 + -4.058646492564995e+03 + -4.066098063999997e+03 + -4.073559940995820e+03 + -4.081032133755226e+03 + -4.088514652104142e+03 + -4.096007504979870e+03 + -4.103510702000000e+03 + -4.111024253097677e+03 + -4.118548167510886e+03 + -4.126082454416708e+03 + -4.133627123225803e+03 + -4.141182184000000e+03 + -4.148747646853033e+03 + -4.156323520486682e+03 + -4.163909814098826e+03 + -4.171506538118098e+03 + -4.179113701999993e+03 + -4.186731314728609e+03 + -4.194359386189681e+03 + -4.201997926116393e+03 + -4.209646943720986e+03 + -4.217306449000000e+03 + -4.224976452122954e+03 + -4.232656961817923e+03 + -4.240347987240753e+03 + -4.248049538718602e+03 + -4.255761626000000e+03 + -4.263484258397591e+03 + -4.271217445246016e+03 + -4.278961196276451e+03 + -4.286715521743729e+03 + -4.294480431000000e+03 + -4.302255933066315e+03 + -4.310042038085233e+03 + -4.317838756007569e+03 + -4.325646096087124e+03 + -4.333464067999995e+03 + -4.341292681602012e+03 + -4.349131946225953e+03 + -4.356981871466000e+03 + -4.364842467485710e+03 + -4.372713744000000e+03 + -4.380595710372718e+03 + -4.388488375861229e+03 + -4.396391750264422e+03 + -4.404305844160006e+03 + -4.412230667000000e+03 + -4.420166227736474e+03 + -4.428112536405837e+03 + -4.436069603088062e+03 + -4.444037437481978e+03 + -4.452016049000000e+03 + -4.460005447074678e+03 + -4.468005641867199e+03 + -4.476016643204103e+03 + -4.484038460200805e+03 + -4.492071102999995e+03 + -4.500114582060231e+03 + -4.508168906299815e+03 + -4.516234085098625e+03 + -4.524310129075975e+03 + -4.532397048000000e+03 + -4.540494851104932e+03 + -4.548603547974418e+03 + -4.556723148595408e+03 + -4.564853663376500e+03 + -4.572995102000000e+03 + -4.581147473755137e+03 + -4.589310788334117e+03 + -4.597485055595826e+03 + -4.605670285497996e+03 + -4.613866488000000e+03 + -4.622073673035890e+03 + -4.630291850451470e+03 + -4.638521029781397e+03 + -4.646761220207367e+03 + -4.655012431999994e+03 + -4.663274675842531e+03 + -4.671547961173437e+03 + -4.679832297267551e+03 + -4.688127693710799e+03 + -4.696434161000000e+03 + -4.704751709752170e+03 + -4.713080348793824e+03 + -4.721420087561993e+03 + -4.729770937031085e+03 + -4.738132907000000e+03 + -4.746506006604411e+03 + -4.754890245715343e+03 + -4.763285634540516e+03 + -4.771692183448750e+03 + -4.780109901999994e+03 + -4.788538799466528e+03 + -4.796978886096308e+03 + -4.805430172162917e+03 + -4.813892667577567e+03 + -4.822366381999994e+03 + -4.830851325088147e+03 + -4.839347507060687e+03 + -4.847854938123849e+03 + -4.856373628221604e+03 + -4.864903587000000e+03 + -4.873444824101478e+03 + -4.881997349859994e+03 + -4.890561174364984e+03 + -4.899136307099045e+03 + -4.907722758000000e+03 + -4.916320537362784e+03 + -4.924929655587317e+03 + -4.933550122528648e+03 + -4.942181947260091e+03 + -4.950825139999994e+03 + -4.959479711457802e+03 + -4.968145671175524e+03 + -4.976823028809056e+03 + -4.985511794647833e+03 + -4.994211978999993e+03 + -5.002923592007461e+03 + -5.011646643245516e+03 + -5.020381142480443e+03 + -5.029127099988444e+03 + -5.037884526000000e+03 + -5.046653430641881e+03 + -5.055433823857634e+03 + -5.064225715527905e+03 + -5.073029115529123e+03 + -5.081844034000000e+03 + -5.090670481166524e+03 + -5.099508466925161e+03 + -5.108358001180135e+03 + -5.117219094017154e+03 + -5.126091755999993e+03 + -5.134975997626908e+03 + -5.143871827953957e+03 + -5.152779256704796e+03 + -5.161698295082459e+03 + -5.170628952999993e+03 + -5.179571239778229e+03 + -5.188525166029071e+03 + -5.197490742130520e+03 + -5.206467977662855e+03 + -5.215456883000000e+03 + -5.224457468752808e+03 + -5.233469744262066e+03 + -5.242493719420175e+03 + -5.251529405361101e+03 + -5.260576812000000e+03 + -5.269635948683524e+03 + -5.278706825911151e+03 + -5.287789454172008e+03 + -5.296883843495917e+03 + -5.305990003999993e+03 + -5.315107945871962e+03 + -5.324237679253967e+03 + -5.333379214299059e+03 + -5.342532561186867e+03 + -5.351697729999993e+03 + -5.360874730862462e+03 + -5.370063574314554e+03 + -5.379264270604972e+03 + -5.388476829418795e+03 + -5.397701261000000e+03 + -5.406937575887649e+03 + -5.416185784228891e+03 + -5.425445896082409e+03 + -5.434717921547790e+03 + -5.444001871000000e+03 + -5.453297754883592e+03 + -5.462605583222671e+03 + -5.471925366229407e+03 + -5.481257114511772e+03 + -5.490600837999993e+03 + -5.499956546429602e+03 + -5.509324250592133e+03 + -5.518703961068116e+03 + -5.528095687743725e+03 + -5.537499440999993e+03 + -5.546915231361128e+03 + -5.556343068498348e+03 + -5.565782962476614e+03 + -5.575234924238452e+03 + -5.584698964000000e+03 + -5.594175091687396e+03 + -5.603663318133420e+03 + -5.613163653827974e+03 + -5.622676108416792e+03 + -5.632200692000000e+03 + -5.641737415094019e+03 + -5.651286288507457e+03 + -5.660847322604374e+03 + -5.670420527008017e+03 + -5.680005911999993e+03 + -5.689603488367688e+03 + -5.699213267038800e+03 + -5.708835258184300e+03 + -5.718469470901945e+03 + -5.728115915999992e+03 + -5.737774605002216e+03 + -5.747445547634946e+03 + -5.757128753652110e+03 + -5.766824233562706e+03 + -5.776531998000000e+03 + -5.786252057564687e+03 + -5.795984422575620e+03 + -5.805729103393980e+03 + -5.815486110517342e+03 + -5.825255454000000e+03 + -5.835037143865786e+03 + -5.844831191184440e+03 + -5.854637606668529e+03 + -5.864456400127173e+03 + -5.874287581999993e+03 + -5.884131163070459e+03 + -5.893987153665426e+03 + -5.903855564090225e+03 + -5.913736404781277e+03 + -5.923629686000000e+03 + -5.933535418061928e+03 + -5.943453611973397e+03 + -5.953384278439786e+03 + -5.963327427438455e+03 + -5.973283069000000e+03 + -5.983251213525049e+03 + -5.993231872573540e+03 + -6.003225056744913e+03 + -6.013230774891098e+03 + -6.023249038000000e+03 + -6.033279857947543e+03 + -6.043323244258373e+03 + -6.053379206808452e+03 + -6.063447756872029e+03 + -6.073528904999992e+03 + -6.083622661322729e+03 + -6.093729036471069e+03 + -6.103848041095609e+03 + -6.113979685699232e+03 + -6.124123981000000e+03 + -6.134280937646101e+03 + -6.144450565430006e+03 + -6.154632874755873e+03 + -6.164827877191976e+03 + -6.175035583000000e+03 + -6.185256001935748e+03 + -6.195489145360168e+03 + -6.205735024246584e+03 + -6.215993648382001e+03 + -6.226265028000000e+03 + -6.236549173839082e+03 + -6.246846097225249e+03 + -6.257155808917487e+03 + -6.267478318647532e+03 + -6.277813636999993e+03 + -6.288161775039973e+03 + -6.298522743318929e+03 + -6.308896552332133e+03 + -6.319283212690151e+03 + -6.329682735000000e+03 + -6.340095129841887e+03 + -6.350520407726451e+03 + -6.360958579315570e+03 + -6.371409655512612e+03 + -6.381873647000000e+03 + -6.392350564293673e+03 + -6.402840417873253e+03 + -6.413343218449801e+03 + -6.423858977046060e+03 + -6.434387704000000e+03 + -6.444929409479182e+03 + -6.455484104800415e+03 + -6.466051800878468e+03 + -6.476632507643834e+03 + -6.487226235999992e+03 + -6.497832997260840e+03 + -6.508452801649206e+03 + -6.519085659445890e+03 + -6.529731581482401e+03 + -6.540390579000000e+03 + -6.551062663116751e+03 + -6.561747843536938e+03 + -6.572446130639311e+03 + -6.583157536279329e+03 + -6.593882071000000e+03 + -6.604619744763902e+03 + -6.615370568925604e+03 + -6.626134554413510e+03 + -6.636911711064461e+03 + -6.647702049999992e+03 + -6.658505582840396e+03 + -6.669322319643610e+03 + -6.680152270785791e+03 + -6.690995447680221e+03 + -6.701851860999993e+03 + -6.712721521095499e+03 + -6.723604439166686e+03 + -6.734500626205399e+03 + -6.745410092579864e+03 + -6.756332849000000e+03 + -6.767268906377159e+03 + -6.778218275408767e+03 + -6.789180966973428e+03 + -6.800156992270870e+03 + -6.811146362000000e+03 + -6.822149086596000e+03 + -6.833165176809022e+03 + -6.844194643583943e+03 + -6.855237498022965e+03 + -6.866293750999992e+03 + -6.877363413246921e+03 + -6.888446495554016e+03 + -6.899543008749155e+03 + -6.910652963678518e+03 + -6.921776370999992e+03 + -6.932913241460104e+03 + -6.944063586638155e+03 + -6.955227417524035e+03 + -6.966404743973952e+03 + -6.977595577000000e+03 + -6.988799928187199e+03 + -7.000017808173458e+03 + -7.011249227729020e+03 + -7.022494198140058e+03 + -7.033752730000000e+03 + -7.045024833685246e+03 + -7.056310520605544e+03 + -7.067609802022496e+03 + -7.078922688575014e+03 + -7.090249190999992e+03 + -7.101589320140114e+03 + -7.112943086874141e+03 + -7.124310502404821e+03 + -7.135691578318693e+03 + -7.147086324999992e+03 + -7.158494752468790e+03 + -7.169916872475182e+03 + -7.181352696510882e+03 + -7.192802235036809e+03 + -7.204265499000000e+03 + -7.215742499578333e+03 + -7.227233247391960e+03 + -7.238737753333357e+03 + -7.250256028887258e+03 + -7.261788085000000e+03 + -7.273333932305611e+03 + -7.284893581718426e+03 + -7.296467044458082e+03 + -7.308054332025791e+03 + -7.319655454999992e+03 + -7.331270423697963e+03 + -7.342899249833889e+03 + -7.354541944862027e+03 + -7.366198519315069e+03 + -7.377868983999992e+03 + -7.389553350032372e+03 + -7.401251628839381e+03 + -7.412963831476617e+03 + -7.424689968390403e+03 + -7.436430051000000e+03 + -7.448184091042194e+03 + -7.459952098877861e+03 + -7.471734085033887e+03 + -7.483530060856024e+03 + -7.495340038000000e+03 + -7.507164027981260e+03 + -7.519002041143363e+03 + -7.530854088088799e+03 + -7.542720180263871e+03 + -7.554600328999991e+03 + -7.566494545476364e+03 + -7.578402840747203e+03 + -7.590325225748156e+03 + -7.602261711334166e+03 + -7.614212308999990e+03 + -7.626177030375796e+03 + -7.638155885970103e+03 + -7.650148886679111e+03 + -7.662156044359298e+03 + -7.674177370000000e+03 + -7.686212874133715e+03 + -7.698262567927072e+03 + -7.710326462858042e+03 + -7.722404570574414e+03 + -7.734496902000000e+03 + -7.746603467726753e+03 + -7.758724278933637e+03 + -7.770859347070250e+03 + -7.783008683730720e+03 + -7.795172299999991e+03 + -7.807350206747154e+03 + -7.819542415323729e+03 + -7.831748936928582e+03 + -7.843969782379609e+03 + -7.856204962999991e+03 + -7.868454490389281e+03 + -7.880718375856472e+03 + -7.892996630459527e+03 + -7.905289265033489e+03 + -7.917596291000000e+03 + -7.929917719977670e+03 + -7.942253562811284e+03 + -7.954603830544003e+03 + -7.966968534806996e+03 + -7.979347687000000e+03 + -7.991741298275563e+03 + -8.004149379532312e+03 + -8.016571941875890e+03 + -8.029008996834256e+03 + -8.041460555999991e+03 + -8.053926630829063e+03 + -8.066407232164392e+03 + -8.078902371093302e+03 + -8.091412059283156e+03 + -8.103936307999991e+03 + -8.116475128348727e+03 + -8.129028531951320e+03 + -8.141596530160446e+03 + -8.154179133753471e+03 + -8.166776354000000e+03 + -8.179388202500601e+03 + -8.192014690774517e+03 + -8.204655830155478e+03 + -8.217311631752342e+03 + -8.229982107000000e+03 + -8.242667267443709e+03 + -8.255367124181290e+03 + -8.268081688506129e+03 + -8.280810972124258e+03 + -8.293554985999990e+03 + -8.306313740930424e+03 + -8.319087249046417e+03 + -8.331875522237864e+03 + -8.344678571514016e+03 + -8.357496408000001e+03 + -8.370329042981793e+03 + -8.383176487893126e+03 + -8.396038754236946e+03 + -8.408915853538949e+03 + -8.421807796999999e+03 + -8.434714595760075e+03 + -8.447636261558551e+03 + -8.460572806112319e+03 + -8.473524240852536e+03 + -8.486490577000000e+03 + -8.499471825734363e+03 + -8.512467998550057e+03 + -8.525479106914012e+03 + -8.538505162160651e+03 + -8.551546175999989e+03 + -8.564602160205039e+03 + -8.577673125806519e+03 + -8.590759084111069e+03 + -8.603860047081474e+03 + -8.616976026000000e+03 + -8.630107031891077e+03 + -8.643253076647045e+03 + -8.656414171854482e+03 + -8.669590328370747e+03 + -8.682781558000001e+03 + -8.695987872906602e+03 + -8.709209284090668e+03 + -8.722445802791402e+03 + -8.735697441007769e+03 + -8.748964210000000e+03 + -8.762246120773598e+03 + -8.775543185390141e+03 + -8.788855415760128e+03 + -8.802182823160678e+03 + -8.815525418999991e+03 + -8.828883214778334e+03 + -8.842256221886048e+03 + -8.855644451817980e+03 + -8.869047916276779e+03 + -8.882466627000000e+03 + -8.895900595656447e+03 + -8.909349833599377e+03 + -8.922814352320849e+03 + -8.936294163622882e+03 + -8.949789279000001e+03 + -8.963299709806102e+03 + -8.976825467697834e+03 + -8.990366564425331e+03 + -9.003923011733321e+03 + -9.017494820999989e+03 + -9.031082003570244e+03 + -9.044684571593196e+03 + -9.058302536801553e+03 + -9.071935910043938e+03 + -9.085584702999991e+03 + -9.099248927805194e+03 + -9.112928596071970e+03 + -9.126623719398038e+03 + -9.140334309541779e+03 + -9.154060378000000e+03 + -9.167801936143134e+03 + -9.181558995552323e+03 + -9.195331568052554e+03 + -9.209119665711642e+03 + -9.222923300000000e+03 + -9.236742482191501e+03 + -9.250577224366738e+03 + -9.264427538348678e+03 + -9.278293435300837e+03 + -9.292174926999989e+03 + -9.306072025514850e+03 + -9.319984742353185e+03 + -9.333913089239737e+03 + -9.347857078372264e+03 + -9.361816720999988e+03 + -9.375792028076301e+03 + -9.389783011923670e+03 + -9.403789684689073e+03 + -9.417812057757259e+03 + -9.431850143000000e+03 + -9.445903952452922e+03 + -9.459973497383473e+03 + -9.474058789280491e+03 + -9.488159840242311e+03 + -9.502276662000000e+03 + -9.516409266099370e+03 + -9.530557664431180e+03 + -9.544721868948835e+03 + -9.558901891554557e+03 + -9.573097743999990e+03 + -9.587309437907974e+03 + -9.601536984771086e+03 + -9.615780396443668e+03 + -9.630039685320193e+03 + -9.644314862999989e+03 + -9.658605940694017e+03 + -9.672912930240365e+03 + -9.687235843732506e+03 + -9.701574693379191e+03 + -9.715929491000001e+03 + -9.730300248146514e+03 + -9.744686976337332e+03 + -9.759089687418043e+03 + -9.773508393705906e+03 + -9.787943106999999e+03 + -9.802393838806964e+03 + -9.816860600913731e+03 + -9.831343405392694e+03 + -9.845842264577648e+03 + -9.860357189999990e+03 + -9.874888192875391e+03 + -9.889435285292540e+03 + -9.903998479577405e+03 + -9.918577788007758e+03 + -9.933173221999989e+03 + -9.947784792760667e+03 + -9.962412512866247e+03 + -9.977056394630243e+03 + -9.991716449514770e+03 + -1.000639269000000e+04 + -1.002108512827329e+04 + -1.003579377260607e+04 + -1.005051863510592e+04 + -1.006525973450424e+04 + -1.008001708000000e+04 + -1.009479067738785e+04 + -1.010958054488143e+04 + -1.012438669671707e+04 + -1.013920913701616e+04 + -1.015404787999999e+04 + -1.016890294387830e+04 + -1.018377433451942e+04 + -1.019866206104612e+04 + -1.021356614193473e+04 + -1.022848658999999e+04 + -1.024342341422066e+04 + -1.025837662498870e+04 + -1.027334623400460e+04 + -1.028833225445739e+04 + -1.030333470000000e+04 + -1.031835358368700e+04 + -1.033338891534757e+04 + -1.034844070623467e+04 + -1.036350897076085e+04 + -1.037859372000000e+04 + -1.039369496373519e+04 + -1.040881271589677e+04 + -1.042394698923176e+04 + -1.043909779325773e+04 + -1.045426513999999e+04 + -1.046944904320307e+04 + -1.048464951626011e+04 + -1.049986657147346e+04 + -1.051510021951631e+04 + -1.053035046999999e+04 + -1.054561733346490e+04 + -1.056090082649770e+04 + -1.057620096382862e+04 + -1.059151775501402e+04 + -1.060685121000000e+04 + -1.062220133982168e+04 + -1.063756815769902e+04 + -1.065295167679853e+04 + -1.066835190951487e+04 + -1.068376887000000e+04 + -1.069920257172756e+04 + -1.071465302083269e+04 + -1.073012022736731e+04 + -1.074560420973240e+04 + -1.076110497999999e+04 + -1.077662254698522e+04 + -1.079215692468130e+04 + -1.080770812602064e+04 + -1.082327616047952e+04 + -1.083886103999999e+04 + -1.085446277832030e+04 + -1.087008138917964e+04 + -1.088571688451933e+04 + -1.090136927381244e+04 + -1.091703857000000e+04 + -1.093272478707855e+04 + -1.094842793381667e+04 + -1.096414802233435e+04 + -1.097988507119364e+04 + -1.099563909000000e+04 + -1.101141008471468e+04 + -1.102719807131307e+04 + -1.104300306633410e+04 + -1.105882508291791e+04 + -1.107466412999999e+04 + -1.109052021511841e+04 + -1.110639335038349e+04 + -1.112228355101184e+04 + -1.113819083451417e+04 + -1.115411520999999e+04 + -1.117005668368749e+04 + -1.118601527242516e+04 + -1.120199099177015e+04 + -1.121798385114509e+04 + -1.123399386000000e+04 + -1.125002102929127e+04 + -1.126606537454743e+04 + -1.128212691149971e+04 + -1.129820565394142e+04 + -1.131430161000000e+04 + -1.133041478676655e+04 + -1.134654520134832e+04 + -1.136269286880464e+04 + -1.137885779750048e+04 + -1.139503999999999e+04 + -1.141123949060806e+04 + -1.142745627830015e+04 + -1.144369037489935e+04 + -1.145994179819328e+04 + -1.147621056000000e+04 + -1.149249666929363e+04 + -1.150880014040390e+04 + -1.152512098586926e+04 + -1.154145921396878e+04 + -1.155781484000000e+04 + -1.157418788116290e+04 + -1.159057834325663e+04 + -1.160698623573862e+04 + -1.162341157750990e+04 + -1.163985437999999e+04 + -1.165631465167572e+04 + -1.167279241049137e+04 + -1.168928767028212e+04 + -1.170580043559586e+04 + -1.172233071999999e+04 + -1.173887854137991e+04 + -1.175544390951204e+04 + -1.177202683447122e+04 + -1.178862732998528e+04 + -1.180524541000000e+04 + -1.182188108744826e+04 + -1.183853437152247e+04 + -1.185520527417456e+04 + -1.187189381268968e+04 + -1.188860000000000e+04 + -1.190532384555031e+04 + -1.192206535739879e+04 + -1.193882454957090e+04 + -1.195560144455150e+04 + -1.197239604999999e+04 + -1.198920836801576e+04 + -1.200603841862060e+04 + -1.202288622005816e+04 + -1.203975178087160e+04 + -1.205663510999999e+04 + -1.207353621858988e+04 + -1.209045512337030e+04 + -1.210739183876488e+04 + -1.212434637372514e+04 + -1.214131874000000e+04 + -1.215830895146994e+04 + -1.217531702204386e+04 + -1.219234296344827e+04 + -1.220938678454126e+04 + -1.222644850000000e+04 + -1.224352812665298e+04 + -1.226062567426862e+04 + -1.227774115320455e+04 + -1.229487457740941e+04 + -1.231202595999999e+04 + -1.232919531361497e+04 + -1.234638265190763e+04 + -1.236358798719477e+04 + -1.238081132944854e+04 + -1.239805268999999e+04 + -1.241531208224867e+04 + -1.243258952373459e+04 + -1.244988502746729e+04 + -1.246719859854622e+04 + -1.248453025000000e+04 + -1.250187999937352e+04 + -1.251924785997314e+04 + -1.253663384322633e+04 + -1.255403795942784e+04 + -1.257146022000000e+04 + -1.258890063824787e+04 + -1.260635923174130e+04 + -1.262383601348307e+04 + -1.264133098848631e+04 + -1.265884416999999e+04 + -1.267637557565735e+04 + -1.269392521755929e+04 + -1.271149310837373e+04 + -1.272907926320349e+04 + -1.274668368999999e+04 + -1.276430639533002e+04 + -1.278194739917555e+04 + -1.279960671868295e+04 + -1.281728436173200e+04 + -1.283498034000000e+04 + -1.285269466774105e+04 + -1.287042735763161e+04 + -1.288817842259290e+04 + -1.290594787633268e+04 + -1.292373573000000e+04 + -1.294154199365508e+04 + -1.295936667981993e+04 + -1.297720980264566e+04 + -1.299507137776778e+04 + -1.301295141999999e+04 + -1.303084994203483e+04 + -1.304876695061188e+04 + -1.306670245535235e+04 + -1.308465647270176e+04 + -1.310262901999999e+04 + -1.312062011245361e+04 + -1.313862975604251e+04 + -1.315665796055223e+04 + -1.317470474462057e+04 + -1.319277012000000e+04 + -1.321085409533674e+04 + -1.322895668661698e+04 + -1.324707790969205e+04 + -1.326521777718879e+04 + -1.328337630000000e+04 + -1.330155348851537e+04 + -1.331974935491385e+04 + -1.333796391362056e+04 + -1.335619718140170e+04 + -1.337444916999999e+04 + -1.339271988874384e+04 + -1.341100935082290e+04 + -1.342931756987852e+04 + -1.344764455872234e+04 + -1.346599032999999e+04 + -1.348435489664480e+04 + -1.350273827293541e+04 + -1.352114047045672e+04 + -1.353956149695641e+04 + -1.355800137000000e+04 + -1.357646010972569e+04 + -1.359493772045508e+04 + -1.361343421245217e+04 + -1.363194960986463e+04 + -1.365048392000000e+04 + -1.366903714425879e+04 + -1.368760930691950e+04 + -1.370620042690972e+04 + -1.372481050688227e+04 + -1.374343955999999e+04 + -1.376208760473736e+04 + -1.378075465005087e+04 + -1.379944070672178e+04 + -1.381814579185646e+04 + -1.383686991999999e+04 + -1.385561310323773e+04 + -1.387437535187516e+04 + -1.389315667691767e+04 + -1.391195709130158e+04 + -1.393077661000000e+04 + -1.394961524865979e+04 + -1.396847302075987e+04 + -1.398734993803961e+04 + -1.400624601054151e+04 + -1.402516125000000e+04 + -1.404409566989268e+04 + -1.406304928589068e+04 + -1.408202211294199e+04 + -1.410101416389908e+04 + -1.412002544999999e+04 + -1.413905598164077e+04 + -1.415810576964489e+04 + -1.417717482985645e+04 + -1.419626318438561e+04 + -1.421537083999999e+04 + -1.423449779870650e+04 + -1.425364408441530e+04 + -1.427280971628262e+04 + -1.429199469812906e+04 + -1.431119904000000e+04 + -1.433042275698022e+04 + -1.434966586475864e+04 + -1.436892837784737e+04 + -1.438821030874347e+04 + -1.440751167000000e+04 + -1.442683247464153e+04 + -1.444617273693482e+04 + -1.446553246845352e+04 + -1.448491167696990e+04 + -1.450431037999999e+04 + -1.452372859766022e+04 + -1.454316633455064e+04 + -1.456262360064903e+04 + -1.458210041914065e+04 + -1.460159679999998e+04 + -1.462111274782618e+04 + -1.464064828275210e+04 + -1.466020342227678e+04 + -1.467977817411577e+04 + -1.469937255000000e+04 + -1.471898656425745e+04 + -1.473862022905967e+04 + -1.475827355783788e+04 + -1.477794656667906e+04 + -1.479763927000000e+04 + -1.481735168022591e+04 + -1.483708380678435e+04 + -1.485683566151257e+04 + -1.487660726097236e+04 + -1.489639861999999e+04 + -1.491620975095499e+04 + -1.493604066191429e+04 + -1.495589136614776e+04 + -1.497576188555746e+04 + -1.499565223000000e+04 + -1.501556240451360e+04 + -1.503549242791256e+04 + -1.505544231896391e+04 + -1.507541209051985e+04 + -1.509540175000000e+04 + -1.511541130465759e+04 + -1.513544077375174e+04 + -1.515549017500820e+04 + -1.517555951933053e+04 + -1.519564881999999e+04 + -1.521575809081157e+04 + -1.523588734003694e+04 + -1.525603658036919e+04 + -1.527620583277191e+04 + -1.529639510999998e+04 + -1.531660442034102e+04 + -1.533683377728779e+04 + -1.535708319443415e+04 + -1.537735268372974e+04 + -1.539764226000000e+04 + -1.541795193921473e+04 + -1.543828173411788e+04 + -1.545863165713528e+04 + -1.547900172144165e+04 + -1.549939194000000e+04 + -1.551980232569260e+04 + -1.554023289178161e+04 + -1.556068365093051e+04 + -1.558115461506463e+04 + -1.560164579999999e+04 + -1.562215722197032e+04 + -1.564268888875843e+04 + -1.566324081295743e+04 + -1.568381301679686e+04 + -1.570440550999999e+04 + -1.572501829745089e+04 + -1.574565139909988e+04 + -1.576630483256334e+04 + -1.578697860613842e+04 + -1.580767273000000e+04 + -1.582838721786272e+04 + -1.584912209052793e+04 + -1.586987736246908e+04 + -1.589065303680525e+04 + -1.591144913000000e+04 + -1.593226566336701e+04 + -1.595310264072689e+04 + -1.597396007256353e+04 + -1.599483798518879e+04 + -1.601573638999999e+04 + -1.603665529105288e+04 + -1.605759470511345e+04 + -1.607855464838671e+04 + -1.609953513161670e+04 + -1.612053616999998e+04 + -1.614155778004294e+04 + -1.616259997081388e+04 + -1.618366275348782e+04 + -1.620474614514243e+04 + -1.622585016000000e+04 + -1.624697481007329e+04 + -1.626812010728422e+04 + -1.628928606592670e+04 + -1.631047270337715e+04 + -1.633168003000000e+04 + -1.635290805454565e+04 + -1.637415679800609e+04 + -1.639542627592296e+04 + -1.641671649164588e+04 + -1.643802745999998e+04 + -1.645935920179283e+04 + -1.648071172935930e+04 + -1.650208505431875e+04 + -1.652347919039644e+04 + -1.654489414999998e+04 + -1.656632994566724e+04 + -1.658778659427021e+04 + -1.660926410974226e+04 + -1.663076250029903e+04 + -1.665228178000000e+04 + -1.667382196574960e+04 + -1.669538306946903e+04 + -1.671696510420377e+04 + -1.673856808645374e+04 + -1.676019203000000e+04 + -1.678183694695306e+04 + -1.680350285053965e+04 + -1.682518975366077e+04 + -1.684689766850574e+04 + -1.686862660999998e+04 + -1.689037659405139e+04 + -1.691214763326874e+04 + -1.693393974023558e+04 + -1.695575292860696e+04 + -1.697758720999999e+04 + -1.699944259677470e+04 + -1.702131910964586e+04 + -1.704321676352935e+04 + -1.706513556209216e+04 + -1.708707552000000e+04 + -1.710903665790321e+04 + -1.713101898938551e+04 + -1.715302252479822e+04 + -1.717504727292061e+04 + -1.719709325000000e+04 + -1.721916047545785e+04 + -1.724124896174670e+04 + -1.726335872072246e+04 + -1.728548976595536e+04 + -1.730764210999998e+04 + -1.732981576548871e+04 + -1.735201074834581e+04 + -1.737422707347338e+04 + -1.739646475299950e+04 + -1.741872379999999e+04 + -1.744100422776899e+04 + -1.746330604747148e+04 + -1.748562927386205e+04 + -1.750797392735042e+04 + -1.753034002000000e+04 + -1.755272756012299e+04 + -1.757513656385630e+04 + -1.759756704698003e+04 + -1.762001902165879e+04 + -1.764249250000000e+04 + -1.766498749503509e+04 + -1.768750402279969e+04 + -1.771004209797560e+04 + -1.773260173223158e+04 + -1.775518293999998e+04 + -1.777778573725219e+04 + -1.780041013823985e+04 + -1.782305615380817e+04 + -1.784572379125812e+04 + -1.786841306999998e+04 + -1.789112401297027e+04 + -1.791385662520795e+04 + -1.793661091551944e+04 + -1.795938690506571e+04 + -1.798218461000000e+04 + -1.800500404157250e+04 + -1.802784520726593e+04 + -1.805070812126097e+04 + -1.807359280836476e+04 + -1.809649928000000e+04 + -1.811942754077938e+04 + -1.814237760512274e+04 + -1.816534949154946e+04 + -1.818834322012563e+04 + -1.821135879999998e+04 + -1.823439623670011e+04 + -1.825745554979138e+04 + -1.828053675712656e+04 + -1.830363986886144e+04 + -1.832676489999998e+04 + -1.834991186721417e+04 + -1.837308077964562e+04 + -1.839627164834952e+04 + -1.841948449011883e+04 + -1.844271932000000e+04 + -1.846597615248214e+04 + -1.848925500551537e+04 + -1.851255589141731e+04 + -1.853587881367952e+04 + -1.855922379000000e+04 + -1.858259084403384e+04 + -1.860597998459446e+04 + -1.862939122130964e+04 + -1.865282457086503e+04 + -1.867628004999998e+04 + -1.869975767367864e+04 + -1.872325745125265e+04 + -1.874677939400045e+04 + -1.877032351829797e+04 + -1.879388983999998e+04 + -1.881747837395618e+04 + -1.884108913346766e+04 + -1.886472213066709e+04 + -1.888837737699991e+04 + -1.891205489000000e+04 + -1.893575468867888e+04 + -1.895947678210836e+04 + -1.898322118083019e+04 + -1.900698790146692e+04 + -1.903077696000000e+04 + -1.905458837122149e+04 + -1.907842214817702e+04 + -1.910227830328199e+04 + -1.912615684887913e+04 + -1.915005779999998e+04 + -1.917398117252360e+04 + -1.919792697876641e+04 + -1.922189523162585e+04 + -1.924588594641727e+04 + -1.926989913999998e+04 + -1.929393482787026e+04 + -1.931799301676958e+04 + -1.934207372057390e+04 + -1.936617696590223e+04 + -1.939030276000000e+04 + -1.941445110358184e+04 + -1.943862202401437e+04 + -1.946281554213054e+04 + -1.948703165938676e+04 + -1.951127038999998e+04 + -1.953553175531375e+04 + -1.955981576784185e+04 + -1.958412243887696e+04 + -1.960845178142548e+04 + -1.963280380999998e+04 + -1.965717854010317e+04 + -1.968157598764983e+04 + -1.970599616682758e+04 + -1.973043908928781e+04 + -1.975490477000000e+04 + -1.977939322508017e+04 + -1.980390446612710e+04 + -1.982843850672890e+04 + -1.985299536464853e+04 + -1.987757505000000e+04 + -1.990217757132781e+04 + -1.992680295149982e+04 + -1.995145120910892e+04 + -1.997612235106852e+04 + -2.000081638999998e+04 + -2.002553334291399e+04 + -2.005027322692212e+04 + -2.007503605456619e+04 + -2.009982183242756e+04 + -2.012463057999998e+04 + -2.014946232050470e+04 + -2.017431705750974e+04 + -2.019919480128925e+04 + -2.022409557869796e+04 + -2.024901940000000e+04 + -2.027396626876084e+04 + -2.029893620825063e+04 + -2.032392923791251e+04 + -2.034894536415857e+04 + -2.037398460000000e+04 + -2.039904696273123e+04 + -2.042413246681671e+04 + -2.044924112749899e+04 + -2.047437296177629e+04 + -2.049952797999998e+04 + -2.052470619105544e+04 + -2.054990761544010e+04 + -2.057513226917888e+04 + -2.060038015795331e+04 + -2.062565129999998e+04 + -2.065094571854568e+04 + -2.067626342205840e+04 + -2.070160442063917e+04 + -2.072696873225083e+04 + -2.075235637000000e+04 + -2.077776734528838e+04 + -2.080320167696219e+04 + -2.082865938214449e+04 + -2.085414047244347e+04 + -2.087964496000000e+04 + -2.090517285805777e+04 + -2.093072418166662e+04 + -2.095629894648643e+04 + -2.098189716820891e+04 + -2.100751885999998e+04 + -2.103316403410750e+04 + -2.105883270565323e+04 + -2.108452489048020e+04 + -2.111024060421226e+04 + -2.113597985999998e+04 + -2.116174267011207e+04 + -2.118752904964539e+04 + -2.121333901446366e+04 + -2.123917258027344e+04 + -2.126502976000000e+04 + -2.129091056582378e+04 + -2.131681501406569e+04 + -2.134274311932949e+04 + -2.136869489244911e+04 + -2.139467035000000e+04 + -2.142066951097057e+04 + -2.144669238829770e+04 + -2.147273899344209e+04 + -2.149880933837191e+04 + -2.152490343999998e+04 + -2.155102131698669e+04 + -2.157716298229367e+04 + -2.160332844744082e+04 + -2.162951772437019e+04 + -2.165573082999998e+04 + -2.168196778300156e+04 + -2.170822859627157e+04 + -2.173451328139519e+04 + -2.176082185053812e+04 + -2.178715432000000e+04 + -2.181351070821111e+04 + -2.183989103150391e+04 + -2.186629530288904e+04 + -2.189272353140778e+04 + -2.191917573000000e+04 + -2.194565191497987e+04 + -2.197215210512397e+04 + -2.199867631637375e+04 + -2.202522455969623e+04 + -2.205179684999998e+04 + -2.207839320363563e+04 + -2.210501363148952e+04 + -2.213165814783550e+04 + -2.215832677350680e+04 + -2.218501951999998e+04 + -2.221173639532642e+04 + -2.223847741910094e+04 + -2.226524260850044e+04 + -2.229203197306979e+04 + -2.231884553000000e+04 + -2.234568329890952e+04 + -2.237254528774074e+04 + -2.239943150806158e+04 + -2.242634198094407e+04 + -2.245327672000000e+04 + -2.248023573585785e+04 + -2.250721904853940e+04 + -2.253422667413636e+04 + -2.256125861967753e+04 + -2.258831489999998e+04 + -2.261539553480252e+04 + -2.264250054093014e+04 + -2.266962993052693e+04 + -2.269678371062339e+04 + -2.272396189999998e+04 + -2.275116452153000e+04 + -2.277839158301363e+04 + -2.280564309534358e+04 + -2.283291907950545e+04 + -2.286021955000000e+04 + -2.288754451771620e+04 + -2.291489399812011e+04 + -2.294226800704672e+04 + -2.296966655917324e+04 + -2.299708967000000e+04 + -2.302453735490683e+04 + -2.305200962641978e+04 + -2.307950649734620e+04 + -2.310702798236498e+04 + -2.313457409999998e+04 + -2.316214486856879e+04 + -2.318974029613145e+04 + -2.321736039447026e+04 + -2.324500518448656e+04 + -2.327267467999998e+04 + -2.330036889175501e+04 + -2.332808783850466e+04 + -2.335583153764027e+04 + -2.338360000121357e+04 + -2.341139324000000e+04 + -2.343921126672951e+04 + -2.346705410394532e+04 + -2.349492176951476e+04 + -2.352281427072812e+04 + -2.355073162000000e+04 + -2.357867383414989e+04 + -2.360664093193761e+04 + -2.363463292919098e+04 + -2.366264983684289e+04 + -2.369069166999998e+04 + -2.371875844507638e+04 + -2.374685017193912e+04 + -2.377496686591387e+04 + -2.380310855200050e+04 + -2.383127523999998e+04 + -2.385946693403709e+04 + -2.388768365690810e+04 + -2.391592542840760e+04 + -2.394419225713039e+04 + -2.397248416000000e+04 + -2.400080115691363e+04 + -2.402914325571854e+04 + -2.405751046800998e+04 + -2.408590281513924e+04 + -2.411432031000000e+04 + -2.414276296293371e+04 + -2.417123079796694e+04 + -2.419972383203242e+04 + -2.422824206706797e+04 + -2.425678551999998e+04 + -2.428535421485334e+04 + -2.431394816200073e+04 + -2.434256737430352e+04 + -2.437121187300517e+04 + -2.439988166999998e+04 + -2.442857677325164e+04 + -2.445729720103540e+04 + -2.448604297194451e+04 + -2.451481410082247e+04 + -2.454361060000000e+04 + -2.457243248182022e+04 + -2.460127976431730e+04 + -2.463015246522100e+04 + -2.465905059942621e+04 + -2.468797417999998e+04 + -2.471692321945616e+04 + -2.474589773222362e+04 + -2.477489773480251e+04 + -2.480392324581322e+04 + -2.483297427999998e+04 + -2.486205084917968e+04 + -2.489115296405446e+04 + -2.492028064024748e+04 + -2.494943390068509e+04 + -2.497861275999998e+04 + -2.500781722863925e+04 + -2.503704732334474e+04 + -2.506630305872901e+04 + -2.509558444448151e+04 + -2.512489150000000e+04 + -2.515422424757818e+04 + -2.518358269497819e+04 + -2.521296685328307e+04 + -2.524237674371633e+04 + -2.527181237999998e+04 + -2.530127377273551e+04 + -2.533076094162915e+04 + -2.536027390311031e+04 + -2.538981266594794e+04 + -2.541937724999998e+04 + -2.544896767801531e+04 + -2.547858395406606e+04 + -2.550822608802068e+04 + -2.553789410507042e+04 + -2.556758802000000e+04 + -2.559730784246711e+04 + -2.562705359189478e+04 + -2.565682528525429e+04 + -2.568662293247029e+04 + -2.571644655000000e+04 + -2.574629615713407e+04 + -2.577617176604459e+04 + -2.580607338891535e+04 + -2.583600104095400e+04 + -2.586595473999998e+04 + -2.589593450400127e+04 + -2.592594034522632e+04 + -2.595597227610649e+04 + -2.598603031176017e+04 + -2.601611446999998e+04 + -2.604622476879967e+04 + -2.607636122042706e+04 + -2.610652383730750e+04 + -2.613671263455693e+04 + -2.616692763000000e+04 + -2.619716884161440e+04 + -2.622743628160562e+04 + -2.625772996246335e+04 + -2.628804989951923e+04 + -2.631839611000000e+04 + -2.634876861165059e+04 + -2.637916741999917e+04 + -2.640959254907723e+04 + -2.644004401148305e+04 + -2.647052181999998e+04 + -2.650102598983489e+04 + -2.653155654491748e+04 + -2.656211350103963e+04 + -2.659269685960091e+04 + -2.662330663999998e+04 + -2.665394286925502e+04 + -2.668460555517195e+04 + -2.671529470758218e+04 + -2.674601034670874e+04 + -2.677675249000000e+04 + -2.680752115154265e+04 + -2.683831634157976e+04 + -2.686913807564717e+04 + -2.689998637786448e+04 + -2.693086126000000e+04 + -2.696176272916689e+04 + -2.699269080763921e+04 + -2.702364551500381e+04 + -2.705462686113689e+04 + -2.708563485999998e+04 + -2.711666952809900e+04 + -2.714773087942122e+04 + -2.717881892991400e+04 + -2.720993369901594e+04 + -2.724107519999998e+04 + -2.727224344400496e+04 + -2.730343845029767e+04 + -2.733466023562860e+04 + -2.736590881019403e+04 + -2.739718419000000e+04 + -2.742848639412166e+04 + -2.745981543741902e+04 + -2.749117133427855e+04 + -2.752255410001475e+04 + -2.755396375000000e+04 + -2.758540029934807e+04 + -2.761686376223195e+04 + -2.764835415476579e+04 + -2.767987149599797e+04 + -2.771141579999997e+04 + -2.774298707825893e+04 + -2.777458534547961e+04 + -2.780621061813955e+04 + -2.783786291408934e+04 + -2.786954224999998e+04 + -2.790124864062599e+04 + -2.793298209650868e+04 + -2.796474263269793e+04 + -2.799653027215609e+04 + -2.802834503000000e+04 + -2.806018691678819e+04 + -2.809205594694776e+04 + -2.812395213748840e+04 + -2.815587550739401e+04 + -2.818782607000000e+04 + -2.821980383662756e+04 + -2.825180882556652e+04 + -2.828384105454396e+04 + -2.831590053775406e+04 + -2.834798728999998e+04 + -2.838010132665882e+04 + -2.841224266303304e+04 + -2.844441131398464e+04 + -2.847660729395342e+04 + -2.850883061999998e+04 + -2.854108131015271e+04 + -2.857335937943549e+04 + -2.860566484233479e+04 + -2.863799771378746e+04 + -2.867035801000000e+04 + -2.870274574643904e+04 + -2.873516093256026e+04 + -2.876760358436649e+04 + -2.880007372909855e+04 + -2.883257138000000e+04 + -2.886509654365616e+04 + -2.889764923870952e+04 + -2.893022948372628e+04 + -2.896283729252707e+04 + -2.899547267999998e+04 + -2.902813566168059e+04 + -2.906082625190441e+04 + -2.909354446650575e+04 + -2.912629032389195e+04 + -2.915906383999998e+04 + -2.919186502919167e+04 + -2.922469390650106e+04 + -2.925755048733507e+04 + -2.929043478725063e+04 + -2.932334682000000e+04 + -2.935628660019129e+04 + -2.938925415043921e+04 + -2.942224948806936e+04 + -2.945527261979538e+04 + -2.948832356000000e+04 + -2.952140232838030e+04 + -2.955450894361826e+04 + -2.958764342089699e+04 + -2.962080577090842e+04 + -2.965399600999997e+04 + -2.968721415715336e+04 + -2.972046022629084e+04 + -2.975373423280592e+04 + -2.978703619592538e+04 + -2.982036612999997e+04 + -2.985372404652352e+04 + -2.988710995909819e+04 + -2.992052388535540e+04 + -2.995396584761164e+04 + -2.998743586000000e+04 + -3.002093393306212e+04 + -3.005446008516229e+04 + -3.008801433415196e+04 + -3.012159669410924e+04 + -3.015520717999998e+04 + -3.018884580728562e+04 + -3.022251259030155e+04 + -3.025620754490144e+04 + -3.028993068950620e+04 + -3.032368203999997e+04 + -3.035746161072607e+04 + -3.039126941699663e+04 + -3.042510547393189e+04 + -3.045896979612689e+04 + -3.049286239999998e+04 + -3.052678330335746e+04 + -3.056073252467363e+04 + -3.059471007984014e+04 + -3.062871598061521e+04 + -3.066275024000000e+04 + -3.069681287356475e+04 + -3.073090390293319e+04 + -3.076502334621974e+04 + -3.079917121404217e+04 + -3.083334751999997e+04 + -3.086755228023463e+04 + -3.090178551167921e+04 + -3.093604723225209e+04 + -3.097033746063074e+04 + -3.100465620999998e+04 + -3.103900349175437e+04 + -3.107337932446759e+04 + -3.110778372555016e+04 + -3.114221670819769e+04 + -3.117667828999997e+04 + -3.121116848957679e+04 + -3.124568731744623e+04 + -3.128023478769976e+04 + -3.131481092268889e+04 + -3.134941574000000e+04 + -3.138404925321087e+04 + -3.141871147408727e+04 + -3.145340241670125e+04 + -3.148812209933996e+04 + -3.152287053999997e+04 + -3.155764775577736e+04 + -3.159245376185456e+04 + -3.162728857279384e+04 + -3.166215220315186e+04 + -3.169704466999997e+04 + -3.173196599132658e+04 + -3.176691618227545e+04 + -3.180189525721778e+04 + -3.183690323068162e+04 + -3.187194012000000e+04 + -3.190700594323912e+04 + -3.194210071422084e+04 + -3.197722444868163e+04 + -3.201237716636405e+04 + -3.204755888000000e+04 + -3.208276960047787e+04 + -3.211800935023939e+04 + -3.215327814773632e+04 + -3.218857600158763e+04 + -3.222390292999997e+04 + -3.225925895535199e+04 + -3.229464408973093e+04 + -3.233005834462857e+04 + -3.236550173499735e+04 + -3.240097427999997e+04 + -3.243647600017841e+04 + -3.247200691143220e+04 + -3.250756702669824e+04 + -3.254315635678932e+04 + -3.257877492000000e+04 + -3.261442273792899e+04 + -3.265009982555360e+04 + -3.268580619650297e+04 + -3.272154186523808e+04 + -3.275730685000000e+04 + -3.279310116911321e+04 + -3.282892483177912e+04 + -3.286477785354587e+04 + -3.290066026202923e+04 + -3.293657206999998e+04 + -3.297251328378459e+04 + -3.300848392497173e+04 + -3.304448401407080e+04 + -3.308051356412061e+04 + -3.311657258999998e+04 + -3.315266110752813e+04 + -3.318877912993135e+04 + -3.322492667413289e+04 + -3.326110376297835e+04 + -3.329731041000000e+04 + -3.333354662539115e+04 + -3.336981243131684e+04 + -3.340610784669931e+04 + -3.344243288124504e+04 + -3.347878755000000e+04 + -3.351517187197135e+04 + -3.355158586607149e+04 + -3.358802954742114e+04 + -3.362450292603845e+04 + -3.366100601999997e+04 + -3.369753885113907e+04 + -3.373410143432544e+04 + -3.377069378324004e+04 + -3.380731591267414e+04 + -3.384396783999997e+04 + -3.388064958319757e+04 + -3.391736115628571e+04 + -3.395410257481684e+04 + -3.399087385786547e+04 + -3.402767502000000e+04 + -3.406450607498593e+04 + -3.410136704569621e+04 + -3.413825794940452e+04 + -3.417517879208190e+04 + -3.421212959000000e+04 + -3.424911036577375e+04 + -3.428612113814977e+04 + -3.432316192150666e+04 + -3.436023272579379e+04 + -3.439733356999998e+04 + -3.443446447621852e+04 + -3.447162545467578e+04 + -3.450881651888796e+04 + -3.454603769159935e+04 + -3.458328898999997e+04 + -3.462057042742187e+04 + -3.465788201822977e+04 + -3.469522377879945e+04 + -3.473259572783916e+04 + -3.476999788000000e+04 + -3.480743024920274e+04 + -3.484489285737491e+04 + -3.488238572274076e+04 + -3.491990885516477e+04 + -3.495746227000000e+04 + -3.499504598627476e+04 + -3.503266002163332e+04 + -3.507030439252002e+04 + -3.510797911399110e+04 + -3.514568419999997e+04 + -3.518341966538968e+04 + -3.522118553105182e+04 + -3.525898181605689e+04 + -3.529680853433678e+04 + -3.533466569999997e+04 + -3.537255332834070e+04 + -3.541047143775194e+04 + -3.544842004484196e+04 + -3.548639916273562e+04 + -3.552440881000000e+04 + -3.556244900769237e+04 + -3.560051977192599e+04 + -3.563862111528000e+04 + -3.567675304773126e+04 + -3.571491559000000e+04 + -3.575310876677514e+04 + -3.579133259025522e+04 + -3.582958707123636e+04 + -3.586787222387752e+04 + -3.590618806999997e+04 + -3.594453463328037e+04 + -3.598291192576281e+04 + -3.602131995866808e+04 + -3.605975874687618e+04 + -3.609822830999997e+04 + -3.613672866872384e+04 + -3.617525983663845e+04 + -3.621382182906358e+04 + -3.625241466608051e+04 + -3.629103836000000e+04 + -3.632969292147818e+04 + -3.636837837559645e+04 + -3.640709474306420e+04 + -3.644584203277781e+04 + -3.648462026000000e+04 + -3.652342944409004e+04 + -3.656226960170399e+04 + -3.660114075023598e+04 + -3.664004290875195e+04 + -3.667897608999997e+04 + -3.671794030512006e+04 + -3.675693557547587e+04 + -3.679596192058374e+04 + -3.683501935339895e+04 + -3.687410788999997e+04 + -3.691322754870546e+04 + -3.695237834723054e+04 + -3.699156030212950e+04 + -3.703077342834224e+04 + -3.707001774000000e+04 + -3.710929325203417e+04 + -3.714859998429457e+04 + -3.718793795692067e+04 + -3.722730718808447e+04 + -3.726670768999997e+04 + -3.730613947399677e+04 + -3.734560256273038e+04 + -3.738509697431393e+04 + -3.742462271654350e+04 + -3.746417980999997e+04 + -3.750376828013417e+04 + -3.754338813660564e+04 + -3.758303939267375e+04 + -3.762272207245150e+04 + -3.766243618999997e+04 + -3.770218175518502e+04 + -3.774195878945678e+04 + -3.778176731257176e+04 + -3.782160733741773e+04 + -3.786147888000000e+04 + -3.790138195864309e+04 + -3.794131659132003e+04 + -3.798128279431332e+04 + -3.802128058166662e+04 + -3.806130996999997e+04 + -3.810137097737330e+04 + -3.814146362046005e+04 + -3.818158791697343e+04 + -3.822174388615621e+04 + -3.826193153999997e+04 + -3.830215088938532e+04 + -3.834240195971708e+04 + -3.838268477130731e+04 + -3.842299933188884e+04 + -3.846334565999997e+04 + -3.850372377842891e+04 + -3.854413369633216e+04 + -3.858457542816681e+04 + -3.862504900090371e+04 + -3.866555443000000e+04 + -3.870609172486966e+04 + -3.874666090332934e+04 + -3.878726198409722e+04 + -3.882789498399841e+04 + -3.886855991999997e+04 + -3.890925680946781e+04 + -3.894998567054054e+04 + -3.899074651957592e+04 + -3.903153937021765e+04 + -3.907236423999997e+04 + -3.911322114736439e+04 + -3.915411010384250e+04 + -3.919503112747372e+04 + -3.923598424716172e+04 + -3.927696946999997e+04 + -3.931798680038228e+04 + -3.935903628964794e+04 + -3.940011793259103e+04 + -3.944123164809230e+04 + -3.948237770000000e+04 + -3.952355620675491e+04 + -3.956476584662273e+04 + -3.960600801403134e+04 + -3.964728735507886e+04 + -3.968859104999997e+04 + 1.819886919039960e+01 + 1.816405130921966e+01 + 1.812918280091599e+01 + 1.809426424653407e+01 + 1.805929622711938e+01 + 1.802427932371738e+01 + 1.798921411737356e+01 + 1.795410118913339e+01 + 1.791894112004234e+01 + 1.788373449114589e+01 + 1.784848188348950e+01 + 1.781318387811866e+01 + 1.777784105607885e+01 + 1.774245399841551e+01 + 1.770702328617416e+01 + 1.767154950040025e+01 + 1.763603322213927e+01 + 1.760047503243666e+01 + 1.756487551233793e+01 + 1.752923524288855e+01 + 1.749355480513398e+01 + 1.745783478011969e+01 + 1.742207574889119e+01 + 1.738627829249391e+01 + 1.735044299197335e+01 + 1.731457042837498e+01 + 1.727866118274428e+01 + 1.724271583612671e+01 + 1.720673496956776e+01 + 1.717071916411290e+01 + 1.713466900080759e+01 + 1.709858506069732e+01 + 1.706246792482757e+01 + 1.702631817424381e+01 + 1.699013638999150e+01 + 1.695392315311613e+01 + 1.691767904466317e+01 + 1.688140464567809e+01 + 1.684510053720637e+01 + 1.680876730029348e+01 + 1.677240551598491e+01 + 1.673601576532611e+01 + 1.669959862936258e+01 + 1.666315468913977e+01 + 1.662668452570318e+01 + 1.659018872009826e+01 + 1.655366785337049e+01 + 1.651712250656536e+01 + 1.648055326072833e+01 + 1.644396069690488e+01 + 1.640734539614049e+01 + 1.637070793948063e+01 + 1.633404890797076e+01 + 1.629736888265638e+01 + 1.626066844458294e+01 + 1.622394817479593e+01 + 1.618720865434082e+01 + 1.615045046426309e+01 + 1.611367418560821e+01 + 1.607688039942165e+01 + 1.604006968674890e+01 + 1.600324262863542e+01 + 1.596639980612668e+01 + 1.592954180026817e+01 + 1.589266919210536e+01 + 1.585578256268371e+01 + 1.581888249304872e+01 + 1.578196956424585e+01 + 1.574504435732057e+01 + 1.570810745331836e+01 + 1.567115943328470e+01 + 1.563420087826506e+01 + 1.559723236930490e+01 + 1.556025448744972e+01 + 1.552326781374499e+01 + 1.548627292923617e+01 + 1.544927041496875e+01 + 1.541226085198818e+01 + 1.537524482133997e+01 + 1.533822290406956e+01 + 1.530119568122245e+01 + 1.526416373384410e+01 + 1.522712764298000e+01 + 1.519008798967561e+01 + 1.515304535497640e+01 + 1.511600031992786e+01 + 1.507895346557546e+01 + 1.504190537296467e+01 + 1.500485662314096e+01 + 1.496780779714983e+01 + 1.493075947603672e+01 + 1.489371224084713e+01 + 1.485666667262652e+01 + 1.481962335242037e+01 + 1.478258286127416e+01 + 1.474554578023335e+01 + 1.470851269034343e+01 + 1.467148417264987e+01 + 1.463446080819813e+01 + 1.459744317803371e+01 + 1.456043186320207e+01 + 1.452342744474869e+01 + 1.448643050371903e+01 + 1.444944162115858e+01 + 1.441246137811282e+01 + 1.437549035562720e+01 + 1.433852913474722e+01 + 1.430157829651834e+01 + 1.426463842198604e+01 + 1.422771009219579e+01 + 1.419079388819307e+01 + 1.415389039102334e+01 + 1.411700018173210e+01 + 1.408012384136481e+01 + 1.404326195096694e+01 + 1.400641509158398e+01 + 1.396958384426139e+01 + 1.393276879004464e+01 + 1.389597050997923e+01 + 1.385918958511061e+01 + 1.382242659648426e+01 + 1.378568212514566e+01 + 1.374895675214029e+01 + 1.371225105851361e+01 + 1.367556562531110e+01 + 1.363890103357824e+01 + 1.360225786436050e+01 + 1.356563669870335e+01 + 1.352903811765228e+01 + 1.349246270225274e+01 + 1.345591103355023e+01 + 1.341938369259021e+01 + 1.338288126041816e+01 + 1.334640431807955e+01 + 1.330995344661985e+01 + 1.327352922708455e+01 + 1.323713224051912e+01 + 1.320076306796902e+01 + 1.316442229047974e+01 + 1.312811048909675e+01 + 1.309182824486553e+01 + 1.305557613883154e+01 + 1.301935475204027e+01 + 1.298316466553718e+01 + 1.294700646036776e+01 + 1.291088071757748e+01 + 1.287478801821181e+01 + 1.283872894331622e+01 + 1.280270407393619e+01 + 1.276671399111720e+01 + 1.273075927590472e+01 + 1.269484050934423e+01 + 1.265895827248119e+01 + 1.262311314636109e+01 + 1.258730571202939e+01 + 1.255153655053158e+01 + 1.251580624291313e+01 + 1.248011537021950e+01 + 1.244446451349619e+01 + 1.240885425378865e+01 + 1.237328517214237e+01 + 1.233775784960283e+01 + 1.230227286721548e+01 + 1.226683080602582e+01 + 1.223143224707931e+01 + 1.219607777142142e+01 + 1.216076796009764e+01 + 1.212550339415344e+01 + 1.209028465463429e+01 + 1.205511232258567e+01 + 1.201998697905305e+01 + 1.198490920508190e+01 + 1.194987958171770e+01 + 1.191489869000593e+01 + 1.187996711099206e+01 + 1.184508542572157e+01 + 1.181025421523992e+01 + 1.177547406059259e+01 + 1.174074554282507e+01 + 1.170606924298281e+01 + 1.167144574211130e+01 + 1.163687562125602e+01 + 1.160235946146243e+01 + 1.156789784377601e+01 + 1.153349134924223e+01 + 1.149914055890657e+01 + 1.146484605381452e+01 + 1.143060841501152e+01 + 1.139642822354308e+01 + 1.136230606045464e+01 + 1.132824250679170e+01 + 1.129423814359974e+01 + 1.126029355192421e+01 + 1.122640931281059e+01 + 1.119258600730437e+01 + 1.115882421645102e+01 + 1.112512452129600e+01 + 1.109148750288479e+01 + 1.105791374226288e+01 + 1.102440382047573e+01 + 1.099095831856882e+01 + 1.095757781758762e+01 + 1.092426289857761e+01 + 1.089101414258426e+01 + 1.085783213065305e+01 + 1.082471744382945e+01 + 1.079167066315892e+01 + 1.075869236968697e+01 + 1.072578314445905e+01 + 1.069294356852064e+01 + 1.066017422291721e+01 + 1.062747568869424e+01 + 1.059484854689720e+01 + 1.056229337857157e+01 + 1.052981076476282e+01 + 1.049740128651643e+01 + 1.046506552487787e+01 + 1.043280406089261e+01 + 1.040061747560614e+01 + 1.036850635006392e+01 + 1.033647126531143e+01 + 1.030451280239414e+01 + 1.027263154235754e+01 + 1.024082806624708e+01 + 1.020910295510825e+01 + 1.017745678998653e+01 + 1.014589015192738e+01 + 1.011440362197629e+01 + 1.008299778117872e+01 + 1.005167321058014e+01 + 1.002043049122605e+01 + 9.989270204161905e+00 + 9.958192930433185e+00 + 9.927199251085362e+00 + 9.896289747163912e+00 + 9.865464999714312e+00 + 9.834725589782032e+00 + 9.804072098412551e+00 + 9.773505106651340e+00 + 9.743025195543877e+00 + 9.712632946135633e+00 + 9.682328939472086e+00 + 9.652113756598707e+00 + 9.621987978560973e+00 + 9.591952186404358e+00 + 9.562006961174337e+00 + 9.532152883916382e+00 + 9.502390535675973e+00 + 9.472720497498580e+00 + 9.443143350429677e+00 + 9.413659675514742e+00 + 9.384270053799249e+00 + 9.354975066328668e+00 + 9.325775294148480e+00 + 9.296671318304156e+00 + 9.267663719841170e+00 + 9.238753079804999e+00 + 9.209939979241115e+00 + 9.181224999194994e+00 + 9.152608720712111e+00 + 9.124091724837941e+00 + 9.095674592617954e+00 + 9.067357905097630e+00 + 9.039142243322441e+00 + 9.011028188337864e+00 + 8.983016292445305e+00 + 8.955106689123129e+00 + 8.927299195109086e+00 + 8.899593619314148e+00 + 8.871989770649284e+00 + 8.844487458025464e+00 + 8.817086490353660e+00 + 8.789786676544839e+00 + 8.762587825509975e+00 + 8.735489746160036e+00 + 8.708492247405990e+00 + 8.681595138158814e+00 + 8.654798227329470e+00 + 8.628101323828931e+00 + 8.601504236568170e+00 + 8.575006774458155e+00 + 8.548608746409855e+00 + 8.522309961334242e+00 + 8.496110228142285e+00 + 8.470009355744955e+00 + 8.444007153053219e+00 + 8.418103428978053e+00 + 8.392297992430423e+00 + 8.366590652321301e+00 + 8.340981217561657e+00 + 8.315469497062459e+00 + 8.290055299734679e+00 + 8.264738434489287e+00 + 8.239518710237252e+00 + 8.214395935889545e+00 + 8.189369920357137e+00 + 8.164440472550996e+00 + 8.139607401382095e+00 + 8.114870515761401e+00 + 8.090229624599887e+00 + 8.065684536808520e+00 + 8.041235061298275e+00 + 8.016881006980118e+00 + 7.992622182765018e+00 + 7.968458397563950e+00 + 7.944389460287880e+00 + 7.920415179847782e+00 + 7.896535365154623e+00 + 7.872749825119373e+00 + 7.849058368653004e+00 + 7.825460804666484e+00 + 7.801956942070785e+00 + 7.778546589776878e+00 + 7.755229556695729e+00 + 7.732005651738313e+00 + 7.708874683815599e+00 + 7.685836461838553e+00 + 7.662890794718152e+00 + 7.640037491365358e+00 + 7.617276360691148e+00 + 7.594607211606489e+00 + 7.572029853022352e+00 + 7.549544093849708e+00 + 7.527149742999527e+00 + 7.504846609382777e+00 + 7.482634501910429e+00 + 7.460513229493455e+00 + 7.438482601042820e+00 + 7.416542425469502e+00 + 7.394692511684465e+00 + 7.372932668598682e+00 + 7.351262705123121e+00 + 7.329682430168755e+00 + 7.308191652646553e+00 + 7.286790181467484e+00 + 7.265477825542517e+00 + 7.244254393782626e+00 + 7.223119695098779e+00 + 7.202073538401945e+00 + 7.181115732603097e+00 + 7.160246086613204e+00 + 7.139464409343233e+00 + 7.118770509704159e+00 + 7.098164196606950e+00 + 7.077645278962576e+00 + 7.057213565682006e+00 + 7.036868865676214e+00 + 7.016610987856166e+00 + 6.996439741132834e+00 + 6.976354934417186e+00 + 6.956356376620197e+00 + 6.936443876652832e+00 + 6.916617243426063e+00 + 6.896876285850862e+00 + 6.877220812838198e+00 + 6.857650633299039e+00 + 6.838165556144358e+00 + 6.818765390285123e+00 + 6.799449944632306e+00 + 6.780219028096876e+00 + 6.761072449589803e+00 + 6.742010018022058e+00 + 6.723031542304612e+00 + 6.704136831348434e+00 + 6.685325694064492e+00 + 6.666597939363759e+00 + 6.647953376157203e+00 + 6.629391813355797e+00 + 6.610913059870509e+00 + 6.592516924612310e+00 + 6.574203216492170e+00 + 6.555971744421058e+00 + 6.537822317309946e+00 + 6.519754744069803e+00 + 6.501768833611600e+00 + 6.483864394846306e+00 + 6.466041236684891e+00 + 6.448299168038327e+00 + 6.430637997817582e+00 + 6.413057534933630e+00 + 6.395557588297437e+00 + 6.378137966819973e+00 + 6.360798479412210e+00 + 6.343538934985117e+00 + 6.326359142449666e+00 + 6.309258910716826e+00 + 6.292238048697567e+00 + 6.275296365302859e+00 + 6.258433669443673e+00 + 6.241649770030978e+00 + 6.224944475975745e+00 + 6.208317596188945e+00 + 6.191768939581546e+00 + 6.175298315064517e+00 + 6.158905531548833e+00 + 6.142590397945462e+00 + 6.126352723165372e+00 + 6.110192316119535e+00 + 6.094108985718921e+00 + 6.078102540874499e+00 + 6.062172790497243e+00 + 6.046319543498117e+00 + 6.030542608788095e+00 + 6.014841795278149e+00 + 5.999216911879244e+00 + 5.983667767502356e+00 + 5.968194171058448e+00 + 5.952795931458497e+00 + 5.937472857613470e+00 + 5.922224758434336e+00 + 5.907051442832068e+00 + 5.891952719717633e+00 + 5.876928398002004e+00 + 5.861978286596150e+00 + 5.847102194411042e+00 + 5.832299930357648e+00 + 5.817571303346941e+00 + 5.802916122289888e+00 + 5.788334196097460e+00 + 5.773825333680629e+00 + 5.759389343950365e+00 + 5.745026035817635e+00 + 5.730735218193414e+00 + 5.716516699988668e+00 + 5.702370290114368e+00 + 5.688295797481486e+00 + 5.674293031000990e+00 + 5.660361799583851e+00 + 5.646501912141039e+00 + 5.632713177583526e+00 + 5.618995404822279e+00 + 5.605348402768269e+00 + 5.591771980332467e+00 + 5.578265946425844e+00 + 5.564830109959368e+00 + 5.551464279844009e+00 + 5.538168264990741e+00 + 5.524941874310529e+00 + 5.511784916714348e+00 + 5.498697201113163e+00 + 5.485678536417949e+00 + 5.472728731539672e+00 + 5.459847595389306e+00 + 5.447031950668924e+00 + 5.434258417587328e+00 + 5.421509378931341e+00 + 5.408790202589383e+00 + 5.396101725502927e+00 + 5.383442552149385e+00 + 5.370812984345576e+00 + 5.358213038980171e+00 + 5.345642546622406e+00 + 5.333101463094469e+00 + 5.320589727671893e+00 + 5.308107264570896e+00 + 5.295654006874094e+00 + 5.283229887334054e+00 + 5.270834837715376e+00 + 5.258468789944041e+00 + 5.246131676000384e+00 + 5.233823428164784e+00 + 5.221543979540164e+00 + 5.209293263422158e+00 + 5.197071212820756e+00 + 5.184877759775030e+00 + 5.172712837552656e+00 + 5.160576380473933e+00 + 5.148468322098453e+00 + 5.136388595958436e+00 + 5.124337135726472e+00 + 5.112313875048248e+00 + 5.100318748362747e+00 + 5.088351690727316e+00 + 5.076412636057685e+00 + 5.064501518839616e+00 + 5.052618274474592e+00 + 5.040762837425790e+00 + 5.028935142799786e+00 + 5.017135126690616e+00 + 5.005362723409764e+00 + 4.993617868159293e+00 + 4.981900498164465e+00 + 4.970210548621045e+00 + 4.958547954855285e+00 + 4.946912653797631e+00 + 4.935304581602165e+00 + 4.923723674430540e+00 + 4.912169869219803e+00 + 4.900643102722201e+00 + 4.889143311682699e+00 + 4.877670433140317e+00 + 4.866224404407401e+00 + 4.854805162838305e+00 + 4.843412645620055e+00 + 4.832046790470948e+00 + 4.820707535433044e+00 + 4.809394818449872e+00 + 4.798108577205163e+00 + 4.786848749644187e+00 + 4.775615274885523e+00 + 4.764408091170769e+00 + 4.753227136475494e+00 + 4.742072350593207e+00 + 4.730943672320340e+00 + 4.719841039761676e+00 + 4.708764392793848e+00 + 4.697713670947889e+00 + 4.686688813308958e+00 + 4.675689760077749e+00 + 4.664716450776882e+00 + 4.653768824355144e+00 + 4.642846821849820e+00 + 4.631950383864691e+00 + 4.621079449911801e+00 + 4.610233960271208e+00 + 4.599413855814052e+00 + 4.588619077726644e+00 + 4.577849566594633e+00 + 4.567105263324451e+00 + 4.556386109499332e+00 + 4.545692045920192e+00 + 4.535023013830533e+00 + 4.524378955558838e+00 + 4.513759812087895e+00 + 4.503165524982657e+00 + 4.492596037765430e+00 + 4.482051291649467e+00 + 4.471531227881555e+00 + 4.461035790191556e+00 + 4.450564920902143e+00 + 4.440118561957067e+00 + 4.429696656605228e+00 + 4.419299148008472e+00 + 4.408925979093191e+00 + 4.398577092681390e+00 + 4.388252432408600e+00 + 4.377951941955149e+00 + 4.367675563647617e+00 + 4.357423241911437e+00 + 4.347194921954801e+00 + 4.336990545878941e+00 + 4.326810057907454e+00 + 4.316653403787664e+00 + 4.306520526395858e+00 + 4.296411370095723e+00 + 4.286325880845187e+00 + 4.276264002847886e+00 + 4.266225680614548e+00 + 4.256210859410026e+00 + 4.246219484255977e+00 + 4.236251500111860e+00 + 4.226306852214246e+00 + 4.216385487075638e+00 + 4.206487350454588e+00 + 4.196612386873075e+00 + 4.186760542346591e+00 + 4.176931763412669e+00 + 4.167125996517556e+00 + 4.157343187931778e+00 + 4.147583283659849e+00 + 4.137846229598449e+00 + 4.128131973158434e+00 + 4.118440461620333e+00 + 4.108771640965568e+00 + 4.099125458381889e+00 + 4.089501861348100e+00 + 4.079900796648449e+00 + 4.070322211621146e+00 + 4.060766054068578e+00 + 4.051232271987449e+00 + 4.041720812903752e+00 + 4.032231624389123e+00 + 4.022764654839604e+00 + 4.013319852102482e+00 + 4.003897163974346e+00 + 3.994496539375360e+00 + 3.985117927005318e+00 + 3.975761275162846e+00 + 3.966426531955261e+00 + 3.957113646635731e+00 + 3.947822569087183e+00 + 3.938553247905866e+00 + 3.929305631685672e+00 + 3.920079669524097e+00 + 3.910875311427285e+00 + 3.901692507178291e+00 + 3.892531206204140e+00 + 3.883391358396833e+00 + 3.874272913657344e+00 + 3.865175821793835e+00 + 3.856100032879644e+00 + 3.847045497225832e+00 + 3.838012165320722e+00 + 3.828999987558789e+00 + 3.820008914413926e+00 + 3.811038896595444e+00 + 3.802089885254187e+00 + 3.793161831237620e+00 + 3.784254684890362e+00 + 3.775368398252875e+00 + 3.766502923100440e+00 + 3.757658209579520e+00 + 3.748834209295840e+00 + 3.740030874433959e+00 + 3.731248156753105e+00 + 3.722486008117639e+00 + 3.713744380387517e+00 + 3.705023225280349e+00 + 3.696322495070079e+00 + 3.687642142318910e+00 + 3.678982119432947e+00 + 3.670342378801804e+00 + 3.661722872867589e+00 + 3.653123554235752e+00 + 3.644544376023881e+00 + 3.635985291634798e+00 + 3.627446254192749e+00 + 3.618927216218316e+00 + 3.610428130335504e+00 + 3.601948951051534e+00 + 3.593489632231203e+00 + 3.585050126838500e+00 + 3.576630388153148e+00 + 3.568230370457688e+00 + 3.559850028500160e+00 + 3.551489315016799e+00 + 3.543148183918080e+00 + 3.534826590807976e+00 + 3.526524489935861e+00 + 3.518241835011243e+00 + 3.509978579991082e+00 + 3.501734680809670e+00 + 3.493510092697708e+00 + 3.485304769188242e+00 + 3.477118665289576e+00 + 3.468951736765370e+00 + 3.460803939385451e+00 + 3.452675227557384e+00 + 3.444565556219660e+00 + 3.436474882078371e+00 + 3.428403160505571e+00 + 3.420350346590145e+00 + 3.412316396304707e+00 + 3.404301265792477e+00 + 3.396304911176187e+00 + 3.388327288491305e+00 + 3.380368354160161e+00 + 3.372428064623584e+00 + 3.364506375833607e+00 + 3.356603244211646e+00 + 3.348718626640740e+00 + 3.340852480236101e+00 + 3.333004761693424e+00 + 3.325175427644580e+00 + 3.317364435527293e+00 + 3.309571742493616e+00 + 3.301797305408617e+00 + 3.294041081414062e+00 + 3.286303028194412e+00 + 3.278583103726798e+00 + 3.270881265443555e+00 + 3.263197471011956e+00 + 3.255531678390248e+00 + 3.247883845209004e+00 + 3.240253929642225e+00 + 3.232641890347850e+00 + 3.225047684908113e+00 + 3.217471271425511e+00 + 3.209912608923088e+00 + 3.202371655667555e+00 + 3.194848370181972e+00 + 3.187342711658790e+00 + 3.179854638881827e+00 + 3.172384110380890e+00 + 3.164931084707051e+00 + 3.157495521384956e+00 + 3.150077380058776e+00 + 3.142676619877814e+00 + 3.135293200073600e+00 + 3.127927079946843e+00 + 3.120578218905895e+00 + 3.113246577218404e+00 + 3.105932114937815e+00 + 3.098634790846833e+00 + 3.091354565058342e+00 + 3.084091398271724e+00 + 3.076845250485478e+00 + 3.069616081755534e+00 + 3.062403852244847e+00 + 3.055208522175611e+00 + 3.048030052549899e+00 + 3.040868404439169e+00 + 3.033723537524932e+00 + 3.026595412527113e+00 + 3.019483991059011e+00 + 3.012389234177664e+00 + 3.005311102428361e+00 + 2.998249556405217e+00 + 2.991204558170310e+00 + 2.984176069441633e+00 + 2.977164051262350e+00 + 2.970168464749357e+00 + 2.963189271462946e+00 + 2.956226433370632e+00 + 2.949279912203107e+00 + 2.942349669912605e+00 + 2.935435668735732e+00 + 2.928537870478263e+00 + 2.921656236941307e+00 + 2.914790730227295e+00 + 2.907941313056430e+00 + 2.901107948114753e+00 + 2.894290597705669e+00 + 2.887489224186201e+00 + 2.880703790004707e+00 + 2.873934257778331e+00 + 2.867180590721052e+00 + 2.860442752157053e+00 + 2.853720705060128e+00 + 2.847014412004529e+00 + 2.840323835735203e+00 + 2.833648939802342e+00 + 2.826989688360529e+00 + 2.820346045051714e+00 + 2.813717971838249e+00 + 2.807105432809830e+00 + 2.800508392739872e+00 + 2.793926814285955e+00 + 2.787360661385112e+00 + 2.780809898806732e+00 + 2.774274490318202e+00 + 2.767754399418414e+00 + 2.761249589980850e+00 + 2.754760027276096e+00 + 2.748285675987487e+00 + 2.741826499952791e+00 + 2.735382462817888e+00 + 2.728953529693038e+00 + 2.722539666591989e+00 + 2.716140837199994e+00 + 2.709757005928732e+00 + 2.703388138477217e+00 + 2.697034199719776e+00 + 2.690695154612969e+00 + 2.684370968516638e+00 + 2.678061606722120e+00 + 2.671767034388748e+00 + 2.665487216663756e+00 + 2.659222119420964e+00 + 2.652971708466898e+00 + 2.646735949170681e+00 + 2.640514807328682e+00 + 2.634308248813400e+00 + 2.628116239351709e+00 + 2.621938745026414e+00 + 2.615775731978081e+00 + 2.609627166192403e+00 + 2.603493014125890e+00 + 2.597373242133807e+00 + 2.591267815972944e+00 + 2.585176702619338e+00 + 2.579099869106871e+00 + 2.573037281022092e+00 + 2.566988905295047e+00 + 2.560954709471109e+00 + 2.554934660124372e+00 + 2.548928723658388e+00 + 2.542936866838776e+00 + 2.536959057454498e+00 + 2.530995262744329e+00 + 2.525045449466102e+00 + 2.519109584724339e+00 + 2.513187636248425e+00 + 2.507279571988880e+00 + 2.501385358975255e+00 + 2.495504964649074e+00 + 2.489638357011514e+00 + 2.483785503877440e+00 + 2.477946372993181e+00 + 2.472120932153270e+00 + 2.466309149445497e+00 + 2.460510993028623e+00 + 2.454726431004756e+00 + 2.448955431279121e+00 + 2.443197962222979e+00 + 2.437453992749473e+00 + 2.431723490873920e+00 + 2.426006424702946e+00 + 2.420302763016215e+00 + 2.414612474849150e+00 + 2.408935529060812e+00 + 2.403271894158963e+00 + 2.397621539164549e+00 + 2.391984432987164e+00 + 2.386360544006294e+00 + 2.380749841635097e+00 + 2.375152295549876e+00 + 2.369567874852914e+00 + 2.363996548806182e+00 + 2.358438286639424e+00 + 2.352893057314327e+00 + 2.347360830870365e+00 + 2.341841577523073e+00 + 2.336335266205249e+00 + 2.330841866367291e+00 + 2.325361348038680e+00 + 2.319893681430557e+00 + 2.314438836350213e+00 + 2.308996782525400e+00 + 2.303567490482044e+00 + 2.298150930506590e+00 + 2.292747072570897e+00 + 2.287355886755380e+00 + 2.281977343203661e+00 + 2.276611412298537e+00 + 2.271258065216148e+00 + 2.265917272602903e+00 + 2.260589004572198e+00 + 2.255273232290290e+00 + 2.249969926489583e+00 + 2.244679057245269e+00 + 2.239400595715412e+00 + 2.234134513404378e+00 + 2.228880781619142e+00 + 2.223639370658599e+00 + 2.218410251587366e+00 + 2.213193396784469e+00 + 2.207988776668006e+00 + 2.202796362022329e+00 + 2.197616125439223e+00 + 2.192448038696543e+00 + 2.187292072961057e+00 + 2.182148199232985e+00 + 2.177016389630072e+00 + 2.171896616273634e+00 + 2.166788850281173e+00 + 2.161693063906081e+00 + 2.156609229682959e+00 + 2.151537319229026e+00 + 2.146477304323076e+00 + 2.141429157241363e+00 + 2.136392850965788e+00 + 2.131368357474257e+00 + 2.126355648357996e+00 + 2.121354696293322e+00 + 2.116365474412895e+00 + 2.111387955644702e+00 + 2.106422111701483e+00 + 2.101467915161587e+00 + 2.096525339256323e+00 + 2.091594356269759e+00 + 2.086674939419586e+00 + 2.081767062556874e+00 + 2.076870697792070e+00 + 2.071985817585338e+00 + 2.067112395385072e+00 + 2.062250405333861e+00 + 2.057399820685575e+00 + 2.052560613690088e+00 + 2.047732757766644e+00 + 2.042916226752948e+00 + 2.038110994473384e+00 + 2.033317034456518e+00 + 2.028534320143143e+00 + 2.023762825026766e+00 + 2.019002522786289e+00 + 2.014253387371010e+00 + 2.009515392980107e+00 + 2.004788513366984e+00 + 2.000072722331970e+00 + 1.995367994084281e+00 + 1.990674302528751e+00 + 1.985991621875054e+00 + 1.981319927135009e+00 + 1.976659192144684e+00 + 1.972009390577169e+00 + 1.967370497271926e+00 + 1.962742487042732e+00 + 1.958125334423246e+00 + 1.953519013586483e+00 + 1.948923499307053e+00 + 1.944338766657414e+00 + 1.939764790288013e+00 + 1.935201544608222e+00 + 1.930649004305472e+00 + 1.926107145248571e+00 + 1.921575942429786e+00 + 1.917055370205972e+00 + 1.912545403920843e+00 + 1.908046018818900e+00 + 1.903557190023903e+00 + 1.899078893306255e+00 + 1.894611103753838e+00 + 1.890153795996532e+00 + 1.885706946418488e+00 + 1.881270530851602e+00 + 1.876844524133795e+00 + 1.872428902057808e+00 + 1.868023640555410e+00 + 1.863628715350075e+00 + 1.859244102235784e+00 + 1.854869776995271e+00 + 1.850505715382929e+00 + 1.846151893336109e+00 + 1.841808287113597e+00 + 1.837474873250078e+00 + 1.833151627500553e+00 + 1.828838525627810e+00 + 1.824535544058799e+00 + 1.820242659570220e+00 + 1.815959848693184e+00 + 1.811687087278560e+00 + 1.807424351697017e+00 + 1.803171618832717e+00 + 1.798928865872756e+00 + 1.794696068965575e+00 + 1.790473204047284e+00 + 1.786260248130819e+00 + 1.782057178544553e+00 + 1.777863972462110e+00 + 1.773680606381003e+00 + 1.769507057021776e+00 + 1.765343301422396e+00 + 1.761189316819223e+00 + 1.757045080431908e+00 + 1.752910569457795e+00 + 1.748785761158555e+00 + 1.744670632638435e+00 + 1.740565161064786e+00 + 1.736469324420783e+00 + 1.732383100261040e+00 + 1.728306465595185e+00 + 1.724239397824929e+00 + 1.720181874717039e+00 + 1.716133874165168e+00 + 1.712095373335323e+00 + 1.708066350172305e+00 + 1.704046783495335e+00 + 1.700036650108550e+00 + 1.696035927584721e+00 + 1.692044595228792e+00 + 1.688062630443701e+00 + 1.684090010658294e+00 + 1.680126714513943e+00 + 1.676172720475745e+00 + 1.672228006742128e+00 + 1.668292551293985e+00 + 1.664366332353613e+00 + 1.660449328407859e+00 + 1.656541518177628e+00 + 1.652642880144113e+00 + 1.648753392536404e+00 + 1.644873033459887e+00 + 1.641001782122826e+00 + 1.637139617814172e+00 + 1.633286518510287e+00 + 1.629442462986198e+00 + 1.625607430332010e+00 + 1.621781398816242e+00 + 1.617964347766720e+00 + 1.614156256896447e+00 + 1.610357104501046e+00 + 1.606566869528149e+00 + 1.602785531410984e+00 + 1.599013068753329e+00 + 1.595249460964023e+00 + 1.591494688099418e+00 + 1.587748729316499e+00 + 1.584011563661260e+00 + 1.580283170366302e+00 + 1.576563528988570e+00 + 1.572852618951484e+00 + 1.569150419635379e+00 + 1.565456911345500e+00 + 1.561772073862832e+00 + 1.558095886092752e+00 + 1.554428327545355e+00 + 1.550769378486889e+00 + 1.547119019605569e+00 + 1.543477229774924e+00 + 1.539843988467248e+00 + 1.536219276909035e+00 + 1.532603074921508e+00 + 1.528995362108325e+00 + 1.525396118748853e+00 + 1.521805324624735e+00 + 1.518222960029189e+00 + 1.514649006474386e+00 + 1.511083443428444e+00 + 1.507526250258141e+00 + 1.503977408416516e+00 + 1.500436898413773e+00 + 1.496904700394939e+00 + 1.493380795245864e+00 + 1.489865163193006e+00 + 1.486357784560608e+00 + 1.482858641108495e+00 + 1.479367713055452e+00 + 1.475884980161266e+00 + 1.472410424507970e+00 + 1.468944026943859e+00 + 1.465485767394345e+00 + 1.462035627142284e+00 + 1.458593587684006e+00 + 1.455159630175685e+00 + 1.451733734815932e+00 + 1.448315882934573e+00 + 1.444906056735074e+00 + 1.441504236540235e+00 + 1.438110403156542e+00 + 1.434724538391589e+00 + 1.431346623689521e+00 + 1.427976640530080e+00 + 1.424614570482120e+00 + 1.421260394583763e+00 + 1.417914094214416e+00 + 1.414575651360096e+00 + 1.411245047472556e+00 + 1.407922264020345e+00 + 1.404607282835409e+00 + 1.401300085863839e+00 + 1.398000654669480e+00 + 1.394708970355802e+00 + 1.391425015768138e+00 + 1.388148773421020e+00 + 1.384880223867722e+00 + 1.381619349200565e+00 + 1.378366132070768e+00 + 1.375120554406545e+00 + 1.371882598139520e+00 + 1.368652245409923e+00 + 1.365429478679455e+00 + 1.362214280063096e+00 + 1.359006631544336e+00 + 1.355806515436745e+00 + 1.352613914349998e+00 + 1.349428810943258e+00 + 1.346251187518860e+00 + 1.343081026346772e+00 + 1.339918309785753e+00 + 1.336763020411036e+00 + 1.333615141050152e+00 + 1.330474654669161e+00 + 1.327341544046787e+00 + 1.324215791384803e+00 + 1.321097378844990e+00 + 1.317986290481943e+00 + 1.314882509034005e+00 + 1.311786015892630e+00 + 1.308696795174431e+00 + 1.305614830505543e+00 + 1.302540104058677e+00 + 1.299472598550161e+00 + 1.296412297281467e+00 + 1.293359183920592e+00 + 1.290313241355424e+00 + 1.287274452437336e+00 + 1.284242800425524e+00 + 1.281218268758570e+00 + 1.278200840971845e+00 + 1.275190500607243e+00 + 1.272187230813499e+00 + 1.269191014801793e+00 + 1.266201836184906e+00 + 1.263219678244913e+00 + 1.260244524601725e+00 + 1.257276359774731e+00 + 1.254315166659556e+00 + 1.251360928048363e+00 + 1.248413628680403e+00 + 1.245473252809453e+00 + 1.242539783866816e+00 + 1.239613204552696e+00 + 1.236693499248504e+00 + 1.233780652886690e+00 + 1.230874648458874e+00 + 1.227975469917747e+00 + 1.225083101914005e+00 + 1.222197528080320e+00 + 1.219318732110022e+00 + 1.216446698153004e+00 + 1.213581411119631e+00 + 1.210722855169846e+00 + 1.207871013783067e+00 + 1.205025871252235e+00 + 1.202187412198626e+00 + 1.199355621285526e+00 + 1.196530482820713e+00 + 1.193711981112556e+00 + 1.190900100540550e+00 + 1.188094825240966e+00 + 1.185296139829988e+00 + 1.182504029475636e+00 + 1.179718478336080e+00 + 1.176939470878376e+00 + 1.174166992368249e+00 + 1.171401026812090e+00 + 1.168641558834780e+00 + 1.165888574598542e+00 + 1.163142057955574e+00 + 1.160401992993886e+00 + 1.157668365957760e+00 + 1.154941161055026e+00 + 1.152220362585152e+00 + 1.149505957086653e+00 + 1.146797929075238e+00 + 1.144096262593592e+00 + 1.141400943461258e+00 + 1.138711957183691e+00 + 1.136029288721929e+00 + 1.133352922646698e+00 + 1.130682844729473e+00 + 1.128019040871752e+00 + 1.125361495026299e+00 + 1.122710192813904e+00 + 1.120065120718956e+00 + 1.117426263156838e+00 + 1.114793605478939e+00 + 1.112167133798269e+00 + 1.109546832911627e+00 + 1.106932688501257e+00 + 1.104324686965964e+00 + 1.101722813149718e+00 + 1.099127052563592e+00 + 1.096537391606932e+00 + 1.093953815681381e+00 + 1.091376310184498e+00 + 1.088804860939953e+00 + 1.086239454210304e+00 + 1.083680075729654e+00 + 1.081126710640132e+00 + 1.078579345517792e+00 + 1.076037966489376e+00 + 1.073502558540405e+00 + 1.070973108232868e+00 + 1.068449602380304e+00 + 1.065932027008114e+00 + 1.063420367175204e+00 + 1.060914608734753e+00 + 1.058414739347230e+00 + 1.055920744292059e+00 + 1.053432609053344e+00 + 1.050950321750202e+00 + 1.048473868029229e+00 + 1.046003233045439e+00 + 1.043538404185668e+00 + 1.041079368203120e+00 + 1.038626111099421e+00 + 1.036178618557632e+00 + 1.033736877638818e+00 + 1.031300875738098e+00 + 1.028870598541307e+00 + 1.026446032279889e+00 + 1.024027163804816e+00 + 1.021613979927458e+00 + 1.019206467749757e+00 + 1.016804614201984e+00 + 1.014408404816817e+00 + 1.012017826403038e+00 + 1.009632866898133e+00 + 1.007253512894913e+00 + 1.004879750675442e+00 + 1.002511566831996e+00 + 1.000148949144661e+00 + 9.977918847145494e-01 + 9.954403597206510e-01 + 9.930943614528360e-01 + 9.907538771638876e-01 + 9.884188936740620e-01 + 9.860893982148410e-01 + 9.837653780132122e-01 + 9.814468200970017e-01 + 9.791337116433633e-01 + 9.768260399168340e-01 + 9.745237922184445e-01 + 9.722269558574113e-01 + 9.699355181124353e-01 + 9.676494662336389e-01 + 9.653687876978898e-01 + 9.630934699744002e-01 + 9.608235003289306e-01 + 9.585588662885656e-01 + 9.562995554410040e-01 + 9.540455551859868e-01 + 9.517968530931009e-01 + 9.495534368089285e-01 + 9.473152938858177e-01 + 9.450824119343849e-01 + 9.428547786515334e-01 + 9.406323818265262e-01 + 9.384152091378831e-01 + 9.362032482272097e-01 + 9.339964869112888e-01 + 9.317949130473703e-01 + 9.295985145023118e-01 + 9.274072791539573e-01 + 9.252211948367504e-01 + 9.230402494034374e-01 + 9.208644309329761e-01 + 9.186937274260252e-01 + 9.165281267878318e-01 + 9.143676170947915e-01 + 9.122121864230248e-01 + 9.100618228160294e-01 + 9.079165144228375e-01 + 9.057762494101416e-01 + 9.036410159434681e-01 + 9.015108022852656e-01 + 8.993855966397851e-01 + 8.972653871281971e-01 + 8.951501621417270e-01 + 8.930399100640233e-01 + 8.909346191274636e-01 + 8.888342777366450e-01 + 8.867388743114113e-01 + 8.846483971822027e-01 + 8.825628348501290e-01 + 8.804821758633001e-01 + 8.784064086961259e-01 + 8.763355218399448e-01 + 8.742695038377348e-01 + 8.722083433110044e-01 + 8.701520288632569e-01 + 8.681005490981059e-01 + 8.660538926659707e-01 + 8.640120483003465e-01 + 8.619750047630993e-01 + 8.599427507590320e-01 + 8.579152750196485e-01 + 8.558925663402844e-01 + 8.538746136199826e-01 + 8.518614056598554e-01 + 8.498529312415619e-01 + 8.478491793952468e-01 + 8.458501390477595e-01 + 8.438557990450950e-01 + 8.418661484572341e-01 + 8.398811762963594e-01 + 8.379008715007146e-01 + 8.359252231704962e-01 + 8.339542204095658e-01 + 8.319878522912720e-01 + 8.300261079597043e-01 + 8.280689765769632e-01 + 8.261164473121154e-01 + 8.241685093974929e-01 + 8.222251520716921e-01 + 8.202863645582569e-01 + 8.183521361157176e-01 + 8.164224560764958e-01 + 8.144973138478017e-01 + 8.125766987264839e-01 + 8.106606000443977e-01 + 8.087490072689457e-01 + 8.068419098143643e-01 + 8.049392971238847e-01 + 8.030411587461223e-01 + 8.011474841608780e-01 + 7.992582628576694e-01 + 7.973734844327619e-01 + 7.954931384582111e-01 + 7.936172145055983e-01 + 7.917457022017643e-01 + 7.898785912282000e-01 + 7.880158712799969e-01 + 7.861575320154112e-01 + 7.843035631939634e-01 + 7.824539546080472e-01 + 7.806086959262468e-01 + 7.787677769423338e-01 + 7.769311875560446e-01 + 7.750989176225404e-01 + 7.732709569908990e-01 + 7.714472955211897e-01 + 7.696279231052234e-01 + 7.678128297211314e-01 + 7.660020053954494e-01 + 7.641954400346342e-01 + 7.623931236302374e-01 + 7.605950462809293e-01 + 7.588011979868697e-01 + 7.570115688113509e-01 + 7.552261489288551e-01 + 7.534449284973839e-01 + 7.516678976340316e-01 + 7.498950464317325e-01 + 7.481263651057625e-01 + 7.463618439154999e-01 + 7.446014731170009e-01 + 7.428452429690515e-01 + 7.410931437272748e-01 + 7.393451656574506e-01 + 7.376012991597445e-01 + 7.358615346418724e-01 + 7.341258624174279e-01 + 7.323942728636854e-01 + 7.306667564033948e-01 + 7.289433034811010e-01 + 7.272239046205010e-01 + 7.255085503342084e-01 + 7.237972310231134e-01 + 7.220899372662432e-01 + 7.203866596915919e-01 + 7.186873887451811e-01 + 7.169921150530395e-01 + 7.153008293485266e-01 + 7.136135222426403e-01 + 7.119301843519804e-01 + 7.102508063464710e-01 + 7.085753789925818e-01 + 7.069038930034782e-01 + 7.052363390742027e-01 + 7.035727080638011e-01 + 7.019129907750439e-01 + 7.002571779481277e-01 + 6.986052604437083e-01 + 6.969572291357313e-01 + 6.953130748988353e-01 + 6.936727887020229e-01 + 6.920363614522840e-01 + 6.904037839913317e-01 + 6.887750473773369e-01 + 6.871501426160253e-01 + 6.855290605871678e-01 + 6.839117924040222e-01 + 6.822983292013320e-01 + 6.806886620123715e-01 + 6.790827818618205e-01 + 6.774806798407976e-01 + 6.758823471448717e-01 + 6.742877749381793e-01 + 6.726969543916550e-01 + 6.711098767181217e-01 + 6.695265330883416e-01 + 6.679469147191758e-01 + 6.663710129607012e-01 + 6.647988190816884e-01 + 6.632303243195694e-01 + 6.616655199754741e-01 + 6.601043974442724e-01 + 6.585469481315596e-01 + 6.569931633409668e-01 + 6.554430344698831e-01 + 6.538965529680326e-01 + 6.523537102216029e-01 + 6.508144977390269e-01 + 6.492789070902356e-01 + 6.477469297088367e-01 + 6.462185570471877e-01 + 6.446937806333431e-01 + 6.431725921186934e-01 + 6.416549831028872e-01 + 6.401409451291091e-01 + 6.386304698205402e-01 + 6.371235488250266e-01 + 6.356201737964903e-01 + 6.341203364004929e-01 + 6.326240283504814e-01 + 6.311312414023209e-01 + 6.296419672600960e-01 + 6.281561976359686e-01 + 6.266739242956670e-01 + 6.251951391194286e-01 + 6.237198339322255e-01 + 6.222480004492102e-01 + 6.207796305731890e-01 + 6.193147162378443e-01 + 6.178532493027741e-01 + 6.163952216645050e-01 + 6.149406252489860e-01 + 6.134894519984210e-01 + 6.120416938491114e-01 + 6.105973427932988e-01 + 6.091563909278429e-01 + 6.077188302116784e-01 + 6.062846526061169e-01 + 6.048538502614210e-01 + 6.034264152733694e-01 + 6.020023397068369e-01 + 6.005816156757759e-01 + 5.991642353027361e-01 + 5.977501907311656e-01 + 5.963394741611191e-01 + 5.949320777981851e-01 + 5.935279938331314e-01 + 5.921272144271874e-01 + 5.907297318514260e-01 + 5.893355384452119e-01 + 5.879446264415406e-01 + 5.865569881004723e-01 + 5.851726157460215e-01 + 5.837915017610071e-01 + 5.824136384878618e-01 + 5.810390182425506e-01 + 5.796676334763571e-01 + 5.782994765837256e-01 + 5.769345398883866e-01 + 5.755728159164172e-01 + 5.742142971757567e-01 + 5.728589760824728e-01 + 5.715068451452269e-01 + 5.701578968814059e-01 + 5.688121237820786e-01 + 5.674695184134964e-01 + 5.661300733449661e-01 + 5.647937811172017e-01 + 5.634606343850478e-01 + 5.621306258016221e-01 + 5.608037479323033e-01 + 5.594799934047830e-01 + 5.581593548937616e-01 + 5.568418251014721e-01 + 5.555273967809645e-01 + 5.542160626584934e-01 + 5.529078153599613e-01 + 5.516026476869607e-01 + 5.503005524879265e-01 + 5.490015224489507e-01 + 5.477055503706971e-01 + 5.464126291394721e-01 + 5.451227516109934e-01 + 5.438359105997491e-01 + 5.425520989229732e-01 + 5.412713094911021e-01 + 5.399935352390902e-01 + 5.387187691034094e-01 + 5.374470040087604e-01 + 5.361782329113639e-01 + 5.349124487848638e-01 + 5.336496445514358e-01 + 5.323898132502169e-01 + 5.311329480044003e-01 + 5.298790417206248e-01 + 5.286280874266170e-01 + 5.273800783238198e-01 + 5.261350074387794e-01 + 5.248928678300107e-01 + 5.236536526638225e-01 + 5.224173550711992e-01 + 5.211839681868924e-01 + 5.199534851768893e-01 + 5.187258992380968e-01 + 5.175012035574781e-01 + 5.162793913049719e-01 + 5.150604557657682e-01 + 5.138443902158373e-01 + 5.126311878387027e-01 + 5.114208419243966e-01 + 5.102133457996210e-01 + 5.090086927559088e-01 + 5.078068761246483e-01 + 5.066078892445277e-01 + 5.054117254264059e-01 + 5.042183780707232e-01 + 5.030278406156905e-01 + 5.018401064512819e-01 + 5.006551689399448e-01 + 4.994730214827681e-01 + 4.982936576173500e-01 + 4.971170708196209e-01 + 4.959432545161117e-01 + 4.947722021784954e-01 + 4.936039073158257e-01 + 4.924383634713059e-01 + 4.912755642215030e-01 + 4.901155031029966e-01 + 4.889581736357913e-01 + 4.878035694426762e-01 + 4.866516841617155e-01 + 4.855025114095978e-01 + 4.843560447562852e-01 + 4.832122778219742e-01 + 4.820712043014287e-01 + 4.809328179063641e-01 + 4.797971123302187e-01 + 4.786640812406847e-01 + 4.775337183184862e-01 + 4.764060173050261e-01 + 4.752809720122764e-01 + 4.741585762111217e-01 + 4.730388236459319e-01 + 4.719217080626099e-01 + 4.708072233097221e-01 + 4.696953632391141e-01 + 4.685861216425942e-01 + 4.674794923970907e-01 + 4.663754693817960e-01 + 4.652740464091041e-01 + 4.641752174102053e-01 + 4.630789763262008e-01 + 4.619853169869301e-01 + 4.608942333544824e-01 + 4.598057194461572e-01 + 4.587197692038988e-01 + 4.576363766028796e-01 + 4.565555356433393e-01 + 4.554772403132010e-01 + 4.544014845983438e-01 + 4.533282625263808e-01 + 4.522575682421708e-01 + 4.511893957834002e-01 + 4.501237391381080e-01 + 4.490605924783677e-01 + 4.479999499003729e-01 + 4.469418054464263e-01 + 4.458861533467765e-01 + 4.448329877503518e-01 + 4.437823027182840e-01 + 4.427340924804984e-01 + 4.416883512661691e-01 + 4.406450732493647e-01 + 4.396042525949710e-01 + 4.385658835318161e-01 + 4.375299603591805e-01 + 4.364964773180230e-01 + 4.354654286529535e-01 + 4.344368086449737e-01 + 4.334106115882546e-01 + 4.323868317973252e-01 + 4.313654636098987e-01 + 4.303465013528135e-01 + 4.293299393608014e-01 + 4.283157719943348e-01 + 4.273039936389254e-01 + 4.262945986862648e-01 + 4.252875815207442e-01 + 4.242829365388910e-01 + 4.232806581737543e-01 + 4.222807409163533e-01 + 4.212831792180283e-01 + 4.202879675160039e-01 + 4.192951002821390e-01 + 4.183045720190468e-01 + 4.173163772308877e-01 + 4.163305103895081e-01 + 4.153469660738844e-01 + 4.143657388899517e-01 + 4.133868232899575e-01 + 4.124102138463511e-01 + 4.114359052122226e-01 + 4.104638919136763e-01 + 4.094941685453110e-01 + 4.085267297797091e-01 + 4.075615702608746e-01 + 4.065986846247451e-01 + 4.056380675149961e-01 + 4.046797136100241e-01 + 4.037236175791848e-01 + 4.027697740867438e-01 + 4.018181778668817e-01 + 4.008688236537185e-01 + 3.999217061664900e-01 + 3.989768201804763e-01 + 3.980341604347806e-01 + 3.970937216243767e-01 + 3.961554986257766e-01 + 3.952194862561716e-01 + 3.942856791749246e-01 + 3.933540722797152e-01 + 3.924246604937682e-01 + 3.914974386057962e-01 + 3.905724013991230e-01 + 3.896495437210211e-01 + 3.887288605266558e-01 + 3.878103467384126e-01 + 3.868939972364765e-01 + 3.859798068708922e-01 + 3.850677705778728e-01 + 3.841578833358520e-01 + 3.832501400994028e-01 + 3.823445358203205e-01 + 3.814410654593080e-01 + 3.805397240004402e-01 + 3.796405064500310e-01 + 3.787434078281850e-01 + 3.778484231550217e-01 + 3.769555474293507e-01 + 3.760647756653435e-01 + 3.751761029803938e-01 + 3.742895244689736e-01 + 3.734050351897463e-01 + 3.725226302131253e-01 + 3.716423046174657e-01 + 3.707640535037849e-01 + 3.698878720450124e-01 + 3.690137553825144e-01 + 3.681416986254492e-01 + 3.672716969834960e-01 + 3.664037456169829e-01 + 3.655378396196581e-01 + 3.646739742353085e-01 + 3.638121447254376e-01 + 3.629523462980791e-01 + 3.620945741078652e-01 + 3.612388233782960e-01 + 3.603850894480597e-01 + 3.595333675652969e-01 + 3.586836529581271e-01 + 3.578359408972814e-01 + 3.569902267099870e-01 + 3.561465057284826e-01 + 3.553047732480240e-01 + 3.544650245489661e-01 + 3.536272549462773e-01 + 3.527914598373741e-01 + 3.519576346281267e-01 + 3.511257746893044e-01 + 3.502958753213151e-01 + 3.494679319027064e-01 + 3.486419398734283e-01 + 3.478178946841421e-01 + 3.469957917542031e-01 + 3.461756264906874e-01 + 3.453573943334770e-01 + 3.445410907466465e-01 + 3.437267112053919e-01 + 3.429142511753236e-01 + 3.421037061357636e-01 + 3.412950715914754e-01 + 3.404883430871680e-01 + 3.396835161220511e-01 + 3.388805861778083e-01 + 3.380795488742585e-01 + 3.372803997559251e-01 + 3.364831342820857e-01 + 3.356877480644493e-01 + 3.348942367473924e-01 + 3.341025959445759e-01 + 3.333128211902062e-01 + 3.325249080644813e-01 + 3.317388522349806e-01 + 3.309546493345364e-01 + 3.301722950123494e-01 + 3.293917849546400e-01 + 3.286131147929292e-01 + 3.278362801760134e-01 + 3.270612768173198e-01 + 3.262881004205997e-01 + 3.255167466776044e-01 + 3.247472112772871e-01 + 3.239794899591130e-01 + 3.232135784952971e-01 + 3.224494726613195e-01 + 3.216871681318617e-01 + 3.209266606147330e-01 + 3.201679460077643e-01 + 3.194110200806869e-01 + 3.186558785671545e-01 + 3.179025173467112e-01 + 3.171509322207308e-01 + 3.164011189537259e-01 + 3.156530734030461e-01 + 3.149067914068442e-01 + 3.141622688005587e-01 + 3.134195014964591e-01 + 3.126784853846941e-01 + 3.119392163234083e-01 + 3.112016901727622e-01 + 3.104659028247113e-01 + 3.097318502149898e-01 + 3.089995283319454e-01 + 3.082689330721005e-01 + 3.075400602663161e-01 + 3.068129059444455e-01 + 3.060874661234649e-01 + 3.053637367362504e-01 + 3.046417137079979e-01 + 3.039213930418579e-01 + 3.032027708160949e-01 + 3.024858429572295e-01 + 3.017706054435225e-01 + 3.010570543910775e-01 + 3.003451858060657e-01 + 2.996349956970866e-01 + 2.989264801519126e-01 + 2.982196352437723e-01 + 2.975144570351090e-01 + 2.968109415899733e-01 + 2.961090850010212e-01 + 2.954088833712375e-01 + 2.947103327960572e-01 + 2.940134293733013e-01 + 2.933181692489545e-01 + 2.926245486550200e-01 + 2.919325636534309e-01 + 2.912422103009782e-01 + 2.905534849067459e-01 + 2.898663836341845e-01 + 2.891809025716850e-01 + 2.884970379499296e-01 + 2.878147859946116e-01 + 2.871341429080666e-01 + 2.864551048929091e-01 + 2.857776681836698e-01 + 2.851018290276999e-01 + 2.844275836247271e-01 + 2.837549281975342e-01 + 2.830838590248334e-01 + 2.824143724756855e-01 + 2.817464648036653e-01 + 2.810801321714598e-01 + 2.804153709678348e-01 + 2.797521775256636e-01 + 2.790905480675208e-01 + 2.784304789557904e-01 + 2.777719665515425e-01 + 2.771150071618110e-01 + 2.764595971293084e-01 + 2.758057328280932e-01 + 2.751534106502780e-01 + 2.745026269461392e-01 + 2.738533780697315e-01 + 2.732056604090851e-01 + 2.725594703611444e-01 + 2.719148043516682e-01 + 2.712716588414341e-01 + 2.706300302048149e-01 + 2.699899148252890e-01 + 2.693513091897057e-01 + 2.687142097803046e-01 + 2.680786130525787e-01 + 2.674445154290628e-01 + 2.668119133864937e-01 + 2.661808034370396e-01 + 2.655511820876730e-01 + 2.649230457933240e-01 + 2.642963910212357e-01 + 2.636712143601993e-01 + 2.630475123173707e-01 + 2.624252813696634e-01 + 2.618045181320832e-01 + 2.611852191534089e-01 + 2.605673809346502e-01 + 2.599510000768786e-01 + 2.593360731482656e-01 + 2.587225966857610e-01 + 2.581105673020023e-01 + 2.574999816218594e-01 + 2.568908362541323e-01 + 2.562831277681717e-01 + 2.556768527608984e-01 + 2.550720078808991e-01 + 2.544685898158210e-01 + 2.538665952029344e-01 + 2.532660206235198e-01 + 2.526668627769018e-01 + 2.520691183671673e-01 + 2.514727840477828e-01 + 2.508778564832650e-01 + 2.502843323450471e-01 + 2.496922083184503e-01 + 2.491014811741685e-01 + 2.485121476502302e-01 + 2.479242043768022e-01 + 2.473376480750936e-01 + 2.467524755330193e-01 + 2.461686835577825e-01 + 2.455862688342147e-01 + 2.450052280577099e-01 + 2.444255580815592e-01 + 2.438472556710649e-01 + 2.432703175621061e-01 + 2.426947405763887e-01 + 2.421205215374063e-01 + 2.415476572469543e-01 + 2.409761444715131e-01 + 2.404059800179439e-01 + 2.398371607406130e-01 + 2.392696835287547e-01 + 2.387035451998669e-01 + 2.381387425460481e-01 + 2.375752724876774e-01 + 2.370131319015487e-01 + 2.364523176042576e-01 + 2.358928264341899e-01 + 2.353346553264918e-01 + 2.347778012643444e-01 + 2.342222610470114e-01 + 2.336680315481888e-01 + 2.331151097630285e-01 + 2.325634926002070e-01 + 2.320131769932555e-01 + 2.314641599195968e-01 + 2.309164382391606e-01 + 2.303700088852327e-01 + 2.298248689205450e-01 + 2.292810152383314e-01 + 2.287384447851956e-01 + 2.281971546639490e-01 + 2.276571417710518e-01 + 2.271184030444375e-01 + 2.265809356255705e-01 + 2.260447364485637e-01 + 2.255098024678978e-01 + 2.249761308577043e-01 + 2.244437185775247e-01 + 2.239125625576918e-01 + 2.233826599324974e-01 + 2.228540078061592e-01 + 2.223266032159731e-01 + 2.218004431318401e-01 + 2.212755246546400e-01 + 2.207518449320089e-01 + 2.202294009962271e-01 + 2.197081899542086e-01 + 2.191882089472320e-01 + 2.186694550184439e-01 + 2.181519252528444e-01 + 2.176356167922184e-01 + 2.171205268020538e-01 + 2.166066524055817e-01 + 2.160939907132310e-01 + 2.155825389406823e-01 + 2.150722942170351e-01 + 2.145632535999474e-01 + 2.140554143128364e-01 + 2.135487735713488e-01 + 2.130433285376680e-01 + 2.125390763961598e-01 + 2.120360143225986e-01 + 2.115341394866721e-01 + 2.110334491301104e-01 + 2.105339404911263e-01 + 2.100356107692428e-01 + 2.095384571593845e-01 + 2.090424768875718e-01 + 2.085476672286868e-01 + 2.080540254649787e-01 + 2.075615488420417e-01 + 2.070702345483775e-01 + 2.065800798220710e-01 + 2.060910819584345e-01 + 2.056032383022174e-01 + 2.051165461242101e-01 + 2.046310026800104e-01 + 2.041466052775261e-01 + 2.036633511978056e-01 + 2.031812377402102e-01 + 2.027002622847228e-01 + 2.022204221329609e-01 + 2.017417145657832e-01 + 2.012641369577989e-01 + 2.007876866366623e-01 + 2.003123609141739e-01 + 1.998381571847571e-01 + 1.993650728223044e-01 + 1.988931051658790e-01 + 1.984222515391579e-01 + 1.979525093566432e-01 + 1.974838760798574e-01 + 1.970163490394448e-01 + 1.965499255924087e-01 + 1.960846031634822e-01 + 1.956203792216227e-01 + 1.951572511456189e-01 + 1.946952162469976e-01 + 1.942342720345011e-01 + 1.937744160087473e-01 + 1.933156455855021e-01 + 1.928579581617409e-01 + 1.924013511601302e-01 + 1.919458220512063e-01 + 1.914913683490730e-01 + 1.910379875218400e-01 + 1.905856769690048e-01 + 1.901344342343178e-01 + 1.896842568245793e-01 + 1.892351421122624e-01 + 1.887870876905098e-01 + 1.883400911382459e-01 + 1.878941498265747e-01 + 1.874492612709913e-01 + 1.870054230608876e-01 + 1.865626327533365e-01 + 1.861208878413260e-01 + 1.856801858136446e-01 + 1.852405242290795e-01 + 1.848019006561455e-01 + 1.843643126785636e-01 + 1.839277579090688e-01 + 1.834922338318696e-01 + 1.830577379260484e-01 + 1.826242679175190e-01 + 1.821918214082343e-01 + 1.817603959062191e-01 + 1.813299890419420e-01 + 1.809005984096844e-01 + 1.804722215715290e-01 + 1.800448561676805e-01 + 1.796184998471616e-01 + 1.791931502428031e-01 + 1.787688049585256e-01 + 1.783454616010165e-01 + 1.779231178061076e-01 + 1.775017712916434e-01 + 1.770814197185338e-01 + 1.766620606643876e-01 + 1.762436917707559e-01 + 1.758263107475443e-01 + 1.754099153420786e-01 + 1.749945031691721e-01 + 1.745800718793870e-01 + 1.741666192327871e-01 + 1.737541428912765e-01 + 1.733426405128906e-01 + 1.729321098263452e-01 + 1.725225486037993e-01 + 1.721139545737653e-01 + 1.717063253659770e-01 + 1.712996586978482e-01 + 1.708939523524782e-01 + 1.704892041326627e-01 + 1.700854117308264e-01 + 1.696825728315613e-01 + 1.692806852413719e-01 + 1.688797467224855e-01 + 1.684797550119726e-01 + 1.680807078836749e-01 + 1.676826031516292e-01 + 1.672854386236076e-01 + 1.668892120257625e-01 + 1.664939211364053e-01 + 1.660995637803862e-01 + 1.657061377653481e-01 + 1.653136408872399e-01 + 1.649220709518862e-01 + 1.645314258210089e-01 + 1.641417032838678e-01 + 1.637529010922642e-01 + 1.633650171697226e-01 + 1.629780493848234e-01 + 1.625919955117979e-01 + 1.622068533722165e-01 + 1.618226208359533e-01 + 1.614392958070308e-01 + 1.610568761629498e-01 + 1.606753597398519e-01 + 1.602947443525945e-01 + 1.599150279328092e-01 + 1.595362083976419e-01 + 1.591582835876941e-01 + 1.587812513806259e-01 + 1.584051096895935e-01 + 1.580298564486488e-01 + 1.576554895336735e-01 + 1.572820068494865e-01 + 1.569094063844064e-01 + 1.565376859935812e-01 + 1.561668435389086e-01 + 1.557968770322644e-01 + 1.554277844430587e-01 + 1.550595637013555e-01 + 1.546922127252644e-01 + 1.543257294352004e-01 + 1.539601117962810e-01 + 1.535953578647676e-01 + 1.532314655336160e-01 + 1.528684326732572e-01 + 1.525062574192351e-01 + 1.521449377506904e-01 + 1.517844715391133e-01 + 1.514248568043021e-01 + 1.510660915767141e-01 + 1.507081738546517e-01 + 1.503511015850067e-01 + 1.499948728018673e-01 + 1.496394855976185e-01 + 1.492849379312514e-01 + 1.489312277844460e-01 + 1.485783532079666e-01 + 1.482263122946834e-01 + 1.478751030581660e-01 + 1.475247234388987e-01 + 1.471751715323068e-01 + 1.468264454469171e-01 + 1.464785432366154e-01 + 1.461314629165984e-01 + 1.457852025452889e-01 + 1.454397602433564e-01 + 1.450951340257908e-01 + 1.447513219420110e-01 + 1.444083221508569e-01 + 1.440661327109938e-01 + 1.437247516822454e-01 + 1.433841772072484e-01 + 1.430444073901374e-01 + 1.427054403077353e-01 + 1.423672740371242e-01 + 1.420299067307786e-01 + 1.416933365389871e-01 + 1.413575615237460e-01 + 1.410225798197031e-01 + 1.406883895901710e-01 + 1.403549889435149e-01 + 1.400223760032770e-01 + 1.396905489310608e-01 + 1.393595059398338e-01 + 1.390292451224472e-01 + 1.386997645496112e-01 + 1.383710625136722e-01 + 1.380431371824567e-01 + 1.377159866326167e-01 + 1.373896091055998e-01 + 1.370640027752214e-01 + 1.367391657592925e-01 + 1.364150963290927e-01 + 1.360917926783734e-01 + 1.357692529162330e-01 + 1.354474752994454e-01 + 1.351264580487222e-01 + 1.348061993122180e-01 + 1.344866973181728e-01 + 1.341679503229942e-01 + 1.338499565682958e-01 + 1.335327142013654e-01 + 1.332162214320106e-01 + 1.329004765902875e-01 + 1.325854779258548e-01 + 1.322712236439755e-01 + 1.319577119415115e-01 + 1.316449410589970e-01 + 1.313329092858229e-01 + 1.310216149439378e-01 + 1.307110562100912e-01 + 1.304012313067827e-01 + 1.300921386594473e-01 + 1.297837764741680e-01 + 1.294761429396481e-01 + 1.291692364743583e-01 + 1.288630553353233e-01 + 1.285575977268189e-01 + 1.282528620001082e-01 + 1.279488464604472e-01 + 1.276455493846629e-01 + 1.273429690961771e-01 + 1.270411039265308e-01 + 1.267399521912149e-01 + 1.264395121606617e-01 + 1.261397821624623e-01 + 1.258407605612416e-01 + 1.255424456647032e-01 + 1.252448358003494e-01 + 1.249479293256652e-01 + 1.246517245985584e-01 + 1.243562199440227e-01 + 1.240614136699035e-01 + 1.237673041530004e-01 + 1.234738897833225e-01 + 1.231811689369122e-01 + 1.228891399541342e-01 + 1.225978011897444e-01 + 1.223071510285007e-01 + 1.220171878485729e-01 + 1.217279100221359e-01 + 1.214393159216827e-01 + 1.211514039479249e-01 + 1.208641725134118e-01 + 1.205776200272462e-01 + 1.202917448643591e-01 + 1.200065453999493e-01 + 1.197220200388544e-01 + 1.194381672203820e-01 + 1.191549853738593e-01 + 1.188724728867774e-01 + 1.185906281982059e-01 + 1.183094497403516e-01 + 1.180289358802264e-01 + 1.177490851172322e-01 + 1.174698959467804e-01 + 1.171913666738082e-01 + 1.169134957966049e-01 + 1.166362818664215e-01 + 1.163597231976005e-01 + 1.160838182295854e-01 + 1.158085654926154e-01 + 1.155339634301463e-01 + 1.152600105152270e-01 + 1.149867052503320e-01 + 1.147140461081585e-01 + 1.144420315115333e-01 + 1.141706598827868e-01 + 1.138999297876022e-01 + 1.136298397724347e-01 + 1.133603883121320e-01 + 1.130915738073908e-01 + 1.128233947404400e-01 + 1.125558496925181e-01 + 1.122889371862514e-01 + 1.120226557224089e-01 + 1.117570037925019e-01 + 1.114919798574519e-01 + 1.112275824488763e-01 + 1.109638101830490e-01 + 1.107006615250319e-01 + 1.104381349837027e-01 + 1.101762291935612e-01 + 1.099149425817940e-01 + 1.096542736337556e-01 + 1.093942210611990e-01 + 1.091347833750166e-01 + 1.088759590309823e-01 + 1.086177465849946e-01 + 1.083601446468375e-01 + 1.081031518187680e-01 + 1.078467666394599e-01 + 1.075909876365943e-01 + 1.073358133695979e-01 + 1.070812424691808e-01 + 1.068272734935852e-01 + 1.065739049762412e-01 + 1.063211355273724e-01 + 1.060689637753818e-01 + 1.058173883167701e-01 + 1.055664076522099e-01 + 1.053160204177178e-01 + 1.050662253083730e-01 + 1.048170208178578e-01 + 1.045684055544151e-01 + 1.043203782231123e-01 + 1.040729373745031e-01 + 1.038260815947437e-01 + 1.035798095388146e-01 + 1.033341198496129e-01 + 1.030890111353506e-01 + 1.028444819874746e-01 + 1.026005310752415e-01 + 1.023571570589419e-01 + 1.021143585605686e-01 + 1.018721342063349e-01 + 1.016304826397723e-01 + 1.013894025261397e-01 + 1.011488925387934e-01 + 1.009089513225976e-01 + 1.006695774897203e-01 + 1.004307697412456e-01 + 1.001925267601519e-01 + 9.995484714928184e-02 + 9.971772961029543e-02 + 9.948117285600222e-02 + 9.924517553207431e-02 + 9.900973631553432e-02 + 9.877485389836264e-02 + 9.854052696321766e-02 + 9.830675420513860e-02 + 9.807353432053509e-02 + 9.784086599274402e-02 + 9.760874792461041e-02 + 9.737717882679013e-02 + 9.714615739564664e-02 + 9.691568233864475e-02 + 9.668575237168565e-02 + 9.645636620785530e-02 + 9.622752256709627e-02 + 9.599922017186617e-02 + 9.577145773476621e-02 + 9.554423398421420e-02 + 9.531754766036228e-02 + 9.509139748951084e-02 + 9.486578220406576e-02 + 9.464070054491446e-02 + 9.441615125061270e-02 + 9.419213306609350e-02 + 9.396864474309072e-02 + 9.374568502973517e-02 + 9.352325267700241e-02 + 9.330134644110695e-02 + 9.307996508088016e-02 + 9.285910736057421e-02 + 9.263877204899980e-02 + 9.241895790580944e-02 + 9.219966369924239e-02 + 9.198088821288498e-02 + 9.176263021506216e-02 + 9.154488848131032e-02 + 9.132766180778470e-02 + 9.111094897456264e-02 + 9.089474876186458e-02 + 9.067905996494766e-02 + 9.046388137755315e-02 + 9.024921179550818e-02 + 9.003505002157913e-02 + 8.982139485394958e-02 + 8.960824509461397e-02 + 8.939559956013639e-02 + 8.918345705664489e-02 + 8.897181638949078e-02 + 8.876067638185769e-02 + 8.855003585659860e-02 + 8.833989363307079e-02 + 8.813024852794749e-02 + 8.792109937217552e-02 + 8.771244500411417e-02 + 8.750428424959714e-02 + 8.729661594151820e-02 + 8.708943892038122e-02 + 8.688275202450782e-02 + 8.667655409894154e-02 + 8.647084399500883e-02 + 8.626562056133152e-02 + 8.606088264727801e-02 + 8.585662910508592e-02 + 8.565285879275303e-02 + 8.544957056834111e-02 + 8.524676329099723e-02 + 8.504443583456130e-02 + 8.484258706983651e-02 + 8.464121585953427e-02 + 8.444032107793580e-02 + 8.423990160173883e-02 + 8.403995630660985e-02 + 8.384048408000297e-02 + 8.364148380739438e-02 + 8.344295436518863e-02 + 8.324489464319758e-02 + 8.304730353631981e-02 + 8.285017993778075e-02 + 8.265352274723796e-02 + 8.245733086610936e-02 + 8.226160319325253e-02 + 8.206633863307464e-02 + 8.187153609323948e-02 + 8.167719448173898e-02 + 8.148331271120264e-02 + 8.128988969775824e-02 + 8.109692435907066e-02 + 8.090441561287741e-02 + 8.071236238052074e-02 + 8.052076359329230e-02 + 8.032961817777003e-02 + 8.013892505875637e-02 + 7.994868317110082e-02 + 7.975889145271257e-02 + 7.956954884117817e-02 + 7.938065427030368e-02 + 7.919220668490153e-02 + 7.900420503735811e-02 + 7.881664826626886e-02 + 7.862953532075569e-02 + 7.844286516277023e-02 + 7.825663674598571e-02 + 7.807084902471831e-02 + 7.788550095723919e-02 + 7.770059150356423e-02 + 7.751611963202924e-02 + 7.733208431808952e-02 + 7.714848452080215e-02 + 7.696531920714285e-02 + 7.678258736267791e-02 + 7.660028796508990e-02 + 7.641841999039113e-02 + 7.623698241865759e-02 + 7.605597423593993e-02 + 7.587539442823407e-02 + 7.569524197859213e-02 + 7.551551588287067e-02 + 7.533621514035872e-02 + 7.515733874432233e-02 + 7.497888568830134e-02 + 7.480085497230264e-02 + 7.462324560850588e-02 + 7.444605660139321e-02 + 7.426928695221670e-02 + 7.409293566891996e-02 + 7.391700176673981e-02 + 7.374148426439617e-02 + 7.356638217848553e-02 + 7.339169452681224e-02 + 7.321742033027905e-02 + 7.304355861552074e-02 + 7.287010840815517e-02 + 7.269706873346385e-02 + 7.252443862192341e-02 + 7.235221710900047e-02 + 7.218040323296335e-02 + 7.200899602923243e-02 + 7.183799453782064e-02 + 7.166739780328910e-02 + 7.149720486628935e-02 + 7.132741477295296e-02 + 7.115802657678706e-02 + 7.098903933028558e-02 + 7.082045208266741e-02 + 7.065226388294948e-02 + 7.048447379884398e-02 + 7.031708089501212e-02 + 7.015008422428966e-02 + 6.998348285005500e-02 + 6.981727584178653e-02 + 6.965146227138255e-02 + 6.948604121024822e-02 + 6.932101172952453e-02 + 6.915637290177927e-02 + 6.899212381049466e-02 + 6.882826353831664e-02 + 6.866479115768206e-02 + 6.850170575553913e-02 + 6.833900642241270e-02 + 6.817669223943085e-02 + 6.801476230171261e-02 + 6.785321570853013e-02 + 6.769205154760372e-02 + 6.753126891476798e-02 + 6.737086691265436e-02 + 6.721084464422118e-02 + 6.705120121123045e-02 + 6.689193571639933e-02 + 6.673304726820108e-02 + 6.657453497707803e-02 + 6.641639795452120e-02 + 6.625863531334618e-02 + 6.610124617104705e-02 + 6.594422964866814e-02 + 6.578758486543299e-02 + 6.563131094165708e-02 + 6.547540700031643e-02 + 6.531987216906585e-02 + 6.516470557837332e-02 + 6.500990635997374e-02 + 6.485547364331327e-02 + 6.470140656204018e-02 + 6.454770425495959e-02 + 6.439436585565442e-02 + 6.424139050518185e-02 + 6.408877735587415e-02 + 6.393652554954243e-02 + 6.378463422860429e-02 + 6.363310254304622e-02 + 6.348192964216136e-02 + 6.333111467891862e-02 + 6.318065681259971e-02 + 6.303055519506789e-02 + 6.288080898167979e-02 + 6.273141734118348e-02 + 6.258237943694549e-02 + 6.243369443027962e-02 + 6.228536148636779e-02 + 6.213737977420020e-02 + 6.198974846629291e-02 + 6.184246673830131e-02 + 6.169553375986874e-02 + 6.154894870195880e-02 + 6.140271075002260e-02 + 6.125681908485271e-02 + 6.111127288458502e-02 + 6.096607133498923e-02 + 6.082121362330186e-02 + 6.067669893778977e-02 + 6.053252647002724e-02 + 6.038869541057570e-02 + 6.024520494995262e-02 + 6.010205428441878e-02 + 5.995924261312039e-02 + 5.981676913606451e-02 + 5.967463305068377e-02 + 5.953283356000520e-02 + 5.939136987405440e-02 + 5.925024120274393e-02 + 5.910944675075109e-02 + 5.896898572024133e-02 + 5.882885733338872e-02 + 5.868906080796419e-02 + 5.854959534943303e-02 + 5.841046018167366e-02 + 5.827165452667381e-02 + 5.813317759547484e-02 + 5.799502862000658e-02 + 5.785720683145133e-02 + 5.771971144629215e-02 + 5.758254169802959e-02 + 5.744569682168989e-02 + 5.730917604038396e-02 + 5.717297859181381e-02 + 5.703710371771920e-02 + 5.690155065150715e-02 + 5.676631863514842e-02 + 5.663140691176988e-02 + 5.649681471550019e-02 + 5.636254129818796e-02 + 5.622858591623749e-02 + 5.609494780637568e-02 + 5.596162621719655e-02 + 5.582862040612746e-02 + 5.569592962469259e-02 + 5.556355312969728e-02 + 5.543149018219354e-02 + 5.529974004038473e-02 + 5.516830196534387e-02 + 5.503717522021812e-02 + 5.490635906464716e-02 + 5.477585276550550e-02 + 5.464565559606692e-02 + 5.451576682310098e-02 + 5.438618571514530e-02 + 5.425691154653305e-02 + 5.412794360021059e-02 + 5.399928115030668e-02 + 5.387092346220070e-02 + 5.374286982722595e-02 + 5.361511953114147e-02 + 5.348767184376742e-02 + 5.336052605560348e-02 + 5.323368145967647e-02 + 5.310713734132736e-02 + 5.298089299097181e-02 + 5.285494769998652e-02 + 5.272930075794034e-02 + 5.260395145882699e-02 + 5.247889910164562e-02 + 5.235414299012888e-02 + 5.222968242359068e-02 + 5.210551670083303e-02 + 5.198164512479836e-02 + 5.185806699751586e-02 + 5.173478162567831e-02 + 5.161178832737659e-02 + 5.148908640731603e-02 + 5.136667516904735e-02 + 5.124455393713079e-02 + 5.112272202541198e-02 + 5.100117874145336e-02 + 5.087992340415878e-02 + 5.075895533879782e-02 + 5.063827387090040e-02 + 5.051787831631814e-02 + 5.039776799903027e-02 + 5.027794224956293e-02 + 5.015840039018700e-02 + 5.003914174911826e-02 + 4.992016566217095e-02 + 4.980147146451835e-02 + 4.968305848565534e-02 + 4.956492605368701e-02 + 4.944707351707827e-02 + 4.932950021762891e-02 + 4.921220548403189e-02 + 4.909518865563611e-02 + 4.897844908008390e-02 + 4.886198610990230e-02 + 4.874579909103274e-02 + 4.862988736593407e-02 + 4.851425027765261e-02 + 4.839888718215429e-02 + 4.828379743688660e-02 + 4.816898039344981e-02 + 4.805443540666063e-02 + 4.794016183433197e-02 + 4.782615903628512e-02 + 4.771242637003347e-02 + 4.759896319572905e-02 + 4.748576888074151e-02 + 4.737284278529100e-02 + 4.726018427180961e-02 + 4.714779271742187e-02 + 4.703566749191604e-02 + 4.692380796083007e-02 + 4.681221349433169e-02 + 4.670088346648602e-02 + 4.658981725459431e-02 + 4.647901423861556e-02 + 4.636847379587147e-02 + 4.625819530165282e-02 + 4.614817813294777e-02 + 4.603842167462240e-02 + 4.592892531626435e-02 + 4.581968844113481e-02 + 4.571071043517037e-02 + 4.560199068716930e-02 + 4.549352858172214e-02 + 4.538532351139604e-02 + 4.527737487551017e-02 + 4.516968206047602e-02 + 4.506224446147816e-02 + 4.495506148526871e-02 + 4.484813252205796e-02 + 4.474145696749865e-02 + 4.463503423038843e-02 + 4.452886371233129e-02 + 4.442294481444441e-02 + 4.431727694184067e-02 + 4.421185950249874e-02 + 4.410669190344264e-02 + 4.400177354999718e-02 + 4.389710385863231e-02 + 4.379268224561385e-02 + 4.368850811842072e-02 + 4.358458089102907e-02 + 4.348089998166713e-02 + 4.337746480946211e-02 + 4.327427479290449e-02 + 4.317132935104961e-02 + 4.306862790538591e-02 + 4.296616988021409e-02 + 4.286395470044128e-02 + 4.276198178905291e-02 + 4.266025057462805e-02 + 4.255876048879523e-02 + 4.245751096050290e-02 + 4.235650142249979e-02 + 4.225573130868781e-02 + 4.215520004666404e-02 + 4.205490707118502e-02 + 4.195485182346907e-02 + 4.185503374212908e-02 + 4.175545226547943e-02 + 4.165610683241490e-02 + 4.155699688308426e-02 + 4.145812186261490e-02 + 4.135948121993713e-02 + 4.126107439911159e-02 + 4.116290084444992e-02 + 4.106496000347922e-02 + 4.096725133001662e-02 + 4.086977427671171e-02 + 4.077252829257245e-02 + 4.067551282880393e-02 + 4.057872734137224e-02 + 4.048217129089726e-02 + 4.038584413229004e-02 + 4.028974532214836e-02 + 4.019387432351806e-02 + 4.009823059884492e-02 + 4.000281360871143e-02 + 3.990762281263916e-02 + 3.981265768084040e-02 + 3.971791768404272e-02 + 3.962340228400397e-02 + 3.952911094818634e-02 + 3.943504314911767e-02 + 3.934119836154264e-02 + 3.924757605343093e-02 + 3.915417569553582e-02 + 3.906099677494528e-02 + 3.896803876462888e-02 + 3.887530113159737e-02 + 3.878278335807980e-02 + 3.869048493136602e-02 + 3.859840533540703e-02 + 3.850654403935494e-02 + 3.841490053099362e-02 + 3.832347430867683e-02 + 3.823226484521238e-02 + 3.814127162485314e-02 + 3.805049414568009e-02 + 3.795993189727254e-02 + 3.786958436838804e-02 + 3.777945104950908e-02 + 3.768953143238456e-02 + 3.759982501241896e-02 + 3.751033128919119e-02 + 3.742104976390229e-02 + 3.733197993101921e-02 + 3.724312127956968e-02 + 3.715447331892652e-02 + 3.706603555538170e-02 + 3.697780748276271e-02 + 3.688978860838169e-02 + 3.680197844047203e-02 + 3.671437648083043e-02 + 3.662698223859462e-02 + 3.653979522528233e-02 + 3.645281495050149e-02 + 3.636604092401768e-02 + 3.627947265609178e-02 + 3.619310965841749e-02 + 3.610695145022991e-02 + 3.602099755007137e-02 + 3.593524746722872e-02 + 3.584970072102484e-02 + 3.576435683502647e-02 + 3.567921532660211e-02 + 3.559427571435374e-02 + 3.550953752116312e-02 + 3.542500027693078e-02 + 3.534066350384148e-02 + 3.525652672170999e-02 + 3.517258946309138e-02 + 3.508885125628003e-02 + 3.500531162678291e-02 + 3.492197010940873e-02 + 3.483882623525817e-02 + 3.475587953263527e-02 + 3.467312953919303e-02 + 3.459057578876406e-02 + 3.450821781198862e-02 + 3.442605515251815e-02 + 3.434408734999198e-02 + 3.426231393723483e-02 + 3.418073445581654e-02 + 3.409934844816886e-02 + 3.401815545480084e-02 + 3.393715501912345e-02 + 3.385634668688416e-02 + 3.377573000557660e-02 + 3.369530452244349e-02 + 3.361506978424043e-02 + 3.353502533757853e-02 + 3.345517073173793e-02 + 3.337550551749302e-02 + 3.329602924688495e-02 + 3.321674147756795e-02 + 3.313764176353938e-02 + 3.305872964905672e-02 + 3.298000469915312e-02 + 3.290146647870388e-02 + 3.282311452928582e-02 + 3.274494841443989e-02 + 3.266696770524984e-02 + 3.258917195579586e-02 + 3.251156072835543e-02 + 3.243413358906379e-02 + 3.235689009597382e-02 + 3.227982981604406e-02 + 3.220295232152501e-02 + 3.212625717796667e-02 + 3.204974395335523e-02 + 3.197341221860803e-02 + 3.189726154412311e-02 + 3.182129149811844e-02 + 3.174550164996338e-02 + 3.166989158019418e-02 + 3.159446086552403e-02 + 3.151920907660396e-02 + 3.144413578635114e-02 + 3.136924057603768e-02 + 3.129452303273958e-02 + 3.121998273049806e-02 + 3.114561924523454e-02 + 3.107143216011856e-02 + 3.099742105799270e-02 + 3.092358552474080e-02 + 3.084992514925186e-02 + 3.077643951122557e-02 + 3.070312819305190e-02 + 3.062999078663620e-02 + 3.055702688363699e-02 + 3.048423607160382e-02 + 3.041161793396379e-02 + 3.033917207015753e-02 + 3.026689807846873e-02 + 3.019479554128978e-02 + 3.012286404982057e-02 + 3.005110320361238e-02 + 2.997951260681279e-02 + 2.990809185255966e-02 + 2.983684053226568e-02 + 2.976575824919644e-02 + 2.969484460522792e-02 + 2.962409920087884e-02 + 2.955352163824170e-02 + 2.948311151647172e-02 + 2.941286843696999e-02 + 2.934279201383451e-02 + 2.927288185012002e-02 + 2.920313754405053e-02 + 2.913355871301240e-02 + 2.906414496442064e-02 + 2.899489589842786e-02 + 2.892581113440367e-02 + 2.885689028447655e-02 + 2.878813295173805e-02 + 2.871953875071393e-02 + 2.865110730129668e-02 + 2.858283822298987e-02 + 2.851473112218837e-02 + 2.844678561226344e-02 + 2.837900131834860e-02 + 2.831137785774055e-02 + 2.824391484910513e-02 + 2.817661191561155e-02 + 2.810946867322990e-02 + 2.804248474261467e-02 + 2.797565975421473e-02 + 2.790899332866750e-02 + 2.784248508684310e-02 + 2.777613465729838e-02 + 2.770994166752542e-02 + 2.764390574354870e-02 + 2.757802651096095e-02 + 2.751230360164656e-02 + 2.744673664902718e-02 + 2.738132528259552e-02 + 2.731606912935157e-02 + 2.725096782054994e-02 + 2.718602099881922e-02 + 2.712122829441404e-02 + 2.705658933533107e-02 + 2.699210376654040e-02 + 2.692777122629619e-02 + 2.686359134766074e-02 + 2.679956376855050e-02 + 2.673568813034148e-02 + 2.667196407449100e-02 + 2.660839123653040e-02 + 2.654496925932325e-02 + 2.648169779209303e-02 + 2.641857647953893e-02 + 2.635560496230991e-02 + 2.629278288047702e-02 + 2.623010988365326e-02 + 2.616758562366618e-02 + 2.610520975031933e-02 + 2.604298190560967e-02 + 2.598090173551063e-02 + 2.591896889422694e-02 + 2.585718303968204e-02 + 2.579554382343284e-02 + 2.573405088890056e-02 + 2.567270388987125e-02 + 2.561150248485746e-02 + 2.555044633271165e-02 + 2.548953508798725e-02 + 2.542876840653832e-02 + 2.536814594843316e-02 + 2.530766736730480e-02 + 2.524733231969847e-02 + 2.518714047250964e-02 + 2.512709148651639e-02 + 2.506718502244662e-02 + 2.500742074738894e-02 + 2.494779831751132e-02 + 2.488831739145854e-02 + 2.482897764751662e-02 + 2.476977875068464e-02 + 2.471072035836960e-02 + 2.465180213557270e-02 + 2.459302375909131e-02 + 2.453438490494015e-02 + 2.447588522684439e-02 + 2.441752439632618e-02 + 2.435930209686822e-02 + 2.430121799563831e-02 + 2.424327176191257e-02 + 2.418546306971884e-02 + 2.412779159320392e-02 + 2.407025701015483e-02 + 2.401285899981317e-02 + 2.395559723318545e-02 + 2.389847138352894e-02 + 2.384148113010493e-02 + 2.378462615809818e-02 + 2.372790614977233e-02 + 2.367132078182184e-02 + 2.361486973012364e-02 + 2.355855267628842e-02 + 2.350236930912776e-02 + 2.344631931433045e-02 + 2.339040237260871e-02 + 2.333461816188979e-02 + 2.327896637661412e-02 + 2.322344670789084e-02 + 2.316805883183489e-02 + 2.311280243515378e-02 + 2.305767721100430e-02 + 2.300268285332459e-02 + 2.294781904779433e-02 + 2.289308548135576e-02 + 2.283848185163481e-02 + 2.278400785322520e-02 + 2.272966317657758e-02 + 2.267544750906289e-02 + 2.262136054576819e-02 + 2.256740198705395e-02 + 2.251357153295969e-02 + 2.245986887460843e-02 + 2.240629370372433e-02 + 2.235284572997816e-02 + 2.229952464835474e-02 + 2.224633014749699e-02 + 2.219326193897085e-02 + 2.214031972536749e-02 + 2.208750320104361e-02 + 2.203481207216736e-02 + 2.198224604054712e-02 + 2.192980480490446e-02 + 2.187748807809197e-02 + 2.182529556587840e-02 + 2.177322696546572e-02 + 2.172128198614426e-02 + 2.166946034002218e-02 + 2.161776173674180e-02 + 2.156618587797776e-02 + 2.151473246970182e-02 + 2.146340122648612e-02 + 2.141219186176581e-02 + 2.136110408595206e-02 + 2.131013760699968e-02 + 2.125929213929720e-02 + 2.120856739943827e-02 + 2.115796310282077e-02 + 2.110747896148574e-02 + 2.105711468773744e-02 + 2.100686999708853e-02 + 2.095674460640470e-02 + 2.090673823470377e-02 + 2.085685060390323e-02 + 2.080708143284909e-02 + 2.075743043865024e-02 + 2.070789733896461e-02 + 2.065848185338519e-02 + 2.060918370358626e-02 + 2.056000261352416e-02 + 2.051093830731156e-02 + 2.046199050841276e-02 + 2.041315893918832e-02 + 2.036444332670061e-02 + 2.031584339811113e-02 + 2.026735887003067e-02 + 2.021898947028043e-02 + 2.017073493461869e-02 + 2.012259498752183e-02 + 2.007456935661649e-02 + 2.002665777297167e-02 + 1.997885996051357e-02 + 1.993117564943755e-02 + 1.988360457741823e-02 + 1.983614647798445e-02 + 1.978880107994216e-02 + 1.974156811085027e-02 + 1.969444731166336e-02 + 1.964743841723802e-02 + 1.960054115227784e-02 + 1.955375525837354e-02 + 1.950708047707909e-02 + 1.946051654129042e-02 + 1.941406318497287e-02 + 1.936772014474174e-02 + 1.932148716119990e-02 + 1.927536398140690e-02 + 1.922935034564512e-02 + 1.918344598004621e-02 + 1.913765063001423e-02 + 1.909196404425976e-02 + 1.904638595925336e-02 + 1.900091612254435e-02 + 1.895555428061772e-02 + 1.891030016469744e-02 + 1.886515352637163e-02 + 1.882011411999479e-02 + 1.877518167521970e-02 + 1.873035594232305e-02 + 1.868563668110157e-02 + 1.864102363246424e-02 + 1.859651653826993e-02 + 1.855211514697215e-02 + 1.850781921692858e-02 + 1.846362849811174e-02 + 1.841954273425713e-02 + 1.837556167722399e-02 + 1.833168508274460e-02 + 1.828791270638760e-02 + 1.824424429587613e-02 + 1.820067960186819e-02 + 1.815721838079814e-02 + 1.811386039242822e-02 + 1.807060539155448e-02 + 1.802745312809132e-02 + 1.798440336027523e-02 + 1.794145584667328e-02 + 1.789861034396235e-02 + 1.785586661445831e-02 + 1.781322441713358e-02 + 1.777068350457842e-02 + 1.772824363606736e-02 + 1.768590457480606e-02 + 1.764366608520215e-02 + 1.760152792777148e-02 + 1.755948986314423e-02 + 1.751755165530868e-02 + 1.747571306959304e-02 + 1.743397386795286e-02 + 1.739233380686453e-02 + 1.735079266012027e-02 + 1.730935019969591e-02 + 1.726800617515195e-02 + 1.722676035861175e-02 + 1.718561252878033e-02 + 1.714456244423200e-02 + 1.710360987245781e-02 + 1.706275458478830e-02 + 1.702199634263978e-02 + 1.698133492251816e-02 + 1.694077010510486e-02 + 1.690030164730331e-02 + 1.685992932068962e-02 + 1.681965290804600e-02 + 1.677947217681714e-02 + 1.673938689521877e-02 + 1.669939683666028e-02 + 1.665950177991986e-02 + 1.661970150205939e-02 + 1.657999577711447e-02 + 1.654038437906895e-02 + 1.650086708478969e-02 + 1.646144367321789e-02 + 1.642211391798679e-02 + 1.638287759543946e-02 + 1.634373448754158e-02 + 1.630468437732129e-02 + 1.626572704186934e-02 + 1.622686225251448e-02 + 1.618808979718675e-02 + 1.614940946170872e-02 + 1.611082101996680e-02 + 1.607232425440962e-02 + 1.603391895041710e-02 + 1.599560489122978e-02 + 1.595738185903658e-02 + 1.591924963852954e-02 + 1.588120801872957e-02 + 1.584325677844977e-02 + 1.580539569909259e-02 + 1.576762457813194e-02 + 1.572994319938502e-02 + 1.569235134492561e-02 + 1.565484881311209e-02 + 1.561743538476679e-02 + 1.558011083839993e-02 + 1.554287497958433e-02 + 1.550572759639263e-02 + 1.546866846769255e-02 + 1.543169739218311e-02 + 1.539481416304957e-02 + 1.535801856691179e-02 + 1.532131039389481e-02 + 1.528468944222785e-02 + 1.524815551153465e-02 + 1.521170838270258e-02 + 1.517534784896265e-02 + 1.513907371629201e-02 + 1.510288577422132e-02 + 1.506678381600347e-02 + 1.503076764230081e-02 + 1.499483704594779e-02 + 1.495899182406707e-02 + 1.492323178010325e-02 + 1.488755670734373e-02 + 1.485196640180961e-02 + 1.481646066682016e-02 + 1.478103929846549e-02 + 1.474570209596410e-02 + 1.471044886603309e-02 + 1.467527940456368e-02 + 1.464019351058384e-02 + 1.460519099481427e-02 + 1.457027165550810e-02 + 1.453543528903122e-02 + 1.450068170024789e-02 + 1.446601069487559e-02 + 1.443142207965322e-02 + 1.439691566271563e-02 + 1.436249124410753e-02 + 1.432814862414793e-02 + 1.429388761459182e-02 + 1.425970802314729e-02 + 1.422560965483165e-02 + 1.419159231701311e-02 + 1.415765581387266e-02 + 1.412379995194592e-02 + 1.409002455116132e-02 + 1.405632941792580e-02 + 1.402271435207008e-02 + 1.398917917303820e-02 + 1.395572369124169e-02 + 1.392234770932106e-02 + 1.388905104392205e-02 + 1.385583350997071e-02 + 1.382269491788422e-02 + 1.378963507987753e-02 + 1.375665380791922e-02 + 1.372375091366015e-02 + 1.369092621121959e-02 + 1.365817951715469e-02 + 1.362551064978645e-02 + 1.359291942581090e-02 + 1.356040565686591e-02 + 1.352796915179558e-02 + 1.349560973812908e-02 + 1.346332723540702e-02 + 1.343112144483463e-02 + 1.339899219360082e-02 + 1.336693930891053e-02 + 1.333496260043193e-02 + 1.330306188734133e-02 + 1.327123699224855e-02 + 1.323948773476910e-02 + 1.320781393642416e-02 + 1.317621541705235e-02 + 1.314469199166208e-02 + 1.311324348463446e-02 + 1.308186972415766e-02 + 1.305057053367403e-02 + 1.301934573289238e-02 + 1.298819514247371e-02 + 1.295711859027024e-02 + 1.292611590142988e-02 + 1.289518689817567e-02 + 1.286433140253433e-02 + 1.283354924175516e-02 + 1.280284024626999e-02 + 1.277220424320462e-02 + 1.274164105512193e-02 + 1.271115050454398e-02 + 1.268073242580564e-02 + 1.265038664885281e-02 + 1.262011299771964e-02 + 1.258991130045622e-02 + 1.255978138928737e-02 + 1.252972309744999e-02 + 1.249973624798444e-02 + 1.246982067106488e-02 + 1.243997620568067e-02 + 1.241020267461013e-02 + 1.238049990730756e-02 + 1.235086774662455e-02 + 1.232130602051964e-02 + 1.229181455798258e-02 + 1.226239319685087e-02 + 1.223304176770951e-02 + 1.220376010347082e-02 + 1.217454804486660e-02 + 1.214540542341299e-02 + 1.211633207079968e-02 + 1.208732782681391e-02 + 1.205839252771904e-02 + 1.202952600673308e-02 + 1.200072809652295e-02 + 1.197199864009422e-02 + 1.194333748171596e-02 + 1.191474445503102e-02 + 1.188621939116683e-02 + 1.185776212675846e-02 + 1.182937251288597e-02 + 1.180105038815069e-02 + 1.177279558400601e-02 + 1.174460794107161e-02 + 1.171648730396891e-02 + 1.168843351609301e-02 + 1.166044641171630e-02 + 1.163252583370262e-02 + 1.160467163102319e-02 + 1.157688364263099e-02 + 1.154916170933044e-02 + 1.152150567570712e-02 + 1.149391538555893e-02 + 1.146639068095789e-02 + 1.143893140447911e-02 + 1.141153740781534e-02 + 1.138420853673168e-02 + 1.135694462854312e-02 + 1.132974552839056e-02 + 1.130261108533809e-02 + 1.127554114921167e-02 + 1.124853556496432e-02 + 1.122159417997002e-02 + 1.119471684585664e-02 + 1.116790340283271e-02 + 1.114115369710312e-02 + 1.111446758944324e-02 + 1.108784492177717e-02 + 1.106128553754357e-02 + 1.103478929728668e-02 + 1.100835604711145e-02 + 1.098198563223047e-02 + 1.095567791133747e-02 + 1.092943273475230e-02 + 1.090324994987072e-02 + 1.087712940981543e-02 + 1.085107096608238e-02 + 1.082507447053145e-02 + 1.079913977931768e-02 + 1.077326674497864e-02 + 1.074745521859533e-02 + 1.072170505538783e-02 + 1.069601611066200e-02 + 1.067038823886727e-02 + 1.064482129354728e-02 + 1.061931513219808e-02 + 1.059386961330933e-02 + 1.056848458681972e-02 + 1.054315990640127e-02 + 1.051789543133304e-02 + 1.049269102209236e-02 + 1.046754653920045e-02 + 1.044246184044743e-02 + 1.041743677288729e-02 + 1.039247119540387e-02 + 1.036756497988909e-02 + 1.034271797805630e-02 + 1.031793004647734e-02 + 1.029320105343422e-02 + 1.026853085073464e-02 + 1.024391929539164e-02 + 1.021936625852857e-02 + 1.019487159885800e-02 + 1.017043517132599e-02 + 1.014605683522427e-02 + 1.012173646293816e-02 + 1.009747392058842e-02 + 1.007326905588179e-02 + 1.004912173629082e-02 + 1.002503183346208e-02 + 1.000099920653610e-02 + 9.977023717087764e-03 + 9.953105229386335e-03 + 9.929243608872406e-03 + 9.905438720973011e-03 + 9.881690430827046e-03 + 9.857998603258316e-03 + 9.834363104786413e-03 + 9.810783802355094e-03 + 9.787260560990621e-03 + 9.763793247942843e-03 + 9.740381731591760e-03 + 9.717025877967826e-03 + 9.693725554510663e-03 + 9.670480630152000e-03 + 9.647290973567844e-03 + 9.624156453317901e-03 + 9.601076938062841e-03 + 9.578052297039983e-03 + 9.555082399653962e-03 + 9.532167115690399e-03 + 9.509306316445141e-03 + 9.486499872512266e-03 + 9.463747653553711e-03 + 9.441049530896879e-03 + 9.418405376662433e-03 + 9.395815063104617e-03 + 9.373278461427175e-03 + 9.350795443499088e-03 + 9.328365882617906e-03 + 9.305989651970981e-03 + 9.283666624824176e-03 + 9.261396674723778e-03 + 9.239179675408710e-03 + 9.217015500757821e-03 + 9.194904024947497e-03 + 9.172845123632957e-03 + 9.150838672277813e-03 + 9.128884544858834e-03 + 9.106982617538152e-03 + 9.085132766987650e-03 + 9.063334868396023e-03 + 9.041588798714301e-03 + 9.019894435479832e-03 + 8.998251654894885e-03 + 8.976660334271958e-03 + 8.955120351755344e-03 + 8.933631585369030e-03 + 8.912193913264732e-03 + 8.890807213885918e-03 + 8.869471366214972e-03 + 8.848186249259440e-03 + 8.826951742251321e-03 + 8.805767725383001e-03 + 8.784634078744540e-03 + 8.763550682317645e-03 + 8.742517416743529e-03 + 8.721534162832835e-03 + 8.700600801502468e-03 + 8.679717214101214e-03 + 8.658883282492529e-03 + 8.638098888885700e-03 + 8.617363915025602e-03 + 8.596678243443276e-03 + 8.576041757693640e-03 + 8.555454340593486e-03 + 8.534915875110577e-03 + 8.514426244844865e-03 + 8.493985333674233e-03 + 8.473593025909724e-03 + 8.453249206335058e-03 + 8.432953759374154e-03 + 8.412706569850796e-03 + 8.392507523503252e-03 + 8.372356505624580e-03 + 8.352253401838657e-03 + 8.332198098800467e-03 + 8.312190482450988e-03 + 8.292230438871008e-03 + 8.272317855399050e-03 + 8.252452619663047e-03 + 8.232634619050869e-03 + 8.212863740331295e-03 + 8.193139871755427e-03 + 8.173462902270600e-03 + 8.153832719956143e-03 + 8.134249213438031e-03 + 8.114712271802299e-03 + 8.095221784035302e-03 + 8.075777639824947e-03 + 8.056379729417138e-03 + 8.037027942938956e-03 + 8.017722170154111e-03 + 7.998462301028743e-03 + 7.979248227381455e-03 + 7.960079840401237e-03 + 7.940957030626293e-03 + 7.921879690329333e-03 + 7.902847711453821e-03 + 7.883860985421759e-03 + 7.864919405368889e-03 + 7.846022864037674e-03 + 7.827171253341911e-03 + 7.808364466777525e-03 + 7.789602397948123e-03 + 7.770884940051978e-03 + 7.752211987488857e-03 + 7.733583434567862e-03 + 7.714999174963997e-03 + 7.696459103766686e-03 + 7.677963116338556e-03 + 7.659511107448910e-03 + 7.641102971817726e-03 + 7.622738605046243e-03 + 7.604417904226960e-03 + 7.586140765207791e-03 + 7.567907083543369e-03 + 7.549716755813157e-03 + 7.531569679511053e-03 + 7.513465752109027e-03 + 7.495404870064185e-03 + 7.477386931039745e-03 + 7.459411833357052e-03 + 7.441479474804043e-03 + 7.423589753808636e-03 + 7.405742569108545e-03 + 7.387937818890154e-03 + 7.370175402616828e-03 + 7.352455220325825e-03 + 7.334777170378787e-03 + 7.317141152312192e-03 + 7.299547066866589e-03 + 7.281994814230038e-03 + 7.264484294760120e-03 + 7.247015409129196e-03 + 7.229588058070396e-03 + 7.212202142721502e-03 + 7.194857564639825e-03 + 7.177554225359851e-03 + 7.160292026920549e-03 + 7.143070871828701e-03 + 7.125890661781023e-03 + 7.108751299151369e-03 + 7.091652687463649e-03 + 7.074594729473165e-03 + 7.057577328092346e-03 + 7.040600386920140e-03 + 7.023663809440293e-03 + 7.006767499621119e-03 + 6.989911362283671e-03 + 6.973095301761098e-03 + 6.956319222246642e-03 + 6.939583028242014e-03 + 6.922886624933184e-03 + 6.906229917977133e-03 + 6.889612813237578e-03 + 6.873035215956658e-03 + 6.856497031627250e-03 + 6.839998167124653e-03 + 6.823538528849983e-03 + 6.807118022950776e-03 + 6.790736556102037e-03 + 6.774394036012678e-03 + 6.758090370503938e-03 + 6.741825465874382e-03 + 6.725599229831710e-03 + 6.709411571076730e-03 + 6.693262397262002e-03 + 6.677151616823099e-03 + 6.661079138883049e-03 + 6.645044871881905e-03 + 6.629048724424302e-03 + 6.613090605727208e-03 + 6.597170426169362e-03 + 6.581288095176429e-03 + 6.565443521393368e-03 + 6.549636616015259e-03 + 6.533867289690503e-03 + 6.518135451870410e-03 + 6.502441013826720e-03 + 6.486783886693610e-03 + 6.471163980906190e-03 + 6.455581208648475e-03 + 6.440035481784445e-03 + 6.424526710952946e-03 + 6.409054808194274e-03 + 6.393619686148648e-03 + 6.378221257389647e-03 + 6.362859434216829e-03 + 6.347534129405275e-03 + 6.332245256702231e-03 + 6.316992728926398e-03 + 6.301776458848883e-03 + 6.286596360249587e-03 + 6.271452347303485e-03 + 6.256344334335297e-03 + 6.241272235637497e-03 + 6.226235965289892e-03 + 6.211235437617355e-03 + 6.196270567881593e-03 + 6.181341271239764e-03 + 6.166447462747857e-03 + 6.151589057705613e-03 + 6.136765971896623e-03 + 6.121978121401951e-03 + 6.107225422158738e-03 + 6.092507790132517e-03 + 6.077825141615326e-03 + 6.063177393866219e-03 + 6.048564463915550e-03 + 6.033986268442907e-03 + 6.019442724444331e-03 + 6.004933749322879e-03 + 5.990459260846786e-03 + 5.976019176895193e-03 + 5.961613415583682e-03 + 5.947241895231883e-03 + 5.932904533953020e-03 + 5.918601250164555e-03 + 5.904331962789744e-03 + 5.890096590861083e-03 + 5.875895053571317e-03 + 5.861727270328322e-03 + 5.847593160694864e-03 + 5.833492644473149e-03 + 5.819425641715960e-03 + 5.805392072163253e-03 + 5.791391855962741e-03 + 5.777424914193254e-03 + 5.763491167389796e-03 + 5.749590536018355e-03 + 5.735722941127108e-03 + 5.721888304505554e-03 + 5.708086547766434e-03 + 5.694317591457392e-03 + 5.680581357894303e-03 + 5.666877769868901e-03 + 5.653206748498332e-03 + 5.639568216215936e-03 + 5.625962096156960e-03 + 5.612388310444156e-03 + 5.598846782244085e-03 + 5.585337435180374e-03 + 5.571860191464977e-03 + 5.558414974887664e-03 + 5.545001710146245e-03 + 5.531620319651646e-03 + 5.518270727342278e-03 + 5.504952858629092e-03 + 5.491666637149790e-03 + 5.478411987013346e-03 + 5.465188833301630e-03 + 5.451997101098349e-03 + 5.438836715460566e-03 + 5.425707601454353e-03 + 5.412609684284744e-03 + 5.399542889597323e-03 + 5.386507143481766e-03 + 5.373502371672819e-03 + 5.360528500347250e-03 + 5.347585456320476e-03 + 5.334673165238440e-03 + 5.321791553290573e-03 + 5.308940548233661e-03 + 5.296120077396591e-03 + 5.283330067535706e-03 + 5.270570445063373e-03 + 5.257841138170363e-03 + 5.245142075198895e-03 + 5.232473183108559e-03 + 5.219834389656078e-03 + 5.207225623450342e-03 + 5.194646813662877e-03 + 5.182097888019903e-03 + 5.169578774433769e-03 + 5.157089403379224e-03 + 5.144629703265777e-03 + 5.132199601938705e-03 + 5.119799030212023e-03 + 5.107427917986021e-03 + 5.095086194230444e-03 + 5.082773788422456e-03 + 5.070490630851877e-03 + 5.058236652191947e-03 + 5.046011782411770e-03 + 5.033815952050060e-03 + 5.021649092096963e-03 + 5.009511132708167e-03 + 4.997402004872952e-03 + 4.985321640554899e-03 + 4.973269971256597e-03 + 4.961246927915243e-03 + 4.949252441362114e-03 + 4.937286444284467e-03 + 4.925348869112448e-03 + 4.913439647407981e-03 + 4.901558711796497e-03 + 4.889705994683389e-03 + 4.877881427891544e-03 + 4.866084944974813e-03 + 4.854316479151838e-03 + 4.842575962313763e-03 + 4.830863328342665e-03 + 4.819178511323663e-03 + 4.807521444209243e-03 + 4.795892060902922e-03 + 4.784290295384500e-03 + 4.772716080911616e-03 + 4.761169351999607e-03 + 4.749650043687660e-03 + 4.738158090505544e-03 + 4.726693426328077e-03 + 4.715255985490762e-03 + 4.703845704177584e-03 + 4.692462517255530e-03 + 4.681106359266913e-03 + 4.669777166671382e-03 + 4.658474874679097e-03 + 4.647199418051047e-03 + 4.635950733693824e-03 + 4.624728757814554e-03 + 4.613533426035275e-03 + 4.602364675085555e-03 + 4.591222441193854e-03 + 4.580106660340010e-03 + 4.569017270133631e-03 + 4.557954207729684e-03 + 4.546917409606468e-03 + 4.535906813063030e-03 + 4.524922355662980e-03 + 4.513963975024341e-03 + 4.503031608847450e-03 + 4.492125194604706e-03 + 4.481244669721014e-03 + 4.470389973075004e-03 + 4.459561043403103e-03 + 4.448757818627942e-03 + 4.437980237157132e-03 + 4.427228237584914e-03 + 4.416501758569245e-03 + 4.405800739475310e-03 + 4.395125119625431e-03 + 4.384474837838549e-03 + 4.373849833628650e-03 + 4.363250046735974e-03 + 4.352675416676511e-03 + 4.342125883231542e-03 + 4.331601386396421e-03 + 4.321101866314548e-03 + 4.310627263408189e-03 + 4.300177518182486e-03 + 4.289752570999768e-03 + 4.279352362569397e-03 + 4.268976833692708e-03 + 4.258625924837488e-03 + 4.248299577436886e-03 + 4.237997733292803e-03 + 4.227720333115902e-03 + 4.217467318344088e-03 + 4.207238631130676e-03 + 4.197034213451204e-03 + 4.186854006892898e-03 + 4.176697952964437e-03 + 4.166565994071401e-03 + 4.156458073034852e-03 + 4.146374132669977e-03 + 4.136314114934178e-03 + 4.126277962231265e-03 + 4.116265617785051e-03 + 4.106277025130100e-03 + 4.096312127225507e-03 + 4.086370866498099e-03 + 4.076453186851077e-03 + 4.066559032318193e-03 + 4.056688346384167e-03 + 4.046841072375552e-03 + 4.037017154234894e-03 + 4.027216536753351e-03 + 4.017439163691562e-03 + 4.007684978984143e-03 + 3.997953927616382e-03 + 3.988245954341580e-03 + 3.978561003853811e-03 + 3.968899020987378e-03 + 3.959259950174110e-03 + 3.949643736363211e-03 + 3.940050325917084e-03 + 3.930479663940112e-03 + 3.920931695287086e-03 + 3.911406366153988e-03 + 3.901903622192779e-03 + 3.892423408886956e-03 + 3.882965672529350e-03 + 3.873530359308857e-03 + 3.864117415340676e-03 + 3.854726787040829e-03 + 3.845358421069152e-03 + 3.836012264203419e-03 + 3.826688263119290e-03 + 3.817386364639579e-03 + 3.808106515834866e-03 + 3.798848664119608e-03 + 3.789612756624806e-03 + 3.780398740246358e-03 + 3.771206562411011e-03 + 3.762036171140922e-03 + 3.752887514765301e-03 + 3.743760540740341e-03 + 3.734655196719529e-03 + 3.725571430980986e-03 + 3.716509192135922e-03 + 3.707468428387338e-03 + 3.698449087559120e-03 + 3.689451119263604e-03 + 3.680474472533827e-03 + 3.671519094850921e-03 + 3.662584935894309e-03 + 3.653671945297336e-03 + 3.644780071212910e-03 + 3.635909263417586e-03 + 3.627059471971036e-03 + 3.618230645951014e-03 + 3.609422734605362e-03 + 3.600635687799489e-03 + 3.591869456282116e-03 + 3.583123989955615e-03 + 3.574399238584414e-03 + 3.565695152845392e-03 + 3.557011682930304e-03 + 3.548348778903569e-03 + 3.539706391532652e-03 + 3.531084471778191e-03 + 3.522482970666699e-03 + 3.513901839239252e-03 + 3.505341028388093e-03 + 3.496800489111114e-03 + 3.488280173119797e-03 + 3.479780031892506e-03 + 3.471300016604386e-03 + 3.462840078578430e-03 + 3.454400170006921e-03 + 3.445980243580072e-03 + 3.437580250715228e-03 + 3.429200142992534e-03 + 3.420839872683964e-03 + 3.412499392748279e-03 + 3.404178655562702e-03 + 3.395877612913789e-03 + 3.387596218304602e-03 + 3.379334424898988e-03 + 3.371092184764359e-03 + 3.362869450661228e-03 + 3.354666175991870e-03 + 3.346482314543412e-03 + 3.338317819175880e-03 + 3.330172642972068e-03 + 3.322046740007612e-03 + 3.313940064235304e-03 + 3.305852569386344e-03 + 3.297784208976364e-03 + 3.289734936574093e-03 + 3.281704706272533e-03 + 3.273693473070924e-03 + 3.265701191257636e-03 + 3.257727814803984e-03 + 3.249773298074863e-03 + 3.241837595979994e-03 + 3.233920663596386e-03 + 3.226022455622204e-03 + 3.218142926555998e-03 + 3.210282031163023e-03 + 3.202439725215123e-03 + 3.194615964134223e-03 + 3.186810703067027e-03 + 3.179023897602838e-03 + 3.171255503398923e-03 + 3.163505475998486e-03 + 3.155773770652277e-03 + 3.148060343553685e-03 + 3.140365151430291e-03 + 3.132688149313277e-03 + 3.125029293443853e-03 + 3.117388541383308e-03 + 3.109765848736372e-03 + 3.102161171408776e-03 + 3.094574466373093e-03 + 3.087005690728782e-03 + 3.079454801280003e-03 + 3.071921754555924e-03 + 3.064406507880629e-03 + 3.056909018530570e-03 + 3.049429243338136e-03 + 3.041967139485151e-03 + 3.034522664377966e-03 + 3.027095775562892e-03 + 3.019686430816400e-03 + 3.012294588055786e-03 + 3.004920205212209e-03 + 2.997563239705405e-03 + 2.990223649173174e-03 + 2.982901392198357e-03 + 2.975596427311395e-03 + 2.968308712822513e-03 + 2.961038206776656e-03 + 2.953784867392937e-03 + 2.946548653201694e-03 + 2.939329523194637e-03 + 2.932127436582907e-03 + 2.924942352269057e-03 + 2.917774228121468e-03 + 2.910623023643702e-03 + 2.903488699056754e-03 + 2.896371212514376e-03 + 2.889270523304112e-03 + 2.882186591679052e-03 + 2.875119376658924e-03 + 2.868068837676741e-03 + 2.861034934776638e-03 + 2.854017627809473e-03 + 2.847016876513213e-03 + 2.840032640686938e-03 + 2.833064880681456e-03 + 2.826113557103426e-03 + 2.819178630449384e-03 + 2.812260060105674e-03 + 2.805357806481411e-03 + 2.798471831428214e-03 + 2.791602094948384e-03 + 2.784748557109701e-03 + 2.777911179108391e-03 + 2.771089922400815e-03 + 2.764284748007964e-03 + 2.757495616304825e-03 + 2.750722488782177e-03 + 2.743965327115693e-03 + 2.737224092409578e-03 + 2.730498746008289e-03 + 2.723789249648372e-03 + 2.717095565426717e-03 + 2.710417654358784e-03 + 2.703755477800617e-03 + 2.697108999073383e-03 + 2.690478179939920e-03 + 2.683862981660020e-03 + 2.677263366970269e-03 + 2.670679298051182e-03 + 2.664110736894467e-03 + 2.657557646411518e-03 + 2.651019989061974e-03 + 2.644497727126834e-03 + 2.637990823748049e-03 + 2.631499241776077e-03 + 2.625022943767310e-03 + 2.618561892619121e-03 + 2.612116051581085e-03 + 2.605685383984310e-03 + 2.599269852507916e-03 + 2.592869420565223e-03 + 2.586484052278204e-03 + 2.580113710814501e-03 + 2.573758359383719e-03 + 2.567417961609419e-03 + 2.561092481362380e-03 + 2.554781882413309e-03 + 2.548486128464411e-03 + 2.542205184036424e-03 + 2.535939013422343e-03 + 2.529687580238901e-03 + 2.523450848436006e-03 + 2.517228782352248e-03 + 2.511021346720505e-03 + 2.504828506444104e-03 + 2.498650225958320e-03 + 2.492486468967693e-03 + 2.486337200680032e-03 + 2.480202386472821e-03 + 2.474081990653096e-03 + 2.467975978460781e-03 + 2.461884315215007e-03 + 2.455806965243642e-03 + 2.449743893777428e-03 + 2.443695066608286e-03 + 2.437660449328877e-03 + 2.431640007119714e-03 + 2.425633705211946e-03 + 2.419641509634817e-03 + 2.413663385966202e-03 + 2.407699299632079e-03 + 2.401749216909934e-03 + 2.395813103777745e-03 + 2.389890925926915e-03 + 2.383982649399740e-03 + 2.378088240561300e-03 + 2.372207665865560e-03 + 2.366340891258303e-03 + 2.360487883037754e-03 + 2.354648608018378e-03 + 2.348823033118129e-03 + 2.343011124539103e-03 + 2.337212848054228e-03 + 2.331428171390765e-03 + 2.325657062040943e-03 + 2.319899486412222e-03 + 2.314155410784699e-03 + 2.308424802407836e-03 + 2.302707629600105e-03 + 2.297003858807791e-03 + 2.291313456815245e-03 + 2.285636391934661e-03 + 2.279972631361444e-03 + 2.274322142294874e-03 + 2.268684892803898e-03 + 2.263060850445441e-03 + 2.257449982675230e-03 + 2.251852257331842e-03 + 2.246267642469826e-03 + 2.240696106224612e-03 + 2.235137616698782e-03 + 2.229592141925271e-03 + 2.224059650030012e-03 + 2.218540109447705e-03 + 2.213033488566449e-03 + 2.207539755750870e-03 + 2.202058879448589e-03 + 2.196590827868759e-03 + 2.191135569406817e-03 + 2.185693073538250e-03 + 2.180263309295804e-03 + 2.174846245146934e-03 + 2.169441849486111e-03 + 2.164050091629877e-03 + 2.158670941358884e-03 + 2.153304367034930e-03 + 2.147950337590323e-03 + 2.142608822828710e-03 + 2.137279792340324e-03 + 2.131963215445541e-03 + 2.126659061301170e-03 + 2.121367299363035e-03 + 2.116087899310018e-03 + 2.110820830985273e-03 + 2.105566064252215e-03 + 2.100323568822542e-03 + 2.095093314356113e-03 + 2.089875271479949e-03 + 2.084669410319118e-03 + 2.079475699908646e-03 + 2.074294110902819e-03 + 2.069124614088778e-03 + 2.063967179294363e-03 + 2.058821776747180e-03 + 2.053688376991423e-03 + 2.048566950718572e-03 + 2.043457468582260e-03 + 2.038359901240009e-03 + 2.033274219398007e-03 + 2.028200393563254e-03 + 2.023138394473247e-03 + 2.018088193625467e-03 + 2.013049761808302e-03 + 2.008023069533293e-03 + 2.003008088023720e-03 + 1.998004788922133e-03 + 1.993013143739461e-03 + 1.988033123025423e-03 + 1.983064698401174e-03 + 1.978107841939004e-03 + 1.973162523959716e-03 + 1.968228716337500e-03 + 1.963306391971906e-03 + 1.958395521239848e-03 + 1.953496075620802e-03 + 1.948608028019931e-03 + 1.943731350228688e-03 + 1.938866013789267e-03 + 1.934011990485791e-03 + 1.929169252901841e-03 + 1.924337773155020e-03 + 1.919517522737516e-03 + 1.914708474346116e-03 + 1.909910600724645e-03 + 1.905123874118434e-03 + 1.900348266781407e-03 + 1.895583751238830e-03 + 1.890830300301618e-03 + 1.886087886140893e-03 + 1.881356481302734e-03 + 1.876636059328014e-03 + 1.871926593048255e-03 + 1.867228054886164e-03 + 1.862540417322874e-03 + 1.857863653843670e-03 + 1.853197738035545e-03 + 1.848542642643349e-03 + 1.843898340659710e-03 + 1.839264805482605e-03 + 1.834642010884087e-03 + 1.830029929675155e-03 + 1.825428534756660e-03 + 1.820837800826725e-03 + 1.816257700994342e-03 + 1.811688207883416e-03 + 1.807129296515363e-03 + 1.802580940478343e-03 + 1.798043112569654e-03 + 1.793515787775185e-03 + 1.788998939681262e-03 + 1.784492541016137e-03 + 1.779996567290557e-03 + 1.775510992946437e-03 + 1.771035790955467e-03 + 1.766570935188881e-03 + 1.762116400368900e-03 + 1.757672161650980e-03 + 1.753238193179788e-03 + 1.748814468721301e-03 + 1.744400962234815e-03 + 1.739997649147064e-03 + 1.735604504535336e-03 + 1.731221502449809e-03 + 1.726848617398671e-03 + 1.722485824273258e-03 + 1.718133098140551e-03 + 1.713790413411143e-03 + 1.709457744960287e-03 + 1.705135068710917e-03 + 1.700822359316501e-03 + 1.696519591326764e-03 + 1.692226740206531e-03 + 1.687943780998710e-03 + 1.683670689031842e-03 + 1.679407440586236e-03 + 1.675154010255336e-03 + 1.670910372590760e-03 + 1.666676504276520e-03 + 1.662452380680008e-03 + 1.658237976786625e-03 + 1.654033269138933e-03 + 1.649838233189823e-03 + 1.645652843894769e-03 + 1.641477077483747e-03 + 1.637310910181093e-03 + 1.633154317979306e-03 + 1.629007276710043e-03 + 1.624869761977525e-03 + 1.620741749539434e-03 + 1.616623216289868e-03 + 1.612514138626602e-03 + 1.608414492386588e-03 + 1.604324254066237e-03 + 1.600243399893093e-03 + 1.596171905775712e-03 + 1.592109748332882e-03 + 1.588056904380057e-03 + 1.584013350591343e-03 + 1.579979063015354e-03 + 1.575954018417919e-03 + 1.571938194458784e-03 + 1.567931567017785e-03 + 1.563934112240304e-03 + 1.559945807641685e-03 + 1.555966630779868e-03 + 1.551996558434039e-03 + 1.548035566441649e-03 + 1.544083632647983e-03 + 1.540140734902817e-03 + 1.536206849423688e-03 + 1.532281953055403e-03 + 1.528366023373284e-03 + 1.524459038476067e-03 + 1.520560975283225e-03 + 1.516671810636623e-03 + 1.512791522697568e-03 + 1.508920088807869e-03 + 1.505057486015793e-03 + 1.501203692209861e-03 + 1.497358685322736e-03 + 1.493522443062933e-03 + 1.489694942724099e-03 + 1.485876162194853e-03 + 1.482066079697174e-03 + 1.478264672890618e-03 + 1.474471919644568e-03 + 1.470687798121700e-03 + 1.466912286519041e-03 + 1.463145362594138e-03 + 1.459387003990816e-03 + 1.455637189563730e-03 + 1.451895897787254e-03 + 1.448163106500405e-03 + 1.444438793947980e-03 + 1.440722938760434e-03 + 1.437015519703072e-03 + 1.433316514691718e-03 + 1.429625901910583e-03 + 1.425943660193772e-03 + 1.422269768088744e-03 + 1.418604204318803e-03 + 1.414946947952337e-03 + 1.411297977567125e-03 + 1.407657271824714e-03 + 1.404024809794961e-03 + 1.400400570139807e-03 + 1.396784531434119e-03 + 1.393176672533590e-03 + 1.389576972970703e-03 + 1.385985412235932e-03 + 1.382401969107074e-03 + 1.378826622443783e-03 + 1.375259351219489e-03 + 1.371700134514839e-03 + 1.368148952079336e-03 + 1.364605783757036e-03 + 1.361070608632290e-03 + 1.357543405835987e-03 + 1.354024154621594e-03 + 1.350512834317403e-03 + 1.347009424934256e-03 + 1.343513906652181e-03 + 1.340026258589547e-03 + 1.336546459739890e-03 + 1.333074489680093e-03 + 1.329610329681042e-03 + 1.326153959375566e-03 + 1.322705357081586e-03 + 1.319264503327096e-03 + 1.315831378649060e-03 + 1.312405962942001e-03 + 1.308988235813027e-03 + 1.305578177302038e-03 + 1.302175767936687e-03 + 1.298780987783750e-03 + 1.295393816813521e-03 + 1.292014235092867e-03 + 1.288642222828844e-03 + 1.285277760581296e-03 + 1.281920829182064e-03 + 1.278571408463430e-03 + 1.275229478684235e-03 + 1.271895021211065e-03 + 1.268568016022571e-03 + 1.265248443161927e-03 + 1.261936283866222e-03 + 1.258631518851812e-03 + 1.255334128626978e-03 + 1.252044093890807e-03 + 1.248761395151665e-03 + 1.245486013113075e-03 + 1.242217929129610e-03 + 1.238957124307338e-03 + 1.235703579418827e-03 + 1.232457274959363e-03 + 1.229218191838193e-03 + 1.225986311430220e-03 + 1.222761615473843e-03 + 1.219544084856006e-03 + 1.216333699988417e-03 + 1.213130441964228e-03 + 1.209934292565758e-03 + 1.206745233676102e-03 + 1.203563245963205e-03 + 1.200388310525061e-03 + 1.197220409053557e-03 + 1.194059523052293e-03 + 1.190905633902454e-03 + 1.187758723052301e-03 + 1.184618772531461e-03 + 1.181485763720917e-03 + 1.178359677471923e-03 + 1.175240496495629e-03 + 1.172128202790389e-03 + 1.169022777016151e-03 + 1.165924201427151e-03 + 1.162832457974330e-03 + 1.159747527667428e-03 + 1.156669393342115e-03 + 1.153598037513975e-03 + 1.150533441195656e-03 + 1.147475586343000e-03 + 1.144424455192698e-03 + 1.141380029693999e-03 + 1.138342292070108e-03 + 1.135311224784675e-03 + 1.132286810371483e-03 + 1.129269030382949e-03 + 1.126257866687504e-03 + 1.123253302806332e-03 + 1.120255320585528e-03 + 1.117263901594027e-03 + 1.114279029333705e-03 + 1.111300685926414e-03 + 1.108328853214485e-03 + 1.105363515126287e-03 + 1.102404653695880e-03 + 1.099452250276090e-03 + 1.096506289065756e-03 + 1.093566752790314e-03 + 1.090633622975802e-03 + 1.087706882683815e-03 + 1.084786514992603e-03 + 1.081872502714538e-03 + 1.078964828755093e-03 + 1.076063476054719e-03 + 1.073168427526746e-03 + 1.070279665938792e-03 + 1.067397174442816e-03 + 1.064520936417689e-03 + 1.061650934087830e-03 + 1.058787150577005e-03 + 1.055929570192933e-03 + 1.053078175257412e-03 + 1.050232948596978e-03 + 1.047393874491941e-03 + 1.044560935952646e-03 + 1.041734115621369e-03 + 1.038913396581877e-03 + 1.036098763274602e-03 + 1.033290199394467e-03 + 1.030487686748966e-03 + 1.027691209748195e-03 + 1.024900752824401e-03 + 1.022116298137994e-03 + 1.019337829638554e-03 + 1.016565331581354e-03 + 1.013798786681050e-03 + 1.011038178744547e-03 + 1.008283492069452e-03 + 1.005534710312413e-03 + 1.002791817167788e-03 + 1.000054796401326e-03 + 9.973236317525795e-04 + 9.945983071579347e-04 + 9.918788066535795e-04 + 9.891651141399331e-04 + 9.864572135795289e-04 + 9.837550890187182e-04 + 9.810587245440681e-04 + 9.783681042165289e-04 + 9.756832121036981e-04 + 9.730040324093254e-04 + 9.703305493295432e-04 + 9.676627470403841e-04 + 9.650006098016114e-04 + 9.623441219373007e-04 + 9.596932677964791e-04 + 9.570480316324495e-04 + 9.544083978205768e-04 + 9.517743508934125e-04 + 9.491458752396962e-04 + 9.465229553012878e-04 + 9.439055756501852e-04 + 9.412937207895038e-04 + 9.386873752809525e-04 + 9.360865238076440e-04 + 9.334911509791107e-04 + 9.309012414140627e-04 + 9.283167798171848e-04 + 9.257377509839119e-04 + 9.231641396993918e-04 + 9.205959306774516e-04 + 9.180331088080401e-04 + 9.154756590138127e-04 + 9.129235660962607e-04 + 9.103768149701878e-04 + 9.078353906321772e-04 + 9.052992780955343e-04 + 9.027684623946166e-04 + 9.002429285800444e-04 + 8.977226617159664e-04 + 8.952076469091944e-04 + 8.926978693026045e-04 + 8.901933140640126e-04 + 8.876939664445818e-04 + 8.851998117353022e-04 + 8.827108351548252e-04 + 8.802270219825557e-04 + 8.777483575705740e-04 + 8.752748272927118e-04 + 8.728064165680979e-04 + 8.703431108522175e-04 + 8.678848955854838e-04 + 8.654317562117358e-04 + 8.629836782258149e-04 + 8.605406473168484e-04 + 8.581026490766350e-04 + 8.556696689660546e-04 + 8.532416927295966e-04 + 8.508187061075220e-04 + 8.484006947265324e-04 + 8.459876443379812e-04 + 8.435795407180884e-04 + 8.411763696276702e-04 + 8.387781169670006e-04 + 8.363847686436173e-04 + 8.339963104952486e-04 + 8.316127284703911e-04 + 8.292340085393665e-04 + 8.268601366269382e-04 + 8.244910987818749e-04 + 8.221268810991789e-04 + 8.197674696351961e-04 + 8.174128504795555e-04 + 8.150630097863373e-04 + 8.127179337987174e-04 + 8.103776086785577e-04 + 8.080420206116735e-04 + 8.057111559859151e-04 + 8.033850010442538e-04 + 8.010635420033148e-04 + 7.987467653572807e-04 + 7.964346575211490e-04 + 7.941272048473775e-04 + 7.918243938145987e-04 + 7.895262109139402e-04 + 7.872326426395918e-04 + 7.849436755404454e-04 + 7.826592961986925e-04 + 7.803794912312032e-04 + 7.781042473133076e-04 + 7.758335511109105e-04 + 7.735673892815593e-04 + 7.713057485643111e-04 + 7.690486157536814e-04 + 7.667959776727836e-04 + 7.645478211004869e-04 + 7.623041328912646e-04 + 7.600649000053854e-04 + 7.578301093264215e-04 + 7.555997477583347e-04 + 7.533738022910566e-04 + 7.511522599897612e-04 + 7.489351079282785e-04 + 7.467223331578882e-04 + 7.445139227962291e-04 + 7.423098639775247e-04 + 7.401101438263767e-04 + 7.379147496119391e-04 + 7.357236686066337e-04 + 7.335368879604144e-04 + 7.313543950177981e-04 + 7.291761771981462e-04 + 7.270022218267318e-04 + 7.248325162821272e-04 + 7.226670479949025e-04 + 7.205058044241288e-04 + 7.183487730730944e-04 + 7.161959414766152e-04 + 7.140472971809699e-04 + 7.119028277561113e-04 + 7.097625208231147e-04 + 7.076263641063239e-04 + 7.054943452576004e-04 + 7.033664519218169e-04 + 7.012426719956601e-04 + 6.991229932510109e-04 + 6.970074033540157e-04 + 6.948958902400331e-04 + 6.927884418689321e-04 + 6.906850461485944e-04 + 6.885856909689247e-04 + 6.864903643067344e-04 + 6.843990542306164e-04 + 6.823117487354689e-04 + 6.802284359113009e-04 + 6.781491039842488e-04 + 6.760737410664455e-04 + 6.740023352773578e-04 + 6.719348748282754e-04 + 6.698713480434448e-04 + 6.678117432465053e-04 + 6.657560487034911e-04 + 6.637042527655789e-04 + 6.616563438432147e-04 + 6.596123103782944e-04 + 6.575721408025738e-04 + 6.555358236048545e-04 + 6.535033473936704e-04 + 6.514747007198682e-04 + 6.494498721244171e-04 + 6.474288502266283e-04 + 6.454116237722280e-04 + 6.433981815196293e-04 + 6.413885120963034e-04 + 6.393826043225120e-04 + 6.373804471016686e-04 + 6.353820291728463e-04 + 6.333873394523912e-04 + 6.313963669651093e-04 + 6.294091005879305e-04 + 6.274255292885601e-04 + 6.254456421348147e-04 + 6.234694281911996e-04 + 6.214968766042923e-04 + 6.195279765738893e-04 + 6.175627172048796e-04 + 6.156010877068991e-04 + 6.136430774102346e-04 + 6.116886755862946e-04 + 6.097378715435904e-04 + 6.077906546697107e-04 + 6.058470144286753e-04 + 6.039069402835228e-04 + 6.019704216802507e-04 + 6.000374481409778e-04 + 5.981080092530985e-04 + 5.961820946588096e-04 + 5.942596940083503e-04 + 5.923407969721301e-04 + 5.904253932616448e-04 + 5.885134726736782e-04 + 5.866050250202793e-04 + 5.847000400980857e-04 + 5.827985078791194e-04 + 5.809004183470366e-04 + 5.790057613579966e-04 + 5.771145269606196e-04 + 5.752267052829598e-04 + 5.733422863841403e-04 + 5.714612604000550e-04 + 5.695836175305099e-04 + 5.677093480037018e-04 + 5.658384420678564e-04 + 5.639708900266142e-04 + 5.621066823022896e-04 + 5.602458092869138e-04 + 5.583882613744714e-04 + 5.565340290827777e-04 + 5.546831029664671e-04 + 5.528354736013446e-04 + 5.509911316015094e-04 + 5.491500676402182e-04 + 5.473122724461832e-04 + 5.454777367810608e-04 + 5.436464514412940e-04 + 5.418184072613321e-04 + 5.399935951226492e-04 + 5.381720060069473e-04 + 5.363536309736959e-04 + 5.345384609776805e-04 + 5.327264870731968e-04 + 5.309177004752346e-04 + 5.291120923593077e-04 + 5.273096539494221e-04 + 5.255103765544823e-04 + 5.237142514407170e-04 + 5.219212699733492e-04 + 5.201314236865307e-04 + 5.183447040146093e-04 + 5.165611024246747e-04 + 5.147806105331874e-04 + 5.130032200114061e-04 + 5.112289225530487e-04 + 5.094577098620987e-04 + 5.076895736854020e-04 + 5.059245058583464e-04 + 5.041624983517820e-04 + 5.024035430839573e-04 + 5.006476319990773e-04 + 4.988947572035529e-04 + 4.971449107942239e-04 + 4.953980848818115e-04 + 4.936542716805855e-04 + 4.919134634956966e-04 + 4.901756526846954e-04 + 4.884408316006080e-04 + 4.867089926403434e-04 + 4.849801282731475e-04 + 4.832542310892493e-04 + 4.815312937309208e-04 + 4.798113088560132e-04 + 4.780942690954599e-04 + 4.763801672259719e-04 + 4.746689961731605e-04 + 4.729607488395965e-04 + 4.712554181333483e-04 + 4.695529970142063e-04 + 4.678534786182833e-04 + 4.661568561056742e-04 + 4.644631226216267e-04 + 4.627722714285577e-04 + 4.610842958399921e-04 + 4.593991892076899e-04 + 4.577169450300335e-04 + 4.560375568336446e-04 + 4.543610181249096e-04 + 4.526873225594637e-04 + 4.510164638467393e-04 + 4.493484356950898e-04 + 4.476832319569833e-04 + 4.460208465382815e-04 + 4.443612733350173e-04 + 4.427045064285867e-04 + 4.410505399510805e-04 + 4.393993679504339e-04 + 4.377509846774214e-04 + 4.361053844834223e-04 + 4.344625616662297e-04 + 4.328225106365193e-04 + 4.311852259097922e-04 + 4.295507020701359e-04 + 4.279189337521271e-04 + 4.262899156467061e-04 + 4.246636425301464e-04 + 4.230401092565931e-04 + 4.214193107478168e-04 + 4.198012419839599e-04 + 4.181858979868060e-04 + 4.165732738692198e-04 + 4.149633649793705e-04 + 4.133561665739344e-04 + 4.117516738453971e-04 + 4.101498823326724e-04 + 4.085507876181731e-04 + 4.069543852399522e-04 + 4.053606707914893e-04 + 4.037696400086752e-04 + 4.021812887746215e-04 + 4.005956129436248e-04 + 3.990126084760660e-04 + 3.974322714846641e-04 + 3.958545980293214e-04 + 3.942795842513983e-04 + 3.927072264688036e-04 + 3.911375211195356e-04 + 3.895704646516968e-04 + 3.880060534869329e-04 + 3.864442842922655e-04 + 3.848851538066537e-04 + 3.833286587063809e-04 + 3.817747958651508e-04 + 3.802235622593628e-04 + 3.786749548688228e-04 + 3.771289708205180e-04 + 3.755856073392733e-04 + 3.740448616816441e-04 + 3.725067311649692e-04 + 3.709712132043737e-04 + 3.694383053790672e-04 + 3.679080053579325e-04 + 3.663803108503057e-04 + 3.648552195606523e-04 + 3.633327293646475e-04 + 3.618128382932607e-04 + 3.602955444344858e-04 + 3.587808459078391e-04 + 3.572687408998931e-04 + 3.557592277792442e-04 + 3.542523049927096e-04 + 3.527479710408145e-04 + 3.512462245019904e-04 + 3.497470640500036e-04 + 3.482504884770882e-04 + 3.467564967437580e-04 + 3.452650877985240e-04 + 3.437762605764859e-04 + 3.422900143250168e-04 + 3.408063483640538e-04 + 3.393252619896345e-04 + 3.378467546103840e-04 + 3.363708257653936e-04 + 3.348974751341996e-04 + 3.334267024660963e-04 + 3.319585075646510e-04 + 3.304928903038992e-04 + 3.290298507212495e-04 + 3.275693889604049e-04 + 3.261115052402690e-04 + 3.246561999078441e-04 + 3.232034733603981e-04 + 3.217533259984899e-04 + 3.203057585048498e-04 + 3.188607716794152e-04 + 3.174183662346357e-04 + 3.159785430822741e-04 + 3.145413032833238e-04 + 3.131066479369465e-04 + 3.116745782136557e-04 + 3.102450954058189e-04 + 3.088182010197684e-04 + 3.073938965650610e-04 + 3.059721835802734e-04 + 3.045530637916020e-04 + 3.031365390386899e-04 + 3.017226112552691e-04 + 3.003112824793533e-04 + 2.989025548234045e-04 + 2.974964304955006e-04 + 2.960929118913988e-04 + 2.946920014422941e-04 + 2.932937016150965e-04 + 2.918980151023185e-04 + 2.905049446767149e-04 + 2.891144931550728e-04 + 2.877266634923031e-04 + 2.863414587568181e-04 + 2.849588821248430e-04 + 2.835789368717610e-04 + 2.822016263526234e-04 + 2.808269540128114e-04 + 2.794549234525530e-04 + 2.780855383858511e-04 + 2.767188026160350e-04 + 2.753547199733251e-04 + 2.739932944362576e-04 + 2.726345302253411e-04 + 2.712784314751994e-04 + 2.699250024118205e-04 + 2.685742475595017e-04 + 2.672261714415124e-04 + 2.658807786371782e-04 + 2.645380739000219e-04 + 2.631980619957597e-04 + 2.618607478073471e-04 + 2.605261365239288e-04 + 2.591942332540738e-04 + 2.578650431168888e-04 + 2.565385715181067e-04 + 2.552148239188000e-04 + 2.538938058269625e-04 + 2.525755228940477e-04 + 2.512599808919393e-04 + 2.499471856899045e-04 + 2.486371432244064e-04 + 2.473298595107682e-04 + 2.460253406646765e-04 + 2.447235929605613e-04 + 2.434246227773826e-04 + 2.421284365723855e-04 + 2.408350408491505e-04 + 2.395444422583646e-04 + 2.382566475978551e-04 + 2.369716636126905e-04 + 2.356894972066169e-04 + 2.344101555067014e-04 + 2.331336455922197e-04 + 2.318599746393059e-04 + 2.305891500038487e-04 + 2.293211790903458e-04 + 2.280560693749351e-04 + 2.267938284406659e-04 + 2.255344639796373e-04 + 2.242779837481179e-04 + 2.230243955584571e-04 + 2.217737074088738e-04 + 2.205259273750273e-04 + 2.192810635194828e-04 + 2.180391240377155e-04 + 2.168001172473318e-04 + 2.155640515683406e-04 + 2.143309354202995e-04 + 2.131007772856537e-04 + 2.118735858306198e-04 + 2.106493698084787e-04 + 2.094281380089542e-04 + 2.082098992208740e-04 + 2.069946624015073e-04 + 2.057824366286796e-04 + 2.045732309577213e-04 + 2.033670545521807e-04 + 2.021639166762670e-04 + 2.009638266139778e-04 + 1.997667937615743e-04 + 1.985728276099416e-04 + 1.973819376249420e-04 + 1.961941334186818e-04 + 1.950094247288422e-04 + 1.938278211445808e-04 + 1.926493324216112e-04 + 1.914739685443153e-04 + 1.903017394244089e-04 + 1.891326549660665e-04 + 1.879667251315557e-04 + 1.868039600691931e-04 + 1.856443699312781e-04 + 1.844879648170397e-04 + 1.833347550402995e-04 + 1.821847509305592e-04 + 1.810379627269367e-04 + 1.798944008330572e-04 + 1.787540757117569e-04 + 1.776169978023503e-04 + 1.764831776083264e-04 + 1.753526257075039e-04 + 1.742253527510645e-04 + 1.731013693254543e-04 + 1.719806860311682e-04 + 1.708633135911149e-04 + 1.697492627671219e-04 + 1.686385443331759e-04 + 1.675311690584120e-04 + 1.664271477052953e-04 + 1.653264910845736e-04 + 1.642292101438652e-04 + 1.631353157524277e-04 + 1.620448187436031e-04 + 1.609577300754615e-04 + 1.598740606888992e-04 + 1.587938214812250e-04 + 1.577170233258566e-04 + 1.566436772168694e-04 + 1.555737942159133e-04 + 1.545073851968179e-04 + 1.534444610998845e-04 + 1.523850329588589e-04 + 1.513291117056277e-04 + 1.502767082718022e-04 + 1.492278336091406e-04 + 1.481824986061289e-04 + 1.471407142013266e-04 + 1.461024913960903e-04 + 1.450678410475480e-04 + 1.440367739871709e-04 + 1.430093010797408e-04 + 1.419854331666308e-04 + 1.409651810501559e-04 + 1.399485554884026e-04 + 1.389355672434459e-04 + 1.379262270311977e-04 + 1.369205454852166e-04 + 1.359185332657421e-04 + 1.349202010239166e-04 + 1.339255593445051e-04 + 1.329346186642912e-04 + 1.319473893926901e-04 + 1.309638820305585e-04 + 1.299841069283569e-04 + 1.290080743627276e-04 + 1.280357946577360e-04 + 1.270672779908064e-04 + 1.261025344295187e-04 + 1.251415740258344e-04 + 1.241844068414831e-04 + 1.232310428585276e-04 + 1.222814917974582e-04 + 1.213357634573326e-04 + 1.203938676644921e-04 + 1.194558139769228e-04 + 1.185216119071049e-04 + 1.175912709437735e-04 + 1.166648004395201e-04 + 1.157422096820540e-04 + 1.148235078941047e-04 + 1.139087041565015e-04 + 1.129978074560730e-04 + 1.120908266963163e-04 + 1.111877706801771e-04 + 1.102886481165339e-04 + 1.093934676008822e-04 + 1.085022375478793e-04 + 1.076149663238430e-04 + 1.067316622735248e-04 + 1.058523335072715e-04 + 1.049769880021785e-04 + 1.041056336506420e-04 + 1.032382782134568e-04 + 1.023749293560659e-04 + 1.015155946454073e-04 + 1.006602814254268e-04 + 9.980899691604087e-05 + 9.896174827822380e-05 + 9.811854249648769e-05 + 9.727938640362722e-05 + 9.644428669167931e-05 + 9.561324990949941e-05 + 9.478628247235907e-05 + 9.396339064407664e-05 + 9.314458052650697e-05 + 9.232985807344924e-05 + 9.151922908266137e-05 + 9.071269919463392e-05 + 8.991027389908417e-05 + 8.911195851702552e-05 + 8.831775819982511e-05 + 8.752767793866873e-05 + 8.674172256023528e-05 + 8.595989672073575e-05 + 8.518220490590348e-05 + 8.440865141920385e-05 + 8.363924039733809e-05 + 8.287397580722622e-05 + 8.211286142104977e-05 + 8.135590084051729e-05 + 8.060309749193613e-05 + 7.985445459658365e-05 + 7.910997520302950e-05 + 7.836966218298747e-05 + 7.763351820320998e-05 + 7.690154574559998e-05 + 7.617374709914362e-05 + 7.545012436145964e-05 + 7.473067944430294e-05 + 7.401541405207780e-05 + 7.330432968888297e-05 + 7.259742767376570e-05 + 7.189470911942744e-05 + 7.119617492698978e-05 + 7.050182580949816e-05 + 6.981166227552030e-05 + 6.912568461899398e-05 + 6.844389294104750e-05 + 6.776628713300132e-05 + 6.709286687130013e-05 + 6.642363163388969e-05 + 6.575858067871575e-05 + 6.509771305493562e-05 + 6.444102761503343e-05 + 6.378852298738557e-05 + 6.314019758583242e-05 + 6.249604962295056e-05 + 6.185607708979005e-05 + 6.122027775903875e-05 + 6.058864919749181e-05 + 5.996118875684596e-05 + 5.933789356597989e-05 + 5.871876053888907e-05 + 5.810378637807827e-05 + 5.749296757228591e-05 + 5.688630039241354e-05 + 5.628378087841946e-05 + 5.568540486517338e-05 + 5.509116797959273e-05 + 5.450106561933422e-05 + 5.391509296529475e-05 + 5.333324497665832e-05 + 5.275551640753726e-05 + 5.218190179953343e-05 + 5.161239545977182e-05 + 5.104699148146081e-05 + 5.048568375913861e-05 + 4.992846596763329e-05 + 4.937533154817934e-05 + 4.882627374147808e-05 + 4.828128557792042e-05 + 4.774035986717692e-05 + 4.720348921279551e-05 + 4.667066599018748e-05 + 4.614188237688400e-05 + 4.561713035194944e-05 + 4.509640165438987e-05 + 4.457968783032909e-05 + 4.406698023190481e-05 + 4.355826996891768e-05 + 4.305354796420506e-05 + 4.255280495560636e-05 + 4.205603144616347e-05 + 4.156321774032957e-05 + 4.107435395613752e-05 + 4.058942999665175e-05 + 4.010843556869507e-05 + 3.963136019085871e-05 + 3.915819316952545e-05 + 3.868892361719641e-05 + 3.822354046226270e-05 + 3.776203244031561e-05 + 3.730438809005640e-05 + 3.685059575689182e-05 + 3.640064360230198e-05 + 3.595451960373741e-05 + 3.551221155134442e-05 + 3.507370705450421e-05 + 3.463899354241650e-05 + 3.420805825588243e-05 + 3.378088825986720e-05 + 3.335747045272196e-05 + 3.293779155463552e-05 + 3.252183811026268e-05 + 3.210959649877579e-05 + 3.170105292073684e-05 + 3.129619341367949e-05 + 3.089500386055728e-05 + 3.049746997239612e-05 + 3.010357730585059e-05 + 2.971331126487755e-05 + 2.932665707619367e-05 + 2.894359982344077e-05 + 2.856412445132613e-05 + 2.818821573877814e-05 + 2.781585831987375e-05 + 2.744703669454755e-05 + 2.708173520948728e-05 + 2.671993807124836e-05 + 2.636162935767626e-05 + 2.600679300301263e-05 + 2.565541280257304e-05 + 2.530747242805460e-05 + 2.496295542635910e-05 + 2.462184521206604e-05 + 2.428412507785163e-05 + 2.394977819771233e-05 + 2.361878762181457e-05 + 2.329113628448998e-05 + 2.296680701293541e-05 + 2.264578251757474e-05 + 2.232804539557172e-05 + 2.201357814530585e-05 + 2.170236315904943e-05 + 2.139438272306276e-05 + 2.108961903561866e-05 + 2.078805419745760e-05 + 2.048967020582218e-05 + 2.019444897050880e-05 + 1.990237231614296e-05 + 1.961342197777572e-05 + 1.932757961462368e-05 + 1.904482680353188e-05 + 1.876514502948172e-05 + 1.848851571515060e-05 + 1.821492021640795e-05 + 1.794433980831370e-05 + 1.767675569461099e-05 + 1.741214902258527e-05 + 1.715050088166273e-05 + 1.689179228451716e-05 + 1.663600419203502e-05 + 1.638311752659296e-05 + 1.613311314360082e-05 + 1.588597184706167e-05 + 1.564167440067926e-05 + 1.540020151829502e-05 + 1.516153387453479e-05 + 1.492565211244357e-05 + 1.469253682958783e-05 + 1.446216858599219e-05 + 1.423452791421707e-05 + 1.400959531801727e-05 + 1.378735127470914e-05 + 1.356777623611474e-05 + 1.335085063337062e-05 + 1.313655487876277e-05 + 1.292486936494610e-05 + 1.271577446639333e-05 + 1.250925054517824e-05 + 1.230527796605363e-05 + 1.210383708165599e-05 + 1.190490823144681e-05 + 1.170847175649016e-05 + 1.151450799717159e-05 + 1.132299729345123e-05 + 1.113391999104486e-05 + 1.094725644586371e-05 + 1.076298702423590e-05 + 1.058109209097755e-05 + 1.040155202830912e-05 + 1.022434724511555e-05 + 1.004945815519835e-05 + 9.876865191434875e-06 + 9.706548817576976e-06 + 9.538489515829050e-06 + 9.372667791395497e-06 + 9.209064181113714e-06 + 9.047659251528417e-06 + 8.888433598428113e-06 + 8.731367852817854e-06 + 8.576442682615395e-06 + 8.423638791088146e-06 + 8.272936922653283e-06 + 8.124317863654203e-06 + 7.977762440936675e-06 + 7.833251528895281e-06 + 7.690766049713759e-06 + 7.550286971453687e-06 + 7.411795314787993e-06 + 7.275272152381047e-06 + 7.140698608042407e-06 + 7.008055865069554e-06 + 6.877325162999093e-06 + 6.748487797069182e-06 + 6.621525126924672e-06 + 6.496418573295794e-06 + 6.373149617974847e-06 + 6.251699810845691e-06 + 6.132050767139283e-06 + 6.014184168887862e-06 + 5.898081770597224e-06 + 5.783725394973654e-06 + 5.671096936050662e-06 + 5.560178365641361e-06 + 5.450951726966341e-06 + 5.343399138917566e-06 + 5.237502802602254e-06 + 5.133244993336976e-06 + 5.030608065950253e-06 + 4.929574461382145e-06 + 4.830126698086031e-06 + 4.732247378331277e-06 + 4.635919192753656e-06 + 4.541124912789582e-06 + 4.447847398958468e-06 + 4.356069602328335e-06 + 4.265774557056776e-06 + 4.176945389932598e-06 + 4.089565320281935e-06 + 4.003617653697645e-06 + 3.919085793399028e-06 + 3.835953236362346e-06 + 3.754203567594348e-06 + 3.673820472678765e-06 + 3.594787733829004e-06 + 3.517089224074456e-06 + 3.440708918621393e-06 + 3.365630891050476e-06 + 3.291839308866544e-06 + 3.219318444550688e-06 + 3.148052669614368e-06 + 3.078026451876596e-06 + 3.009224366838337e-06 + 2.941631088840265e-06 + 2.875231390541817e-06 + 2.810010156134118e-06 + 2.745952369256776e-06 + 2.683043113446972e-06 + 2.621267584374269e-06 + 2.560611078236376e-06 + 2.501058994098514e-06 + 2.442596843868032e-06 + 2.385210240033010e-06 + 2.328884899906223e-06 + 2.273606655048426e-06 + 2.219361438179688e-06 + 2.166135288678882e-06 + 2.113914360085187e-06 + 2.062684907096663e-06 + 2.012433292648363e-06 + 1.963145994708785e-06 + 1.914809593193956e-06 + 1.867410777738679e-06 + 1.820936351687259e-06 + 1.775373220609589e-06 + 1.730708401732302e-06 + 1.686929024947410e-06 + 1.644022323067757e-06 + 1.601975642837270e-06 + 1.560776441656341e-06 + 1.520412279298016e-06 + 1.480870830315665e-06 + 1.442139880698395e-06 + 1.404207319625165e-06 + 1.367061149777226e-06 + 1.330689483155573e-06 + 1.295080535963615e-06 + 1.260222640873472e-06 + 1.226104237141460e-06 + 1.192713867093549e-06 + 1.160040189500492e-06 + 1.128071969424555e-06 + 1.096798076008267e-06 + 1.066207493417804e-06 + 1.036289310659725e-06 + 1.007032721493895e-06 + 9.784270338592895e-07 + 9.504616588874799e-07 + 9.231261128777845e-07 + 8.964100256766087e-07 + 8.703031286825232e-07 + 8.447952585150839e-07 + 8.198763641874936e-07 + 7.955364948286083e-07 + 7.717658049346632e-07 + 7.485545598914059e-07 + 7.258931238786546e-07 + 7.037719665410901e-07 + 6.821816664988581e-07 + 6.611128998791421e-07 + 6.405564483055948e-07 + 6.205032003278225e-07 + 6.009441408121614e-07 + 5.818703600152187e-07 + 5.632730528626383e-07 + 5.451435094097432e-07 + 5.274731247677268e-07 + 5.102533965000488e-07 + 4.934759162480692e-07 + 4.771323800991179e-07 + 4.612145841662162e-07 + 4.457144176881476e-07 + 4.306238737382497e-07 + 4.159350428538716e-07 + 4.016401077051372e-07 + 3.877313539402903e-07 + 3.742011622647174e-07 + 3.610420046499709e-07 + 3.482464547968944e-07 + 3.358071790150475e-07 + 3.237169340794661e-07 + 3.119685770955318e-07 + 3.005550554375413e-07 + 2.894694062090586e-07 + 2.787047650882795e-07 + 2.682543557659438e-07 + 2.581114909968296e-07 + 2.482695801165890e-07 + 2.387221182727572e-07 + 2.294626890008751e-07 + 2.204849703818711e-07 + 2.117827241548630e-07 + 2.033497996067267e-07 + 1.951801384624956e-07 + 1.872677641420447e-07 + 1.796067867093452e-07 + 1.721914062527642e-07 + 1.650159027896018e-07 + 1.580746421942365e-07 + 1.513620776690125e-07 + 1.448727406655565e-07 + 1.386012477684331e-07 + 1.325423000867771e-07 + 1.266906752868051e-07 + 1.210412352836648e-07 + 1.155889239131470e-07 + 1.103287600176973e-07 + 1.052558455547364e-07 + 1.003653617115032e-07 + 9.565256321109978e-08 + 9.111278671292873e-08 + 8.674144556415159e-08 + 8.253402527034383e-08 + 7.848609177536523e-08 + 7.459328520671134e-08 + 7.085131661573615e-08 + 6.725597593242166e-08 + 6.380312478868793e-08 + 6.048869454868003e-08 + 5.730869376312298e-08 + 5.425920030057914e-08 + 5.133636064443935e-08 + 4.853639660958214e-08 + 4.585559704969457e-08 + 4.329031836991616e-08 + 4.083699031045398e-08 + 3.849210751108346e-08 + 3.625223113812888e-08 + 3.411399355038575e-08 + 3.207408998824199e-08 + 3.012928121511996e-08 + 2.827639696078167e-08 + 2.651232793166280e-08 + 2.483402934338396e-08 + 2.323852311474514e-08 + 2.172289034680298e-08 + 2.028427559044146e-08 + 1.891988778877689e-08 + 1.762699339728036e-08 + 1.640292125402531e-08 + 1.524506224991720e-08 + 1.415086320949184e-08 + 1.311783222446413e-08 + 1.214353712508316e-08 + 1.122560019257377e-08 + 1.036170378687137e-08 + 9.549587722603005e-09 + 8.787044877843281e-09 + 8.071926961605364e-09 + 7.402140903161946e-09 + 6.775645388472587e-09 + 6.190456604129221e-09 + 5.644643792323680e-09 + 5.136326726146161e-09 + 4.663681262833891e-09 + 4.224934232596794e-09 + 3.818361845837701e-09 + 3.442294891302550e-09 + 3.095113141246544e-09 + 2.775244665814773e-09 + 2.481170514027403e-09 + 2.211418834557044e-09 + 1.964565051498559e-09 + 1.739235878927056e-09 + 1.534103359980556e-09 + 1.347885831336700e-09 + 1.179351163642234e-09 + 1.027310894648576e-09 + 8.906218977827446e-10 + 7.681887799049897e-10 + 6.589582548472836e-10 + 5.619214238374888e-10 + 4.761152985830568e-10 + 4.006175343477256e-10 + 3.345492227248809e-10 + 2.770755362128679e-10 + 2.274009179016229e-10 + 1.847722840116674e-10 + 1.484788129161787e-10 + 1.178476648704315e-10 + 9.224748788184573e-11 + 7.108739902161543e-11 + 5.381328706422649e-11 + 3.991151299277479e-11 + 2.890715266694438e-11 + 2.036091648096713e-11 + 1.387293445657948e-11 + 9.080348320515963e-12 + 5.654863833225597e-12 + 3.306507674167889e-12 + 1.780689492983021e-12 + 8.563646514161430e-13 + 3.496037276049943e-13 + 1.104943293034196e-13 + 2.161027650041889e-14 + 5.028049336448720e-16 + 0.000000000000000e+00 + 0.000000000000000e+00 + 7.485272593961791e+04 + 1.493519318697679e+05 + 2.234970831294996e+05 + 2.972876533345640e+05 + 3.707231243774131e+05 + 4.438029864272003e+05 + 5.165267379297798e+05 + 5.888938856077072e+05 + 6.609039444602393e+05 + 7.325564377633340e+05 + 8.038508970696502e+05 + 8.747868622085482e+05 + 9.453638812860892e+05 + 1.015581510685036e+06 + 1.085439315064852e+06 + 1.154936867361703e+06 + 1.224073748788454e+06 + 1.292849548834672e+06 + 1.361263865266626e+06 + 1.429316304127287e+06 + 1.497006479736322e+06 + 1.564334014690106e+06 + 1.631298539861710e+06 + 1.697899694400910e+06 + 1.764137125734180e+06 + 1.830010489564698e+06 + 1.895519449872339e+06 + 1.960663678913685e+06 + 2.025442857222013e+06 + 2.089856673607306e+06 + 2.153904825156247e+06 + 2.217587017232217e+06 + 2.280902963475303e+06 + 2.343852385802290e+06 + 2.406435014406665e+06 + 2.468650587758616e+06 + 2.530498852605031e+06 + 2.591979563969503e+06 + 2.653092485152322e+06 + 2.713837387730482e+06 + 2.774214051557675e+06 + 2.834222264764299e+06 + 2.893861823757447e+06 + 2.953132533220919e+06 + 3.012034206115212e+06 + 3.070566663677527e+06 + 3.128729735421764e+06 + 3.186523259138526e+06 + 3.243947080895115e+06 + 3.301001055035538e+06 + 3.357685044180497e+06 + 3.413998919227402e+06 + 3.469942559350358e+06 + 3.525515852000176e+06 + 3.580718692904367e+06 + 3.635550986067140e+06 + 3.690012643769410e+06 + 3.744103586568788e+06 + 3.797823743299592e+06 + 3.851173051072835e+06 + 3.904151455276238e+06 + 3.956758909574216e+06 + 4.008995375907890e+06 + 4.060860824495079e+06 + 4.112355233830309e+06 + 4.163478590684799e+06 + 4.214230890106475e+06 + 4.264612135419963e+06 + 4.314622338226588e+06 + 4.364261518404378e+06 + 4.413529704108060e+06 + 4.462426931769069e+06 + 4.510953246095533e+06 + 4.559108700072286e+06 + 4.606893354960858e+06 + 4.654307280299488e+06 + 4.701350553903108e+06 + 4.748023261863358e+06 + 4.794325498548575e+06 + 4.840257366603798e+06 + 4.885818976950768e+06 + 4.931010448787929e+06 + 4.975831909590418e+06 + 5.020283495110085e+06 + 5.064365349375471e+06 + 5.108077624691823e+06 + 5.151420481641092e+06 + 5.194394089081923e+06 + 5.236998624149668e+06 + 5.279234272256376e+06 + 5.321101227090800e+06 + 5.362599690618395e+06 + 5.403729873081312e+06 + 5.444491992998408e+06 + 5.484886277165242e+06 + 5.524912960654069e+06 + 5.564572286813851e+06 + 5.603864507270245e+06 + 5.642789881925615e+06 + 5.681348678959022e+06 + 5.719541174826232e+06 + 5.757367654259708e+06 + 5.794828410268617e+06 + 5.831923744138825e+06 + 5.868653965432902e+06 + 5.905019391990117e+06 + 5.941020349926441e+06 + 5.976657173634544e+06 + 6.011930205783804e+06 + 6.046839797320290e+06 + 6.081386307466779e+06 + 6.115570103722748e+06 + 6.149391561864376e+06 + 6.182851065944540e+06 + 6.215949008292822e+06 + 6.248685789515501e+06 + 6.281061818495561e+06 + 6.313077512392686e+06 + 6.344733296643257e+06 + 6.376029604960365e+06 + 6.406966879333793e+06 + 6.437545570030033e+06 + 6.467766135592271e+06 + 6.497629042840399e+06 + 6.527134766871008e+06 + 6.556283791057392e+06 + 6.585076607049545e+06 + 6.613513714774162e+06 + 6.641595622434638e+06 + 6.669322846511072e+06 + 6.696695911760264e+06 + 6.723715351215711e+06 + 6.750381706187615e+06 + 6.776695526262878e+06 + 6.802657369305105e+06 + 6.828267801454599e+06 + 6.853527397128366e+06 + 6.878436739020114e+06 + 6.902996418100249e+06 + 6.927207033615881e+06 + 6.951069193090819e+06 + 6.974583512325577e+06 + 6.997750615397365e+06 + 7.020571134660101e+06 + 7.043045710744397e+06 + 7.065174992557568e+06 + 7.086959637283633e+06 + 7.108400310383311e+06 + 7.129497685594021e+06 + 7.150252444929884e+06 + 7.170665278681723e+06 + 7.190736885417059e+06 + 7.210467971980117e+06 + 7.229859253491821e+06 + 7.248911453349800e+06 + 7.267625303228384e+06 + 7.286001543078595e+06 + 7.304040921128170e+06 + 7.321744193881537e+06 + 7.339112126119829e+06 + 7.356145490900878e+06 + 7.372845069559221e+06 + 7.389211651706094e+06 + 7.405246035229432e+06 + 7.420949026293878e+06 + 7.436321439340766e+06 + 7.451364097088138e+06 + 7.466077830530737e+06 + 7.480463478940005e+06 + 7.494521889864086e+06 + 7.508253919127826e+06 + 7.521660430832772e+06 + 7.534742297357169e+06 + 7.547500399355968e+06 + 7.559935625760819e+06 + 7.572048873780073e+06 + 7.583841048898782e+06 + 7.595313064878696e+06 + 7.606465843758276e+06 + 7.617300315852673e+06 + 7.627817419753745e+06 + 7.638018102330050e+06 + 7.647903318726848e+06 + 7.657474032366097e+06 + 7.666731214946462e+06 + 7.675675846443305e+06 + 7.684308915108687e+06 + 7.692631417471377e+06 + 7.700644358336839e+06 + 7.708348750787240e+06 + 7.715745616181449e+06 + 7.722835984155037e+06 + 7.729620892620270e+06 + 7.736101387766127e+06 + 7.742278524058277e+06 + 7.748153364239094e+06 + 7.753726979327655e+06 + 7.759000448619737e+06 + 7.763974859687817e+06 + 7.768651308381077e+06 + 7.773030898825390e+06 + 7.777114743423343e+06 + 7.780903962854218e+06 + 7.784399686073998e+06 + 7.787603050315368e+06 + 7.790515201087713e+06 + 7.793137292177119e+06 + 7.795470485646380e+06 + 7.797515951834979e+06 + 7.799274869359110e+06 + 7.800748425111664e+06 + 7.801937814262233e+06 + 7.802844240257111e+06 + 7.803468914819297e+06 + 7.803813057948484e+06 + 7.803877897921070e+06 + 7.803664671290156e+06 + 7.803174622885538e+06 + 7.802409005813722e+06 + 7.801369081457905e+06 + 7.800056119477995e+06 + 7.798471397810592e+06 + 7.796616202669007e+06 + 7.794491828543244e+06 + 7.792099578200010e+06 + 7.789440762682715e+06 + 7.786516701311473e+06 + 7.783328721683091e+06 + 7.779878159671083e+06 + 7.776166359425665e+06 + 7.772194673373749e+06 + 7.767964462218953e+06 + 7.763477094941595e+06 + 7.758733948798692e+06 + 7.753736409323964e+06 + 7.748485870327833e+06 + 7.742983733897421e+06 + 7.737231410396548e+06 + 7.731230318465743e+06 + 7.724981885022229e+06 + 7.718487545259933e+06 + 7.711748742649483e+06 + 7.704766928938209e+06 + 7.697543564150140e+06 + 7.690080116586007e+06 + 7.682378062823243e+06 + 7.674438887715982e+06 + 7.666264084395060e+06 + 7.657855154268012e+06 + 7.649213607019073e+06 + 7.640340960609185e+06 + 7.631238741275986e+06 + 7.621908483533815e+06 + 7.612351730173716e+06 + 7.602570032263434e+06 + 7.592564949147409e+06 + 7.582338048446788e+06 + 7.571890906059416e+06 + 7.561225106159844e+06 + 7.550342241199316e+06 + 7.539243911905785e+06 + 7.527931727283904e+06 + 7.516407304615024e+06 + 7.504672269457198e+06 + 7.492728241314434e+06 + 7.480576638153253e+06 + 7.468218723741564e+06 + 7.455655760425863e+06 + 7.442889013018662e+06 + 7.429919748798485e+06 + 7.416749237509867e+06 + 7.403378751363355e+06 + 7.389809565035504e+06 + 7.376042955668893e+06 + 7.362080202872100e+06 + 7.347922588719724e+06 + 7.333571397752370e+06 + 7.319027916976659e+06 + 7.304293435865226e+06 + 7.289369246356713e+06 + 7.274256642855777e+06 + 7.258956922233087e+06 + 7.243471383825325e+06 + 7.227801329435183e+06 + 7.211948063331366e+06 + 7.195912892248592e+06 + 7.179697125387593e+06 + 7.163302074415106e+06 + 7.146729053463890e+06 + 7.129979379132709e+06 + 7.113054370486340e+06 + 7.095955349055575e+06 + 7.078683638837219e+06 + 7.061240566294081e+06 + 7.043627460354994e+06 + 7.025845652414794e+06 + 7.007896476334333e+06 + 6.989781268440476e+06 + 6.971501367526096e+06 + 6.953058114850082e+06 + 6.934452854137335e+06 + 6.915686931578767e+06 + 6.896761695831302e+06 + 6.877678498017876e+06 + 6.858438691727439e+06 + 6.839043633014950e+06 + 6.819494680401385e+06 + 6.799793194873727e+06 + 6.779940539884974e+06 + 6.759938081354136e+06 + 6.739787187666233e+06 + 6.719489229672301e+06 + 6.699045580689385e+06 + 6.678457616500544e+06 + 6.657726715354848e+06 + 6.636854257967380e+06 + 6.615841627519235e+06 + 6.594690209657516e+06 + 6.573401392495350e+06 + 6.551976566611862e+06 + 6.530417125052197e+06 + 6.508724463327511e+06 + 6.486899979414972e+06 + 6.464945073757762e+06 + 6.442861149265066e+06 + 6.420649611312096e+06 + 6.398311867740063e+06 + 6.375849328856199e+06 + 6.353263407433745e+06 + 6.330555518711952e+06 + 6.307727080396090e+06 + 6.284779512657427e+06 + 6.261714238133261e+06 + 6.238532681926889e+06 + 6.215236271607627e+06 + 6.191826437210800e+06 + 6.168304611237747e+06 + 6.144672228655818e+06 + 6.120930726898375e+06 + 6.097081545864792e+06 + 6.073126127920459e+06 + 6.049065917896771e+06 + 6.024902363091142e+06 + 6.000636913266994e+06 + 5.976271020653763e+06 + 5.951806139946896e+06 + 5.927243728307854e+06 + 5.902585245364108e+06 + 5.877832153209144e+06 + 5.852985916402456e+06 + 5.828048001969552e+06 + 5.803019879401957e+06 + 5.777903020657199e+06 + 5.752698900158828e+06 + 5.727408994796395e+06 + 5.702034783925475e+06 + 5.676577749367647e+06 + 5.651039375410506e+06 + 5.625421148807654e+06 + 5.599724558778713e+06 + 5.573951097009313e+06 + 5.548102257651095e+06 + 5.522179537321717e+06 + 5.496184435104840e+06 + 5.470118452550147e+06 + 5.443983093673326e+06 + 5.417779864956085e+06 + 5.391510275346133e+06 + 5.365175836257203e+06 + 5.338778061569033e+06 + 5.312318467627374e+06 + 5.285798573243992e+06 + 5.259219899696662e+06 + 5.232583970729172e+06 + 5.205892312551324e+06 + 5.179146453838928e+06 + 5.152347925733812e+06 + 5.125498261843814e+06 + 5.098598998242779e+06 + 5.071651673470572e+06 + 5.044657828533064e+06 + 5.017619006902143e+06 + 4.990536754515706e+06 + 4.963412619777663e+06 + 4.936248153557934e+06 + 4.909044909192459e+06 + 4.881804442483181e+06 + 4.854528311698059e+06 + 4.827218077571064e+06 + 4.799875303302180e+06 + 4.772501554557402e+06 + 4.745098399468737e+06 + 4.717667408634203e+06 + 4.690210155117837e+06 + 4.662728214449679e+06 + 4.635223164625784e+06 + 4.607696586108224e+06 + 4.580150061825077e+06 + 4.552585177170437e+06 + 4.525003520004408e+06 + 4.497406680653105e+06 + 4.469796251908661e+06 + 4.442173829029216e+06 + 4.414541009738924e+06 + 4.386899394227949e+06 + 4.359250585152469e+06 + 4.331596187634676e+06 + 4.303937809262771e+06 + 4.276277060090966e+06 + 4.248615552639492e+06 + 4.220954901894585e+06 + 4.193296725308497e+06 + 4.165642642799490e+06 + 4.137994276751838e+06 + 4.110353252015833e+06 + 4.082721195907769e+06 + 4.055099738209962e+06 + 4.027490511170732e+06 + 3.999895149504419e+06 + 3.972315290391369e+06 + 3.944752573477943e+06 + 3.917208640876513e+06 + 3.889685137165464e+06 + 3.862183709389194e+06 + 3.834706007058111e+06 + 3.807253682148634e+06 + 3.779828389103201e+06 + 3.752431784830255e+06 + 3.725065528704253e+06 + 3.697731282565667e+06 + 3.670430710720977e+06 + 3.643165479942679e+06 + 3.615937259469279e+06 + 3.588747721005293e+06 + 3.561598538721256e+06 + 3.534491389253709e+06 + 3.507427951705208e+06 + 3.480409907644318e+06 + 3.453438941105620e+06 + 3.426516738589706e+06 + 3.399644989063177e+06 + 3.372825383958653e+06 + 3.346070405300663e+06 + 3.319465764034668e+06 + 3.293075510459727e+06 + 3.266878224028187e+06 + 3.240869303162930e+06 + 3.215052515525467e+06 + 3.189425288821125e+06 + 3.163986116962672e+06 + 3.138734196986282e+06 + 3.113668261011591e+06 + 3.088787111422079e+06 + 3.064089610843659e+06 + 3.039574592322599e+06 + 3.015240898358055e+06 + 2.991087379108502e+06 + 2.967112887579310e+06 + 2.943316282301805e+06 + 2.919696425519270e+06 + 2.896252185185970e+06 + 2.872982434032476e+06 + 2.849886048305633e+06 + 2.826961910405915e+06 + 2.804208907130155e+06 + 2.781625929123051e+06 + 2.759211873644148e+06 + 2.736965641930875e+06 + 2.714886139213983e+06 + 2.692972277509980e+06 + 2.671222972611625e+06 + 2.649637144535781e+06 + 2.628213719738566e+06 + 2.606951628696007e+06 + 2.585849806893141e+06 + 2.564907195391811e+06 + 2.544122739173453e+06 + 2.523495389008342e+06 + 2.503024101022326e+06 + 2.482707834844927e+06 + 2.462545556054412e+06 + 2.442536236120740e+06 + 2.422678849863131e+06 + 2.402972377820649e+06 + 2.383415806334849e+06 + 2.364008125120872e+06 + 2.344748329835404e+06 + 2.325635421794777e+06 + 2.306668405813749e+06 + 2.287846292982958e+06 + 2.269168099757709e+06 + 2.250632846050586e+06 + 2.232239558087812e+06 + 2.213987266957964e+06 + 2.195875007435864e+06 + 2.177901821473463e+06 + 2.160066755143500e+06 + 2.142368857948063e+06 + 2.124807187017495e+06 + 2.107380803567688e+06 + 2.090088772295509e+06 + 2.072930165358991e+06 + 2.055904058912681e+06 + 2.039009532618940e+06 + 2.022245673041180e+06 + 2.005611571515091e+06 + 1.989106323649552e+06 + 1.972729030298793e+06 + 1.956478797095825e+06 + 1.940354734917566e+06 + 1.924355960016194e+06 + 1.908481592593613e+06 + 1.892730757829583e+06 + 1.877102587255698e+06 + 1.861596215902473e+06 + 1.846210783186639e+06 + 1.830945435012332e+06 + 1.815799321323061e+06 + 1.800771597138300e+06 + 1.785861422681660e+06 + 1.771067961851037e+06 + 1.756390384114008e+06 + 1.741827864373811e+06 + 1.727379581118251e+06 + 1.713044718232857e+06 + 1.698822464839031e+06 + 1.684712013805151e+06 + 1.670712563808684e+06 + 1.656823318006428e+06 + 1.643043483055245e+06 + 1.629372271790869e+06 + 1.615808901794204e+06 + 1.602352594044968e+06 + 1.589002574975609e+06 + 1.575758076322615e+06 + 1.562618333634041e+06 + 1.549582587186907e+06 + 1.536650081999105e+06 + 1.523820067124634e+06 + 1.511091797064626e+06 + 1.498464530780005e+06 + 1.485937531189585e+06 + 1.473510066471004e+06 + 1.461181408848471e+06 + 1.448950834625962e+06 + 1.436817625926470e+06 + 1.424781068397768e+06 + 1.412840451407005e+06 + 1.400995070309470e+06 + 1.389244224245752e+06 + 1.377587216393601e+06 + 1.366023355231714e+06 + 1.354551952598923e+06 + 1.343172324427429e+06 + 1.331883792566280e+06 + 1.320685682335073e+06 + 1.309577322956444e+06 + 1.298558048406515e+06 + 1.287627196362487e+06 + 1.276784109496439e+06 + 1.266028134784103e+06 + 1.255358622320226e+06 + 1.244774926866806e+06 + 1.234276408467881e+06 + 1.223862430240984e+06 + 1.213532358913532e+06 + 1.203285566621101e+06 + 1.193121428991948e+06 + 1.183039325468658e+06 + 1.173038640304339e+06 + 1.163118761212139e+06 + 1.153279079987504e+06 + 1.143518992564451e+06 + 1.133837898317260e+06 + 1.124235201394960e+06 + 1.114710310121780e+06 + 1.105262636013577e+06 + 1.095891594215875e+06 + 1.086596604396163e+06 + 1.077377090233802e+06 + 1.068232479242110e+06 + 1.059162202266971e+06 + 1.050165693700918e+06 + 1.041242393088349e+06 + 1.032391743039430e+06 + 1.023613189178869e+06 + 1.014906181731556e+06 + 1.006270174355871e+06 + 9.977046244036794e+05 + 9.892089935677024e+05 + 9.807827465832273e+05 + 9.724253516436865e+05 + 9.641362810046984e+05 + 9.559150102252770e+05 + 9.477610188719215e+05 + 9.396737899139688e+05 + 9.316528091743080e+05 + 9.236975665277143e+05 + 9.158075561735997e+05 + 9.079822751134991e+05 + 9.002212236533747e+05 + 8.925239057014534e+05 + 8.848898281604102e+05 + 8.773185021612709e+05 + 8.698094424648809e+05 + 8.623621663528095e+05 + 8.549761944053203e+05 + 8.476510510446843e+05 + 8.403862637513808e+05 + 8.331813638606276e+05 + 8.260358860837971e+05 + 8.189493675787508e+05 + 8.119213484350282e+05 + 8.049513730504208e+05 + 7.980389891667470e+05 + 7.911837471192866e+05 + 7.843852003599521e+05 + 7.776429053419405e+05 + 7.709564221446526e+05 + 7.643253140173181e+05 + 7.577491471803443e+05 + 7.512274911598503e+05 + 7.447599180231568e+05 + 7.383460026539576e+05 + 7.319853241781880e+05 + 7.256774641850799e+05 + 7.194220067146936e+05 + 7.132185395309449e+05 + 7.070666529520636e+05 + 7.009659400728574e+05 + 6.949159975085063e+05 + 6.889164244086053e+05 + 6.829668228095188e+05 + 6.770667979456050e+05 + 6.712159574726904e+05 + 6.654139120281750e+05 + 6.596602754201142e+05 + 6.539546638052445e+05 + 6.482966962660485e+05 + 6.426859949531512e+05 + 6.371221843431445e+05 + 6.316048918617199e+05 + 6.261337478197705e+05 + 6.207083848228143e+05 + 6.153284385055641e+05 + 6.099935472682050e+05 + 6.047033517354903e+05 + 5.994574954765684e+05 + 5.942556247644721e+05 + 5.890973881315375e+05 + 5.839824370501742e+05 + 5.789104255366600e+05 + 5.738810098259439e+05 + 5.688938490610282e+05 + 5.639486048510919e+05 + 5.590449410545636e+05 + 5.541825244030216e+05 + 5.493610239467514e+05 + 5.445801109611018e+05 + 5.398394595653637e+05 + 5.351387461239359e+05 + 5.304776492529869e+05 + 5.258558503354813e+05 + 5.212730328842620e+05 + 5.167288826663374e+05 + 5.122230881980868e+05 + 5.077553400369428e+05 + 5.033253309731948e+05 + 4.989327564601310e+05 + 4.945773139811803e+05 + 4.902587033143605e+05 + 4.859766266490913e+05 + 4.817307881770549e+05 + 4.775208945372272e+05 + 4.733466546121029e+05 + 4.692077791818720e+05 + 4.651039814988278e+05 + 4.610349770333872e+05 + 4.570004830895544e+05 + 4.530002193970305e+05 + 4.490339078641191e+05 + 4.451012721716975e+05 + 4.412020383482487e+05 + 4.373359346346440e+05 + 4.335026910461138e+05 + 4.297020398291695e+05 + 4.259337153289251e+05 + 4.221974536699117e+05 + 4.184929932504728e+05 + 4.148200744355357e+05 + 4.111784393468680e+05 + 4.075678324287459e+05 + 4.039879999559653e+05 + 4.004386899065928e+05 + 3.969196525836200e+05 + 3.934306400492930e+05 + 3.899714060581240e+05 + 3.865417066557175e+05 + 3.831412995849459e+05 + 3.797699442729113e+05 + 3.764274023566474e+05 + 3.731134371233769e+05 + 3.698278135622827e+05 + 3.665702987827645e+05 + 3.633406615280775e+05 + 3.601386722825769e+05 + 3.569641034535824e+05 + 3.538167290782596e+05 + 3.506963250581825e+05 + 3.476026690576235e+05 + 3.445355402744773e+05 + 3.414947198014101e+05 + 3.384799905288533e+05 + 3.354911367821696e+05 + 3.325279446663284e+05 + 3.295902021531574e+05 + 3.266776986085196e+05 + 3.237902250531212e+05 + 3.209275743946921e+05 + 3.180895409646722e+05 + 3.152759207164211e+05 + 3.124865112943467e+05 + 3.097211117598721e+05 + 3.069795229448014e+05 + 3.042615472339510e+05 + 3.015669883675390e+05 + 2.988956518738279e+05 + 2.962473447339852e+05 + 2.936218752484120e+05 + 2.910190535228683e+05 + 2.884386911235514e+05 + 2.858806009067059e+05 + 2.833445973150213e+05 + 2.808304963224871e+05 + 2.783381153152796e+05 + 2.758672731341289e+05 + 2.734177900017136e+05 + 2.709894875428715e+05 + 2.685821889451269e+05 + 2.661957187314447e+05 + 2.638299027837526e+05 + 2.614845683836235e+05 + 2.591595441417535e+05 + 2.568546601323142e+05 + 2.545697477628685e+05 + 2.523046396706559e+05 + 2.500591699434964e+05 + 2.478331740644226e+05 + 2.456264886409644e+05 + 2.434389516128850e+05 + 2.412704023657298e+05 + 2.391206814330098e+05 + 2.369896306904384e+05 + 2.348770933197599e+05 + 2.327829136173563e+05 + 2.307069372775990e+05 + 2.286490111745211e+05 + 2.266089832623643e+05 + 2.245867029748616e+05 + 2.225820209237310e+05 + 2.205947887183862e+05 + 2.186248592921233e+05 + 2.166720868039439e+05 + 2.147363264605261e+05 + 2.128174347706827e+05 + 2.109152693889452e+05 + 2.090296889843596e+05 + 2.071605535077064e+05 + 2.053077240108720e+05 + 2.034710625599661e+05 + 2.016504325007521e+05 + 1.998456982030229e+05 + 1.980567250370223e+05 + 1.962833797138016e+05 + 1.945255298870997e+05 + 1.927830441323482e+05 + 1.910557922964590e+05 + 1.893436452442803e+05 + 1.876464748512476e+05 + 1.859641540986015e+05 + 1.842965568967865e+05 + 1.826435581738182e+05 + 1.810050340354470e+05 + 1.793808614993985e+05 + 1.777709185575675e+05 + 1.761750842223128e+05 + 1.745932384546507e+05 + 1.730252623268235e+05 + 1.714710377603550e+05 + 1.699304475698536e+05 + 1.684033757560533e+05 + 1.668897071565261e+05 + 1.653893274220018e+05 + 1.639021233234161e+05 + 1.624279825159149e+05 + 1.609667934564489e+05 + 1.595184456811041e+05 + 1.580828296015031e+05 + 1.566598363968120e+05 + 1.552493582649543e+05 + 1.538512882702573e+05 + 1.524655202573447e+05 + 1.510919491064028e+05 + 1.497304705096718e+05 + 1.483809809029638e+05 + 1.470433777326744e+05 + 1.457175592117312e+05 + 1.444034242942686e+05 + 1.431008729933990e+05 + 1.418098060421684e+05 + 1.405301248671864e+05 + 1.392617319094920e+05 + 1.380045303259556e+05 + 1.367584239714278e+05 + 1.355233176631985e+05 + 1.342991169575939e+05 + 1.330857281367400e+05 + 1.318830583220329e+05 + 1.306910153813551e+05 + 1.295095079605118e+05 + 1.283384454277201e+05 + 1.271777378619642e+05 + 1.260272961737778e+05 + 1.248870320778846e+05 + 1.237568578844075e+05 + 1.226366865814357e+05 + 1.215264320654815e+05 + 1.204260088572700e+05 + 1.193353321256488e+05 + 1.182543179024696e+05 + 1.171828828329888e+05 + 1.161209441971572e+05 + 1.150684200368906e+05 + 1.140252290172446e+05 + 1.129912905290012e+05 + 1.119665246989244e+05 + 1.109508522377596e+05 + 1.099441945145749e+05 + 1.089464735474612e+05 + 1.079576119759672e+05 + 1.069775332658437e+05 + 1.060061613940063e+05 + 1.050434208615313e+05 + 1.040892370245828e+05 + 1.031435357647047e+05 + 1.022062434696232e+05 + 1.012772873208916e+05 + 1.003565950684523e+05 + 9.944409498683986e+04 + 9.853971604124359e+04 + 9.764338777276211e+04 + 9.675504027159151e+04 + 9.587460425195689e+04 + 9.500201099602232e+04 + 9.413719236230559e+04 + 9.328008081668093e+04 + 9.243060935253548e+04 + 9.158871152497102e+04 + 9.075432150057999e+04 + 8.992737394814150e+04 + 8.910780408381460e+04 + 8.829554772812854e+04 + 8.749054118346877e+04 + 8.669272128864085e+04 + 8.590202548133289e+04 + 8.511839166524833e+04 + 8.434175826486869e+04 + 8.357206428147563e+04 + 8.280924917713979e+04 + 8.205325293940101e+04 + 8.130401609766987e+04 + 8.056147962909786e+04 + 7.982558503959214e+04 + 7.909627435336930e+04 + 7.837349003289423e+04 + 7.765717506702039e+04 + 7.694727293156023e+04 + 7.624372753229675e+04 + 7.554648331079309e+04 + 7.485548516457589e+04 + 7.417067840840557e+04 + 7.349200889763620e+04 + 7.281942291879337e+04 + 7.215286716185752e+04 + 7.149228885123666e+04 + 7.083763563584826e+04 + 7.018885556144921e+04 + 6.954589717270633e+04 + 6.890870943981725e+04 + 6.827724174191867e+04 + 6.765144392787226e+04 + 6.703126624692857e+04 + 6.641665935636370e+04 + 6.580757439267423e+04 + 6.520396286738058e+04 + 6.460577668030654e+04 + 6.401296820028803e+04 + 6.342549017134596e+04 + 6.284329573351656e+04 + 6.226633845994343e+04 + 6.169457228450359e+04 + 6.112795154311772e+04 + 6.056643099463883e+04 + 6.000996574136701e+04 + 5.945851127624574e+04 + 5.891202351267684e+04 + 5.837045869755570e+04 + 5.783377345558033e+04 + 5.730192482224794e+04 + 5.677487016169035e+04 + 5.625256720967715e+04 + 5.573497408983271e+04 + 5.522204924870504e+04 + 5.471375150828325e+04 + 5.421004006036229e+04 + 5.371087441354901e+04 + 5.321621444975662e+04 + 5.272602039469258e+04 + 5.224025278120609e+04 + 5.175887251825204e+04 + 5.128184085536604e+04 + 5.080911934597862e+04 + 5.034066989608183e+04 + 4.987645473684455e+04 + 4.941643640415772e+04 + 4.896057778639881e+04 + 4.850884207769758e+04 + 4.806119277190277e+04 + 4.761759372195903e+04 + 4.717800906318111e+04 + 4.674240321587012e+04 + 4.631074096205200e+04 + 4.588298735942246e+04 + 4.545910774537743e+04 + 4.503906779825089e+04 + 4.462283347373459e+04 + 4.421037101686000e+04 + 4.380164697651120e+04 + 4.339662816996042e+04 + 4.299528171729217e+04 + 4.259757504220949e+04 + 4.220347581706527e+04 + 4.181295199605238e+04 + 4.142597183433527e+04 + 4.104250383855765e+04 + 4.066251680332360e+04 + 4.028597979815155e+04 + 3.991286213370509e+04 + 3.954313341370015e+04 + 3.917676350485435e+04 + 3.881372250661203e+04 + 3.845398080843414e+04 + 3.809750905736740e+04 + 3.774427812448402e+04 + 3.739425915595785e+04 + 3.704742356207799e+04 + 3.670374297805164e+04 + 3.636318929176292e+04 + 3.602573463927982e+04 + 3.569135138416072e+04 + 3.536001215672983e+04 + 3.503168981923036e+04 + 3.470635744840465e+04 + 3.438398837321134e+04 + 3.406455615393208e+04 + 3.374803457309094e+04 + 3.343439765983423e+04 + 3.312361965186892e+04 + 3.281567499925836e+04 + 3.251053841177231e+04 + 3.220818480242078e+04 + 3.190858928930939e+04 + 3.161172723179306e+04 + 3.131757418548949e+04 + 3.102610591483113e+04 + 3.073729842710719e+04 + 3.045112791454117e+04 + 3.016757076691534e+04 + 2.988660361584138e+04 + 2.960820327321778e+04 + 2.933234674204399e+04 + 2.905901125674980e+04 + 2.878817423326317e+04 + 2.851981328372868e+04 + 2.825390622682930e+04 + 2.799043106146120e+04 + 2.772936599846356e+04 + 2.747068942978120e+04 + 2.721437991599866e+04 + 2.696041623625263e+04 + 2.670877735469368e+04 + 2.645944239579323e+04 + 2.621239068106394e+04 + 2.596760172331024e+04 + 2.572505519859007e+04 + 2.548473096648493e+04 + 2.524660907253227e+04 + 2.501066972455740e+04 + 2.477689330466365e+04 + 2.454526037995217e+04 + 2.431575168300119e+04 + 2.408834810912627e+04 + 2.386303072521571e+04 + 2.363978076395976e+04 + 2.341857962988761e+04 + 2.319940888531031e+04 + 2.298225025100848e+04 + 2.276708562349589e+04 + 2.255389704384009e+04 + 2.234266670372097e+04 + 2.213337697918792e+04 + 2.192601038403303e+04 + 2.172054957464747e+04 + 2.151697738916041e+04 + 2.131527680276242e+04 + 2.111543093252480e+04 + 2.091742306098243e+04 + 2.072123660422193e+04 + 2.052685512482519e+04 + 2.033426234716574e+04 + 2.014344212098682e+04 + 1.995437843426629e+04 + 1.976705544479930e+04 + 1.958145743480243e+04 + 1.939756881213320e+04 + 1.921537413684756e+04 + 1.903485810052154e+04 + 1.885600554387774e+04 + 1.867880143037429e+04 + 1.850323084111479e+04 + 1.832927901452323e+04 + 1.815693131178678e+04 + 1.798617320994377e+04 + 1.781699034075772e+04 + 1.764936845332217e+04 + 1.748329340510924e+04 + 1.731875119617678e+04 + 1.715572795110660e+04 + 1.699420990919751e+04 + 1.683418344358223e+04 + 1.667563503873475e+04 + 1.651855128954979e+04 + 1.636291893365341e+04 + 1.620872481606836e+04 + 1.605595588699932e+04 + 1.590459923143625e+04 + 1.575464203906202e+04 + 1.560607160533683e+04 + 1.545887535681969e+04 + 1.531304082443131e+04 + 1.516855564487381e+04 + 1.502540756956110e+04 + 1.488358445432479e+04 + 1.474307426938193e+04 + 1.460386509288249e+04 + 1.446594509814283e+04 + 1.432930256776952e+04 + 1.419392589719183e+04 + 1.405980357729324e+04 + 1.392692420658459e+04 + 1.379527648729671e+04 + 1.366484921045415e+04 + 1.353563127160237e+04 + 1.340761167272577e+04 + 1.328077950662285e+04 + 1.315512397004589e+04 + 1.303063434970357e+04 + 1.290730001970431e+04 + 1.278511047006237e+04 + 1.266405528001385e+04 + 1.254412410683634e+04 + 1.242530670538782e+04 + 1.230759293022181e+04 + 1.219097272173875e+04 + 1.207543611008671e+04 + 1.196097321598039e+04 + 1.184757424417080e+04 + 1.173522949287449e+04 + 1.162392934478714e+04 + 1.151366426384512e+04 + 1.140442480834694e+04 + 1.129620161634352e+04 + 1.118898540470730e+04 + 1.108276698378667e+04 + 1.097753724013271e+04 + 1.087328713748510e+04 + 1.077000773143687e+04 + 1.066769015148828e+04 + 1.056632560317495e+04 + 1.046590537899618e+04 + 1.036642084361778e+04 + 1.026786343864257e+04 + 1.017022468912003e+04 + 1.007349618928313e+04 + 9.977669609110566e+03 + 9.882736699245763e+03 + 9.788689277747368e+03 + 9.695519238085319e+03 + 9.603218551128393e+03 + 9.511779252802526e+03 + 9.421193453258842e+03 + 9.331453338088948e+03 + 9.242551155902798e+03 + 9.154479227014926e+03 + 9.067229945268125e+03 + 8.980795767109534e+03 + 8.895169220035050e+03 + 8.810342900070031e+03 + 8.726309463870590e+03 + 8.643061639313548e+03 + 8.560592220958239e+03 + 8.478894062727115e+03 + 8.397960086705119e+03 + 8.317783279679375e+03 + 8.238356687487571e+03 + 8.159673421912185e+03 + 8.081726656780256e+03 + 8.004509624589081e+03 + 7.928015622553179e+03 + 7.852238006535134e+03 + 7.777170190090504e+03 + 7.702805651284038e+03 + 7.629137923269694e+03 + 7.556160595291118e+03 + 7.483867321684783e+03 + 7.412251809739373e+03 + 7.341307820993124e+03 + 7.271029178292266e+03 + 7.201409756944273e+03 + 7.132443488123264e+03 + 7.064124361053455e+03 + 6.996446414793790e+03 + 6.929403743407001e+03 + 6.862990497342477e+03 + 6.797200875139910e+03 + 6.732029129721631e+03 + 6.667469570161823e+03 + 6.603516551523545e+03 + 6.540164480186692e+03 + 6.477407817185619e+03 + 6.415241069687017e+03 + 6.353658796883785e+03 + 6.292655608307650e+03 + 6.232226157518487e+03 + 6.172365149982773e+03 + 6.113067340658933e+03 + 6.054327527853383e+03 + 5.996140560115221e+03 + 5.938501333076975e+03 + 5.881404784939355e+03 + 5.824845903828315e+03 + 5.768819722461626e+03 + 5.713321315085574e+03 + 5.658345805472132e+03 + 5.603888359764917e+03 + 5.549944184846402e+03 + 5.496508537213697e+03 + 5.443576713947663e+03 + 5.391144051368102e+03 + 5.339205932358427e+03 + 5.287757780134069e+03 + 5.236795058291703e+03 + 5.186313275265001e+03 + 5.136307977685731e+03 + 5.086774751617681e+03 + 5.037709226449485e+03 + 4.989107068721900e+03 + 4.940963984504954e+03 + 4.893275721702213e+03 + 4.846038063285039e+03 + 4.799246830729600e+03 + 4.752897888312231e+03 + 4.706987133608333e+03 + 4.661510499922751e+03 + 4.616463963202395e+03 + 4.571843532909213e+03 + 4.527645253893021e+03 + 4.483865209302220e+03 + 4.440499515324517e+03 + 4.397544325863069e+03 + 4.354995830474305e+03 + 4.312850249817291e+03 + 4.271103841242844e+03 + 4.229752898292512e+03 + 4.188793745275323e+03 + 4.148222740634546e+03 + 4.108036277499050e+03 + 4.068230779797937e+03 + 4.028802705831530e+03 + 3.989748546349355e+03 + 3.951064821861195e+03 + 3.912748086813885e+03 + 3.874794926585441e+03 + 3.837201956255947e+03 + 3.799965825835211e+03 + 3.763083213042953e+03 + 3.726550823283946e+03 + 3.690365398229224e+03 + 3.654523706502605e+03 + 3.619022543185401e+03 + 3.583858737044133e+03 + 3.549029144619817e+03 + 3.514530650270483e+03 + 3.480360168271582e+03 + 3.446514639490202e+03 + 3.412991033253400e+03 + 3.379786348393921e+03 + 3.346897609451590e+03 + 3.314321868815865e+03 + 3.282056205761040e+03 + 3.250097724763853e+03 + 3.218443559530467e+03 + 3.187090870156345e+03 + 3.156036839918450e+03 + 3.125278679259401e+03 + 3.094813626126841e+03 + 3.064638941053712e+03 + 3.034751909739014e+03 + 3.005149845581890e+03 + 2.975830084543172e+03 + 2.946789986568892e+03 + 2.918026936797055e+03 + 2.889538343339952e+03 + 2.861321641252218e+03 + 2.833374286700636e+03 + 2.805693756659566e+03 + 2.778277556469224e+03 + 2.751123212016022e+03 + 2.724228269242043e+03 + 2.697590301914011e+03 + 2.671206904109025e+03 + 2.645075688971241e+03 + 2.619194294405700e+03 + 2.593560380141823e+03 + 2.568171626715687e+03 + 2.543025737542386e+03 + 2.518120435211838e+03 + 2.493453462030127e+03 + 2.469022585172818e+03 + 2.444825590018873e+03 + 2.420860280374604e+03 + 2.397124484732366e+03 + 2.373616049394018e+03 + 2.350332838436762e+03 + 2.327272738609171e+03 + 2.304433655218562e+03 + 2.281813512846715e+03 + 2.259410255601743e+03 + 2.237221844685107e+03 + 2.215246260811306e+03 + 2.193481506123082e+03 + 2.171925598942706e+03 + 2.150576574458654e+03 + 2.129432488764117e+03 + 2.108491414442484e+03 + 2.087751441300839e+03 + 2.067210678851041e+03 + 2.046867252353801e+03 + 2.026719303565292e+03 + 2.006764993817073e+03 + 1.987002500493615e+03 + 1.967430017215378e+03 + 1.948045754608514e+03 + 1.928847939062915e+03 + 1.909834815497017e+03 + 1.891004644375446e+03 + 1.872355700452609e+03 + 1.853886275802303e+03 + 1.835594678424827e+03 + 1.817479231062563e+03 + 1.799538272810955e+03 + 1.781770158220284e+03 + 1.764173256646331e+03 + 1.746745953278825e+03 + 1.729486647489427e+03 + 1.712393752933919e+03 + 1.695465699568615e+03 + 1.678700930817119e+03 + 1.662097903850073e+03 + 1.645655091966725e+03 + 1.629370981458450e+03 + 1.613244072202941e+03 + 1.597272880432907e+03 + 1.581455933682248e+03 + 1.565791771294578e+03 + 1.550278951289465e+03 + 1.534916043199639e+03 + 1.519701626827566e+03 + 1.504634298568118e+03 + 1.489712666891078e+03 + 1.474935351996483e+03 + 1.460300988758884e+03 + 1.445808223853118e+03 + 1.431455716350942e+03 + 1.417242138357507e+03 + 1.403166173386480e+03 + 1.389226518010158e+03 + 1.375421881007569e+03 + 1.361750982089843e+03 + 1.348212553926219e+03 + 1.334805341072190e+03 + 1.321528098720046e+03 + 1.308379594477261e+03 + 1.295358607457036e+03 + 1.282463927300085e+03 + 1.269694355823267e+03 + 1.257048705895313e+03 + 1.244525800699936e+03 + 1.232124475203576e+03 + 1.219843575042076e+03 + 1.207681956049559e+03 + 1.195638485404256e+03 + 1.183712040621380e+03 + 1.171901509388141e+03 + 1.160205790292893e+03 + 1.148623791762007e+03 + 1.137154432301071e+03 + 1.125796641257775e+03 + 1.114549357263016e+03 + 1.103411528660030e+03 + 1.092382114454342e+03 + 1.081460082700689e+03 + 1.070644411095300e+03 + 1.059934087579793e+03 + 1.049328108720195e+03 + 1.038825480494848e+03 + 1.028425218971371e+03 + 1.018126348582361e+03 + 1.007927902927668e+03 + 9.978289254104482e+02 + 9.878284675679866e+02 + 9.779255898738762e+02 + 9.681193621623704e+02 + 9.584088623012988e+02 + 9.487931771658360e+02 + 9.392714022763922e+02 + 9.298426409505287e+02 + 9.205060057185552e+02 + 9.112606172437117e+02 + 9.021056036681564e+02 + 8.930401022331439e+02 + 8.840632580637879e+02 + 8.751742235821227e+02 + 8.663721600224155e+02 + 8.576562363107178e+02 + 8.490256285516970e+02 + 8.404795212594149e+02 + 8.320171064683727e+02 + 8.236375833541338e+02 + 8.153401590664150e+02 + 8.071240480244585e+02 + 7.989884718119321e+02 + 7.909326598382663e+02 + 7.829558483527250e+02 + 7.750572805483002e+02 + 7.672362073200991e+02 + 7.594918862313154e+02 + 7.518235817392584e+02 + 7.442305655888715e+02 + 7.367121158939042e+02 + 7.292675176279554e+02 + 7.218960630476328e+02 + 7.145970504767349e+02 + 7.073697847820883e+02 + 7.002135778527235e+02 + 6.931277475695276e+02 + 6.861116184218383e+02 + 6.791645215373005e+02 + 6.722857937677298e+02 + 6.654747784472139e+02 + 6.587308253877042e+02 + 6.520532900189463e+02 + 6.454415341582659e+02 + 6.388949257382737e+02 + 6.324128381494658e+02 + 6.259946511571761e+02 + 6.196397502963263e+02 + 6.133475264197438e+02 + 6.071173768094372e+02 + 6.009487042352257e+02 + 5.948409165626937e+02 + 5.887934278801442e+02 + 5.828056577149104e+02 + 5.768770306335107e+02 + 5.710069770500405e+02 + 5.651949326339243e+02 + 5.594403381224611e+02 + 5.537426401234757e+02 + 5.481012901152631e+02 + 5.425157443662293e+02 + 5.369854648776029e+02 + 5.315099185870243e+02 + 5.260885773011631e+02 + 5.207209179649268e+02 + 5.154064223119497e+02 + 5.101445770921985e+02 + 5.049348740937783e+02 + 4.997768095346806e+02 + 4.946698844046663e+02 + 4.896136047862775e+02 + 4.846074811232588e+02 + 4.796510285618583e+02 + 4.747437670096575e+02 + 4.698852205388910e+02 + 4.650749179071526e+02 + 4.603123926691712e+02 + 4.555971823563725e+02 + 4.509288288727789e+02 + 4.463068788207030e+02 + 4.417308827523721e+02 + 4.372003955530256e+02 + 4.327149765501697e+02 + 4.282741889383891e+02 + 4.238776002586995e+02 + 4.195247821802792e+02 + 4.152153101301976e+02 + 4.109487639564401e+02 + 4.067247273397822e+02 + 4.025427875685178e+02 + 3.984025363858318e+02 + 3.943035692477232e+02 + 3.902454851077117e+02 + 3.862278871965347e+02 + 3.822503823673904e+02 + 3.783125809462488e+02 + 3.744140974100913e+02 + 3.705545496319825e+02 + 3.667335588488600e+02 + 3.629507504654170e+02 + 3.592057531920941e+02 + 3.554981990087823e+02 + 3.518277237997781e+02 + 3.481939667588409e+02 + 3.445965704231110e+02 + 3.410351808614133e+02 + 3.375094474036331e+02 + 3.340190228447264e+02 + 3.305635632260179e+02 + 3.271427276872922e+02 + 3.237561788604219e+02 + 3.204035826343936e+02 + 3.170846077594078e+02 + 3.137989262426811e+02 + 3.105462135925898e+02 + 3.073261480789523e+02 + 3.041384108878384e+02 + 3.009826866110996e+02 + 2.978586626197819e+02 + 2.947660292278245e+02 + 2.917044799774912e+02 + 2.886737110898206e+02 + 2.856734216347172e+02 + 2.827033138205502e+02 + 2.797630925185924e+02 + 2.768524653868921e+02 + 2.739711430162750e+02 + 2.711188386172004e+02 + 2.682952682417776e+02 + 2.655001507075566e+02 + 2.627332073441469e+02 + 2.599941621974095e+02 + 2.572827420191948e+02 + 2.545986760992428e+02 + 2.519416963867954e+02 + 2.493115374004747e+02 + 2.467079361520411e+02 + 2.441306322519956e+02 + 2.415793676749632e+02 + 2.390538868162835e+02 + 2.365539368283623e+02 + 2.340792670618049e+02 + 2.316296291092408e+02 + 2.292047772816256e+02 + 2.268044680583002e+02 + 2.244284601550858e+02 + 2.220765149177526e+02 + 2.197483957886078e+02 + 2.174438683808943e+02 + 2.151627007150871e+02 + 2.129046628991759e+02 + 2.106695273325222e+02 + 2.084570687245048e+02 + 2.062670637405841e+02 + 2.040992912166423e+02 + 2.019535322675416e+02 + 1.998295699356443e+02 + 1.977271893831886e+02 + 1.956461779038128e+02 + 1.935863247355415e+02 + 1.915474213798307e+02 + 1.895292611678483e+02 + 1.875316391958052e+02 + 1.855543528482985e+02 + 1.835972014670904e+02 + 1.816599861324650e+02 + 1.797425099308505e+02 + 1.778445779263947e+02 + 1.759659969484732e+02 + 1.741065756739388e+02 + 1.722661246609050e+02 + 1.704444562587039e+02 + 1.686413847511611e+02 + 1.668567261386198e+02 + 1.650902980928235e+02 + 1.633419201885993e+02 + 1.616114136777071e+02 + 1.598986014805750e+02 + 1.582033083619683e+02 + 1.565253606828442e+02 + 1.548645864376032e+02 + 1.532208154238845e+02 + 1.515938789750561e+02 + 1.499836100268175e+02 + 1.483898432531407e+02 + 1.468124148175615e+02 + 1.452511624739094e+02 + 1.437059256375598e+02 + 1.421765451600804e+02 + 1.406628634671130e+02 + 1.391647245894945e+02 + 1.376819739463268e+02 + 1.362144585029153e+02 + 1.347620267668397e+02 + 1.333245285902542e+02 + 1.319018153476916e+02 + 1.304937399101563e+02 + 1.291001564496927e+02 + 1.277209206089626e+02 + 1.263558894821456e+02 + 1.250049214462411e+02 + 1.236678763269726e+02 + 1.223446153162142e+02 + 1.210350008534558e+02 + 1.197388968258891e+02 + 1.184561684202564e+02 + 1.171866820332312e+02 + 1.159303054814129e+02 + 1.146869078350233e+02 + 1.134563593521917e+02 + 1.122385316718855e+02 + 1.110332976232454e+02 + 1.098405311979204e+02 + 1.086601077608261e+02 + 1.074919038117468e+02 + 1.063357969778381e+02 + 1.051916662270612e+02 + 1.040593916259906e+02 + 1.029388543466298e+02 + 1.018299368341955e+02 + 1.007325226151947e+02 + 9.964649633160360e+01 + 9.857174381861689e+01 + 9.750815195741964e+01 + 9.645560874783912e+01 + 9.541400332430685e+01 + 9.438322582683587e+01 + 9.336316750654088e+01 + 9.235372071959520e+01 + 9.135477878367931e+01 + 9.036623609174318e+01 + 8.938798812110527e+01 + 8.841993128541375e+01 + 8.746196304638978e+01 + 8.651398190997742e+01 + 8.557588728936869e+01 + 8.464757961904559e+01 + 8.372896034435935e+01 + 8.281993180539014e+01 + 8.192039735296085e+01 + 8.103026128111058e+01 + 8.014942874495698e+01 + 7.927780590401203e+01 + 7.841529983858037e+01 + 7.756181847617736e+01 + 7.671727071446733e+01 + 7.588156631674119e+01 + 7.505461587372024e+01 + 7.423633095310755e+01 + 7.342662394299153e+01 + 7.262540803009301e+01 + 7.183259736215271e+01 + 7.104810687905130e+01 + 7.027185230277652e+01 + 6.950375028269488e+01 + 6.874371822566854e+01 + 6.799167430460268e+01 + 6.724753760248480e+01 + 6.651122793902957e+01 + 6.578266588744034e+01 + 6.506177289160765e+01 + 6.434847109859102e+01 + 6.364268339115934e+01 + 6.294433351272004e+01 + 6.225334589309644e+01 + 6.156964567793161e+01 + 6.089315882540817e+01 + 6.022381196253272e+01 + 5.956153243192431e+01 + 5.890624835784802e+01 + 5.825788851805937e+01 + 5.761638239540161e+01 + 5.698166019812880e+01 + 5.635365277455941e+01 + 5.573229169643452e+01 + 5.511750921575248e+01 + 5.450923819422894e+01 + 5.390741220139490e+01 + 5.331196548214332e+01 + 5.272283288224639e+01 + 5.213994992103501e+01 + 5.156325275682114e+01 + 5.099267813956413e+01 + 5.042816349744366e+01 + 4.986964685712531e+01 + 4.931706681520628e+01 + 4.877036264381848e+01 + 4.822947419115650e+01 + 4.769434186145065e+01 + 4.716490671386568e+01 + 4.664111036072763e+01 + 4.612289496053356e+01 + 4.561020331071316e+01 + 4.510297873475514e+01 + 4.460116508782800e+01 + 4.410470685618211e+01 + 4.361354903662819e+01 + 4.312763714409483e+01 + 4.264691728541290e+01 + 4.217133607138089e+01 + 4.170084064118655e+01 + 4.123537869187841e+01 + 4.077489840121474e+01 + 4.031934846724150e+01 + 3.986867813088396e+01 + 3.942283710103097e+01 + 3.898177559914080e+01 + 3.854544435810492e+01 + 3.811379455962712e+01 + 3.768677789368684e+01 + 3.726434656726136e+01 + 3.684645322013298e+01 + 3.643305096011807e+01 + 3.602409338288954e+01 + 3.561953452360934e+01 + 3.521932891649128e+01 + 3.482343153306697e+01 + 3.443179775085747e+01 + 3.404438343901749e+01 + 3.366114490993698e+01 + 3.328203887889210e+01 + 3.290702252020257e+01 + 3.253605343510379e+01 + 3.216908962260722e+01 + 3.180608952699355e+01 + 3.144701200592500e+01 + 3.109181631097627e+01 + 3.074046212448961e+01 + 3.039290952642646e+01 + 3.004911898312090e+01 + 2.970905136572861e+01 + 2.937266792505025e+01 + 2.903993030352619e+01 + 2.871080057194543e+01 + 2.838524113862799e+01 + 2.806321475945346e+01 + 2.774468462701056e+01 + 2.742961427426368e+01 + 2.711796757973106e+01 + 2.680970882898042e+01 + 2.650480264209962e+01 + 2.620321398820359e+01 + 2.590490820737408e+01 + 2.560985096325826e+01 + 2.531800827674159e+01 + 2.502934653543958e+01 + 2.474383242809609e+01 + 2.446143296972707e+01 + 2.418211555345684e+01 + 2.390584787621480e+01 + 2.363259794654911e+01 + 2.336233413255323e+01 + 2.309502509558490e+01 + 2.283063979379989e+01 + 2.256914753657127e+01 + 2.231051792982671e+01 + 2.205472087402378e+01 + 2.180172658443576e+01 + 2.155150556919213e+01 + 2.130402866286656e+01 + 2.105926696778129e+01 + 2.081719185319289e+01 + 2.057777503962135e+01 + 2.034098850331043e+01 + 2.010680447006149e+01 + 1.987519550063876e+01 + 1.964613442528577e+01 + 1.941959432580914e+01 + 1.919554855827313e+01 + 1.897397075515799e+01 + 1.875483482712853e+01 + 1.853811494931854e+01 + 1.832378554313712e+01 + 1.811182128985819e+01 + 1.790219714756028e+01 + 1.769488831219516e+01 + 1.748987022919803e+01 + 1.728711861052799e+01 + 1.708660939866258e+01 + 1.688831878332332e+01 + 1.669222321036695e+01 + 1.649829934690638e+01 + 1.630652410217020e+01 + 1.611687463464447e+01 + 1.592932831698975e+01 + 1.574386275792588e+01 + 1.556045580679505e+01 + 1.537908552081595e+01 + 1.519973018767385e+01 + 1.502236832516821e+01 + 1.484697865494502e+01 + 1.467354012841447e+01 + 1.450203191274907e+01 + 1.433243337158686e+01 + 1.416472409674524e+01 + 1.399888388846149e+01 + 1.383489273805109e+01 + 1.367273085728561e+01 + 1.351237865788487e+01 + 1.335381673920349e+01 + 1.319702591751814e+01 + 1.304198719993403e+01 + 1.288868177614133e+01 + 1.273709104714046e+01 + 1.258719659672914e+01 + 1.243898018736778e+01 + 1.229242378719869e+01 + 1.214750954184370e+01 + 1.200421977308728e+01 + 1.186253699764556e+01 + 1.172244390266849e+01 + 1.158392335040491e+01 + 1.144695839482221e+01 + 1.131153225250660e+01 + 1.117762830926809e+01 + 1.104523013501555e+01 + 1.091432145806843e+01 + 1.078488617566414e+01 + 1.065690836231677e+01 + 1.053037224372639e+01 + 1.040526220941685e+01 + 1.028156282144551e+01 + 1.015925878941852e+01 + 1.003833498402321e+01 + 9.918776439618298e+00 + 9.800568334039683e+00 + 9.683696005431957e+00 + 9.568144946174611e+00 + 9.453900787919052e+00 + 9.340949322860192e+00 + 9.229276489267212e+00 + 9.118868359446189e+00 + 9.009711163642070e+00 + 8.901791273796370e+00 + 8.795095191785947e+00 + 8.689609570648638e+00 + 8.585321201853549e+00 + 8.482217005296002e+00 + 8.380284046581689e+00 + 8.279509523576925e+00 + 8.179880760330461e+00 + 8.081385224417300e+00 + 7.984010508389771e+00 + 7.887744326863491e+00 + 7.792574535206823e+00 + 7.698489109519010e+00 + 7.605476146133772e+00 + 7.513523878187234e+00 + 7.422620653280592e+00 + 7.332754935473541e+00 + 7.243915322674125e+00 + 7.156090524556396e+00 + 7.069269365741123e+00 + 6.983440797017198e+00 + 6.898593877074525e+00 + 6.814717779007339e+00 + 6.731801797516630e+00 + 6.649835331100769e+00 + 6.568807890289103e+00 + 6.488709103685009e+00 + 6.409528699865692e+00 + 6.331256515961359e+00 + 6.253882503643640e+00 + 6.177396712290002e+00 + 6.101789297547966e+00 + 6.027050522250886e+00 + 5.953170745278158e+00 + 5.880140434362498e+00 + 5.807950156612827e+00 + 5.736590570385212e+00 + 5.666052441485230e+00 + 5.596326632559173e+00 + 5.527404095182261e+00 + 5.459275884510260e+00 + 5.391933149962507e+00 + 5.325367127621260e+00 + 5.259569151260657e+00 + 5.194530647433734e+00 + 5.130243129823101e+00 + 5.066698206416531e+00 + 5.003887570881058e+00 + 4.941803001453549e+00 + 4.880436373900456e+00 + 4.819779643419130e+00 + 4.759824844738533e+00 + 4.700564110200850e+00 + 4.641989649988083e+00 + 4.584093751736625e+00 + 4.526868793565781e+00 + 4.470307230074105e+00 + 4.414401594729965e+00 + 4.359144507919768e+00 + 4.304528662004969e+00 + 4.250546825086277e+00 + 4.197191851627622e+00 + 4.144456666159979e+00 + 4.092334266478478e+00 + 4.040817732600400e+00 + 3.989900213340690e+00 + 3.939574930757155e+00 + 3.889835183170760e+00 + 3.840674335613269e+00 + 3.792085827136757e+00 + 3.744063169627271e+00 + 3.696599938809659e+00 + 3.649689781941428e+00 + 3.603326418080199e+00 + 3.557503628382808e+00 + 3.512215261374414e+00 + 3.467455236465362e+00 + 3.423217534816642e+00 + 3.379496201686086e+00 + 3.336285348654454e+00 + 3.293579148649655e+00 + 3.251371841226642e+00 + 3.209657725464606e+00 + 3.168431157939940e+00 + 3.127686563504146e+00 + 3.087418425005168e+00 + 3.047621281181530e+00 + 3.008289735457784e+00 + 2.969418446796439e+00 + 2.931002129285814e+00 + 2.893035560502260e+00 + 2.855513571053400e+00 + 2.818431044951714e+00 + 2.781782927380857e+00 + 2.745564215550934e+00 + 2.709769959926351e+00 + 2.674395268211684e+00 + 2.639435297946605e+00 + 2.604885259637663e+00 + 2.570740420206636e+00 + 2.536996093812299e+00 + 2.503647644969884e+00 + 2.470690493614489e+00 + 2.438120105903952e+00 + 2.405931997235836e+00 + 2.374121735683807e+00 + 2.342684934097179e+00 + 2.311617253784086e+00 + 2.280914406216957e+00 + 2.250572146781167e+00 + 2.220586279273893e+00 + 2.190952653478714e+00 + 2.161667162009881e+00 + 2.132725747880213e+00 + 2.104124396422602e+00 + 2.075859132902099e+00 + 2.047926031580743e+00 + 2.020321209266692e+00 + 1.993040822266800e+00 + 1.966081073072867e+00 + 1.939438205166541e+00 + 1.913108500802827e+00 + 1.887088286918805e+00 + 1.861373929947225e+00 + 1.835961834345823e+00 + 1.810848447621857e+00 + 1.786030255213551e+00 + 1.761503779943143e+00 + 1.737265586321442e+00 + 1.713312275115213e+00 + 1.689640483715834e+00 + 1.666246890168493e+00 + 1.643128207113473e+00 + 1.620281182788458e+00 + 1.597702604935450e+00 + 1.575389294699183e+00 + 1.553338108095691e+00 + 1.531545938928566e+00 + 1.510009713407906e+00 + 1.488726392261371e+00 + 1.467692971937850e+00 + 1.446906480207705e+00 + 1.426363979153670e+00 + 1.406062564959602e+00 + 1.385999363940103e+00 + 1.366171536058216e+00 + 1.346576274130787e+00 + 1.327210800124654e+00 + 1.308072368828510e+00 + 1.289158266486426e+00 + 1.270465807665891e+00 + 1.251992339187792e+00 + 1.233735237889272e+00 + 1.215691907984316e+00 + 1.197859785096899e+00 + 1.180236333919823e+00 + 1.162819045897185e+00 + 1.145605442741095e+00 + 1.128593073973588e+00 + 1.111779515284567e+00 + 1.095162371880828e+00 + 1.078739275475473e+00 + 1.062507883334588e+00 + 1.046465881646835e+00 + 1.030610981874106e+00 + 1.014940920295882e+00 + 9.994534611152829e-01 + 9.841463930086627e-01 + 9.690175291593270e-01 + 9.540647093618980e-01 + 9.392857969051628e-01 + 9.246786792729085e-01 + 9.102412696338000e-01 + 8.959715035501604e-01 + 8.818673402027871e-01 + 8.679267639726700e-01 + 8.541477806828601e-01 + 8.405284189410931e-01 + 8.270667316623297e-01 + 8.137607925445277e-01 + 8.006086976874616e-01 + 7.876085666023992e-01 + 7.747585387876328e-01 + 7.620567754307064e-01 + 7.495014602973783e-01 + 7.370907966216457e-01 + 7.248230088485250e-01 + 7.126963429559050e-01 + 7.007090638413013e-01 + 6.888594572109231e-01 + 6.771458292779966e-01 + 6.655665045807759e-01 + 6.541198279036498e-01 + 6.428041637825399e-01 + 6.316178947210964e-01 + 6.205594229105623e-01 + 6.096271692194244e-01 + 5.988195719936049e-01 + 5.881350889221284e-01 + 5.775721954595900e-01 + 5.671293841176136e-01 + 5.568051664208875e-01 + 5.465980707194322e-01 + 5.365066418737608e-01 + 5.265294432641988e-01 + 5.166650544231339e-01 + 5.069120710261563e-01 + 4.972691065208522e-01 + 4.877347899800689e-01 + 4.783077664492044e-01 + 4.689866979017357e-01 + 4.597702612073233e-01 + 4.506571489021905e-01 + 4.416460700276524e-01 + 4.327357479310139e-01 + 4.239249212044141e-01 + 4.152123442913231e-01 + 4.065967853823845e-01 + 3.980770275573000e-01 + 3.896518691214699e-01 + 3.813201217302061e-01 + 3.730806117264635e-01 + 3.649321798861945e-01 + 3.568736798624809e-01 + 3.489039797704173e-01 + 3.410219613244947e-01 + 3.332265187013593e-01 + 3.255165605086018e-01 + 3.178910083898543e-01 + 3.103487958586359e-01 + 3.028888701360887e-01 + 2.955101913786422e-01 + 2.882117315106021e-01 + 2.809924755137371e-01 + 2.738514204626846e-01 + 2.667875748867065e-01 + 2.597999604522775e-01 + 2.528876103054370e-01 + 2.460495685714950e-01 + 2.392848919788540e-01 + 2.325926484518769e-01 + 2.259719167857841e-01 + 2.194217877354759e-01 + 2.129413627175208e-01 + 2.065297538274627e-01 + 2.001860848470379e-01 + 1.939094897086202e-01 + 1.876991127051828e-01 + 1.815541094417414e-01 + 1.754736452756007e-01 + 1.694568956974621e-01 + 1.635030470498157e-01 + 1.576112950508609e-01 + 1.517808453564031e-01 + 1.460109140364644e-01 + 1.403007261766342e-01 + 1.346495165784953e-01 + 1.290565300180611e-01 + 1.235210199704221e-01 + 1.180422494287421e-01 + 1.126194909047226e-01 + 1.072520253105725e-01 + 1.019391428882170e-01 + 9.668014295417104e-02 + 9.147433293860986e-02 + 8.632102939670111e-02 + 8.121955756354367e-02 + 7.616925052308399e-02 + 7.116945022739722e-02 + 6.621950693894064e-02 + 6.131877854698166e-02 + 5.646663155698649e-02 + 5.166244039926220e-02 + 4.690558691983024e-02 + 4.219546135291571e-02 + 3.753146148005052e-02 + 3.291299228995687e-02 + 2.833946692296159e-02 + 2.381030573044091e-02 + 1.932493608562982e-02 + 1.488279323951757e-02 + 1.048331933378767e-02 + 6.125963364112487e-03 + 1.810181928441051e-03 + -2.464561780353438e-03 + -6.698797872311066e-03 + -1.089304914227914e-02 + -1.504783206514415e-02 + -1.916365656771068e-02 + -2.324102555134537e-02 + -2.728043585421590e-02 + -3.128237791440187e-02 + -3.524733543518016e-02 + -3.917578628623598e-02 + -4.306820207473552e-02 + -4.692504798332473e-02 + -5.074678357205704e-02 + -5.453386225593208e-02 + -5.828673132987599e-02 + -6.200583267980977e-02 + -6.569160218738558e-02 + -6.934446987831328e-02 + -7.296486055963237e-02 + -7.655319319650564e-02 + -8.010988114185758e-02 + -8.363533270338043e-02 + -8.712995053780122e-02 + -9.059413193193132e-02 + -9.402826926020177e-02 + -9.743274938790861e-02 + -1.008079541064505e-01 + -1.041542604608216e-01 + -1.074720401131623e-01 + -1.107616598682374e-01 + -1.140234819193764e-01 + -1.172578632830942e-01 + -1.204651563472630e-01 + -1.236457089966172e-01 + -1.267998640735528e-01 + -1.299279600346222e-01 + -1.330303309819349e-01 + -1.361073061299021e-01 + -1.391592104828469e-01 + -1.421863647988072e-01 + -1.451890851855495e-01 + -1.481676836992942e-01 + -1.511224682100277e-01 + -1.540537421399553e-01 + -1.569618050599837e-01 + -1.598469524764272e-01 + -1.627094755828420e-01 + -1.655496618778017e-01 + -1.683677949236447e-01 + -1.711641541517163e-01 + -1.739390154343296e-01 + -1.766926508045859e-01 + -1.794253283544359e-01 + -1.821373127460387e-01 + -1.848288648981078e-01 + -1.875002419786971e-01 + -1.901516978443653e-01 + -1.927834826976386e-01 + -1.953958431846420e-01 + -1.979890227559544e-01 + -2.005632612952037e-01 + -2.031187952951354e-01 + -2.056558581586019e-01 + -2.081746798356756e-01 + -2.106754870586183e-01 + -2.131585035595214e-01 + -2.156239496932611e-01 + -2.180720427808593e-01 + -2.205029972492180e-01 + -2.229170242226580e-01 + -2.253143319488933e-01 + -2.276951258807381e-01 + -2.300596082676421e-01 + -2.324079786235529e-01 + -2.347404337552020e-01 + -2.370571673694448e-01 + -2.393583705568364e-01 + -2.416442317867498e-01 + -2.439149365965413e-01 + -2.461706679522762e-01 + -2.484116061949506e-01 + -2.506379289887288e-01 + -2.528498115997045e-01 + -2.550474267213886e-01 + -2.572309443123204e-01 + -2.594005320626763e-01 + -2.615563552809436e-01 + -2.636985766193694e-01 + -2.658273564814242e-01 + -2.679428529152917e-01 + -2.700452216727706e-01 + -2.721346163277996e-01 + -2.742111879420810e-01 + -2.762750854486131e-01 + -2.783264557785196e-01 + -2.803654434689624e-01 + -2.823921908087395e-01 + -2.844068381756654e-01 + -2.864095239109061e-01 + -2.884003839950212e-01 + -2.903795524955394e-01 + -2.923471617904532e-01 + -2.943033420191942e-01 + -2.962482212763220e-01 + -2.981819256807660e-01 + -3.001045795717013e-01 + -3.020163055275332e-01 + -3.039172239885258e-01 + -3.058074538261057e-01 + -3.076871122705734e-01 + -3.095563141516905e-01 + -3.114151727559051e-01 + -3.132637999546681e-01 + -3.151023057635463e-01 + -3.169307984585886e-01 + -3.187493845723606e-01 + -3.205581689117248e-01 + -3.223572548745565e-01 + -3.241467443749414e-01 + -3.259267374142799e-01 + -3.276973325691862e-01 + -3.294586270053045e-01 + -3.312107160890871e-01 + -3.329536937962613e-01 + -3.346876527428858e-01 + -3.364126840777460e-01 + -3.381288775295205e-01 + -3.398363212268976e-01 + -3.415351019703767e-01 + -3.432253052941175e-01 + -3.449070152459444e-01 + -3.465803145779997e-01 + -3.482452848007116e-01 + -3.499020059501182e-01 + -3.515505568826250e-01 + -3.531910152859590e-01 + -3.548234573752725e-01 + -3.564479582224311e-01 + -3.580645917875246e-01 + -3.596734306319520e-01 + -3.612745463110025e-01 + -3.628680093075062e-01 + -3.644538885605036e-01 + -3.660322520458910e-01 + -3.676031668332329e-01 + -3.691666987516095e-01 + -3.707229124779602e-01 + -3.722718716670178e-01 + -3.738136390608672e-01 + -3.753482761467254e-01 + -3.768758432749062e-01 + -3.783964002465877e-01 + -3.799100056353694e-01 + -3.814167166997948e-01 + -3.829165900177774e-01 + -3.844096813827866e-01 + -3.858960456354959e-01 + -3.873757364088854e-01 + -3.888488064600710e-01 + -3.903153077375676e-01 + -3.917752913452379e-01 + -3.932288075353338e-01 + -3.946759055987133e-01 + -3.961166339473677e-01 + -3.975510402779339e-01 + -3.989791715122148e-01 + -4.004010735521537e-01 + -4.018167915706618e-01 + -4.032263700983349e-01 + -4.046298528561717e-01 + -4.060272827301947e-01 + -4.074187016384220e-01 + -4.088041511045764e-01 + -4.101836719923956e-01 + -4.115573039941210e-01 + -4.129250863351419e-01 + -4.142870577221169e-01 + -4.156432559911067e-01 + -4.169937183587125e-01 + -4.183384813924751e-01 + -4.196775807600287e-01 + -4.210110517890521e-01 + -4.223389293056927e-01 + -4.236612469859990e-01 + -4.249780381358575e-01 + -4.262893357128523e-01 + -4.275951718561576e-01 + -4.288955781548201e-01 + -4.301905856512787e-01 + -4.314802245948859e-01 + -4.327645248125306e-01 + -4.340435157401011e-01 + -4.353172261319986e-01 + -4.365856843065750e-01 + -4.378489181140523e-01 + -4.391069545111200e-01 + -4.403598201489721e-01 + -4.416075414900063e-01 + -4.428501441617211e-01 + -4.440876533447125e-01 + -4.453200938994337e-01 + -4.465474900663732e-01 + -4.477698656375739e-01 + -4.489872440132049e-01 + -4.501996482091757e-01 + -4.514071007293397e-01 + -4.526096234423685e-01 + -4.538072381973440e-01 + -4.549999664346564e-01 + -4.561878287239103e-01 + -4.573708454564815e-01 + -4.585490367840940e-01 + -4.597224223838171e-01 + -4.608910215483196e-01 + -4.620548532229073e-01 + -4.632139359470852e-01 + -4.643682878385313e-01 + -4.655179266829866e-01 + -4.666628700026897e-01 + -4.678031349656974e-01 + -4.689387384105899e-01 + -4.700696968538801e-01 + -4.711960263436956e-01 + -4.723177425902391e-01 + -4.734348612979066e-01 + -4.745473978039702e-01 + -4.756553669920032e-01 + -4.767587833133353e-01 + -4.778576611599107e-01 + -4.789520147944609e-01 + -4.800418577229591e-01 + -4.811272034289876e-01 + -4.822080654661853e-01 + -4.832844566828151e-01 + -4.843563896207272e-01 + -4.854238767122197e-01 + -4.864869303811440e-01 + -4.875455627166793e-01 + -4.885997853037442e-01 + -4.896496095260600e-01 + -4.906950466868944e-01 + -4.917361080077966e-01 + -4.927728043326091e-01 + -4.938051461993963e-01 + -4.948331438840640e-01 + -4.958568075736277e-01 + -4.968761473512240e-01 + -4.978911731225704e-01 + -4.989018945002663e-01 + -4.999083208482176e-01 + -5.009104613021941e-01 + -5.019083248972503e-01 + -5.029019205602130e-01 + -5.038912570358333e-01 + -5.048763428126108e-01 + -5.058571862004539e-01 + -5.068337954285524e-01 + -5.078061785541023e-01 + -5.087743434557772e-01 + -5.097382978215317e-01 + -5.106980492175714e-01 + -5.116536051046018e-01 + -5.126049728172039e-01 + -5.135521595357905e-01 + -5.144951722594635e-01 + -5.154340177423219e-01 + -5.163687027408718e-01 + -5.172992340140365e-01 + -5.182256180631845e-01 + -5.191478612397169e-01 + -5.200659697981208e-01 + -5.209799499565910e-01 + -5.218898077706294e-01 + -5.227955490891694e-01 + -5.236971796844924e-01 + -5.245947053253370e-01 + -5.254881317688337e-01 + -5.263774645512288e-01 + -5.272627090126604e-01 + -5.281438703531137e-01 + -5.290209539977515e-01 + -5.298939652703512e-01 + -5.307629091340913e-01 + -5.316277904778723e-01 + -5.324886142110474e-01 + -5.333453852986519e-01 + -5.341981085648844e-01 + -5.350467886587517e-01 + -5.358914300418783e-01 + -5.367320373706596e-01 + -5.375686152841432e-01 + -5.384011680734792e-01 + -5.392297001047128e-01 + -5.400542157871245e-01 + -5.408747194069761e-01 + -5.416912150785628e-01 + -5.425037068232484e-01 + -5.433121987186342e-01 + -5.441166948984431e-01 + -5.449171994793097e-01 + -5.457137163885653e-01 + -5.465062494211190e-01 + -5.472948023341125e-01 + -5.480793790518950e-01 + -5.488599834110802e-01 + -5.496366191273470e-01 + -5.504092899443518e-01 + -5.511779994928784e-01 + -5.519427513029639e-01 + -5.527035490710489e-01 + -5.534603964642170e-01 + -5.542132970155166e-01 + -5.549622540841348e-01 + -5.557072711574547e-01 + -5.564483519201564e-01 + -5.571854997347668e-01 + -5.579187179919134e-01 + -5.586480102698358e-01 + -5.593733798207278e-01 + -5.600948299343745e-01 + -5.608123641702291e-01 + -5.615259858006734e-01 + -5.622356980349319e-01 + -5.629415042452564e-01 + -5.636434078437447e-01 + -5.643414121719204e-01 + -5.650355204128609e-01 + -5.657257359067179e-01 + -5.664120620074502e-01 + -5.670945018655422e-01 + -5.677730587569899e-01 + -5.684477360210265e-01 + -5.691185368946378e-01 + -5.697854646230838e-01 + -5.704485224954847e-01 + -5.711077138615709e-01 + -5.717630418949810e-01 + -5.724145097236072e-01 + -5.730621208149326e-01 + -5.737058785857441e-01 + -5.743457862811681e-01 + -5.749818469287452e-01 + -5.756140639044424e-01 + -5.762424408221788e-01 + -5.768669806578469e-01 + -5.774876866716016e-01 + -5.781045625480883e-01 + -5.787176116189466e-01 + -5.793268371095267e-01 + -5.799322422747063e-01 + -5.805338304985933e-01 + -5.811316053198213e-01 + -5.817255703521417e-01 + -5.823157287697720e-01 + -5.829020838792335e-01 + -5.834846393919914e-01 + -5.840633987477251e-01 + -5.846383653791880e-01 + -5.852095428966003e-01 + -5.857769347455314e-01 + -5.863405444497650e-01 + -5.869003757905312e-01 + -5.874564321893059e-01 + -5.880087171065591e-01 + -5.885572344676900e-01 + -5.891019879107314e-01 + -5.896429809482692e-01 + -5.901802172691951e-01 + -5.907137006344829e-01 + -5.912434348345698e-01 + -5.917694236578408e-01 + -5.922916709366750e-01 + -5.928101804947136e-01 + -5.933249560457028e-01 + -5.938360014505492e-01 + -5.943433207019595e-01 + -5.948469177828256e-01 + -5.953467966083078e-01 + -5.958429610716830e-01 + -5.963354151930935e-01 + -5.968241629967538e-01 + -5.973092085065336e-01 + -5.977905558475627e-01 + -5.982682091919279e-01 + -5.987421727060626e-01 + -5.992124504335189e-01 + -5.996790465513929e-01 + -6.001419654039183e-01 + -6.006012111567195e-01 + -6.010567880969444e-01 + -6.015087007096906e-01 + -6.019569532209035e-01 + -6.024015499278780e-01 + -6.028424953542454e-01 + -6.032797938560218e-01 + -6.037134498528102e-01 + -6.041434679778884e-01 + -6.045698527611498e-01 + -6.049926086966888e-01 + -6.054117403213235e-01 + -6.058272522301629e-01 + -6.062391491063152e-01 + -6.066474357377917e-01 + -6.070521167750024e-01 + -6.074531969194140e-01 + -6.078506811508277e-01 + -6.082445741313003e-01 + -6.086348804777613e-01 + -6.090216052590219e-01 + -6.094047534940796e-01 + -6.097843300758768e-01 + -6.101603397998798e-01 + -6.105327877567767e-01 + -6.109016791586246e-01 + -6.112670188349342e-01 + -6.116288118244032e-01 + -6.119870634033990e-01 + -6.123417788015699e-01 + -6.126929631816058e-01 + -6.130406216857481e-01 + -6.133847595846266e-01 + -6.137253821544012e-01 + -6.140624946688724e-01 + -6.143961025253998e-01 + -6.147262111225434e-01 + -6.150528258526081e-01 + -6.153759522847098e-01 + -6.156955958775930e-01 + -6.160117619293205e-01 + -6.163244560848270e-01 + -6.166336840291214e-01 + -6.169394513190288e-01 + -6.172417635406579e-01 + -6.175406263458638e-01 + -6.178360454670117e-01 + -6.181280266570051e-01 + -6.184165756494230e-01 + -6.187016981615432e-01 + -6.189834001775048e-01 + -6.192616876391890e-01 + -6.195365661632112e-01 + -6.198080416901866e-01 + -6.200761202902517e-01 + -6.203408078682074e-01 + -6.206021103982113e-01 + -6.208600339819080e-01 + -6.211145848662764e-01 + -6.213657689929696e-01 + -6.216135922609831e-01 + -6.218580610691606e-01 + -6.220991816188346e-01 + -6.223369599996067e-01 + -6.225714026287344e-01 + -6.228025157602012e-01 + -6.230303055354752e-01 + -6.232547783843585e-01 + -6.234759406837302e-01 + -6.236937987287140e-01 + -6.239083589372102e-01 + -6.241196278170006e-01 + -6.243276119301496e-01 + -6.245323178052680e-01 + -6.247337518435782e-01 + -6.249319204108740e-01 + -6.251268303788342e-01 + -6.253184884524475e-01 + -6.255069009766967e-01 + -6.256920746682559e-01 + -6.258740163701917e-01 + -6.260527328726909e-01 + -6.262282307801417e-01 + -6.264005167589047e-01 + -6.265695976839164e-01 + -6.267354804070123e-01 + -6.268981717782514e-01 + -6.270576786775590e-01 + -6.272140080071444e-01 + -6.273671666729116e-01 + -6.275171615794166e-01 + -6.276639997579259e-01 + -6.278076882572314e-01 + -6.279482340166599e-01 + -6.280856440870369e-01 + -6.282199255881125e-01 + -6.283510856176625e-01 + -6.284791313265480e-01 + -6.286040698559974e-01 + -6.287259082244099e-01 + -6.288446536460758e-01 + -6.289603134448759e-01 + -6.290728947784819e-01 + -6.291824049507972e-01 + -6.292888513285516e-01 + -6.293922409846477e-01 + -6.294925812787570e-01 + -6.295898797737625e-01 + -6.296841435561055e-01 + -6.297753800606519e-01 + -6.298635970427606e-01 + -6.299488015475858e-01 + -6.300310009429952e-01 + -6.301102030719377e-01 + -6.301864151575183e-01 + -6.302596446383568e-01 + -6.303298993873763e-01 + -6.303971867419816e-01 + -6.304615141097702e-01 + -6.305228892569528e-01 + -6.305813198299395e-01 + -6.306368133767261e-01 + -6.306893774095105e-01 + -6.307390196514921e-01 + -6.307857478830511e-01 + -6.308295697989852e-01 + -6.308704929504492e-01 + -6.309085250312678e-01 + -6.309436740902202e-01 + -6.309759477467396e-01 + -6.310053535533821e-01 + -6.310318994644450e-01 + -6.310555933584764e-01 + -6.310764430051494e-01 + -6.310944561130111e-01 + -6.311096405770181e-01 + -6.311220043955681e-01 + -6.311315554778594e-01 + -6.311383016234806e-01 + -6.311422506666780e-01 + -6.311434107412031e-01 + -6.311417897164122e-01 + -6.311373953205278e-01 + -6.311302357049288e-01 + -6.311203189243589e-01 + -6.311076528624781e-01 + -6.310922454230109e-01 + -6.310741047155799e-01 + -6.310532389686786e-01 + -6.310296560496979e-01 + -6.310033639455465e-01 + -6.309743708604478e-01 + -6.309426848479184e-01 + -6.309083139781310e-01 + -6.308712663782245e-01 + -6.308315500367087e-01 + -6.307891730465176e-01 + -6.307441436976022e-01 + -6.306964701860184e-01 + -6.306461606431758e-01 + -6.305932231701467e-01 + -6.305376659121696e-01 + -6.304794970062023e-01 + -6.304187245700347e-01 + -6.303553569805046e-01 + -6.302894025962669e-01 + -6.302208694979176e-01 + -6.301497657748981e-01 + -6.300760996409697e-01 + -6.299998795100692e-01 + -6.299211136514613e-01 + -6.298398102240708e-01 + -6.297559773915322e-01 + -6.296696236381925e-01 + -6.295807574013027e-01 + -6.294893864698483e-01 + -6.293955192818200e-01 + -6.292991645521353e-01 + -6.292003301703828e-01 + -6.290990243894826e-01 + -6.289952557946986e-01 + -6.288890326523825e-01 + -6.287803632353285e-01 + -6.286692558944412e-01 + -6.285557190083507e-01 + -6.284397608763040e-01 + -6.283213897612988e-01 + -6.282006141317734e-01 + -6.280774423035318e-01 + -6.279518824496017e-01 + -6.278239431715447e-01 + -6.276936329659792e-01 + -6.275609600561086e-01 + -6.274259327786519e-01 + -6.272885594754783e-01 + -6.271488484695774e-01 + -6.270068083043618e-01 + -6.268624473656752e-01 + -6.267157737459064e-01 + -6.265667960682418e-01 + -6.264155229504741e-01 + -6.262619625962428e-01 + -6.261061232036680e-01 + -6.259480131935671e-01 + -6.257876413121770e-01 + -6.256250157840939e-01 + -6.254601447340036e-01 + -6.252930366972961e-01 + -6.251237002207461e-01 + -6.249521437614608e-01 + -6.247783756272571e-01 + -6.246024042025771e-01 + -6.244242378628240e-01 + -6.242438848129136e-01 + -6.240613535991787e-01 + -6.238766528797192e-01 + -6.236897908519111e-01 + -6.235007757894835e-01 + -6.233096161276966e-01 + -6.231163203935827e-01 + -6.229208970207127e-01 + -6.227233543185018e-01 + -6.225237005131427e-01 + -6.223219439896936e-01 + -6.221180932821515e-01 + -6.219121568092019e-01 + -6.217041429518334e-01 + -6.214940600654550e-01 + -6.212819164164759e-01 + -6.210677203231969e-01 + -6.208514802076882e-01 + -6.206332045396580e-01 + -6.204129016762088e-01 + -6.201905798335827e-01 + -6.199662474146647e-01 + -6.197399127976954e-01 + -6.195115842446862e-01 + -6.192812702436963e-01 + -6.190489791767655e-01 + -6.188147191241502e-01 + -6.185784984898128e-01 + -6.183403256742428e-01 + -6.181002088023494e-01 + -6.178581563548247e-01 + -6.176141767575239e-01 + -6.173682779492388e-01 + -6.171204683326561e-01 + -6.168707564823446e-01 + -6.166191506410910e-01 + -6.163656588303806e-01 + -6.161102891648960e-01 + -6.158530502595377e-01 + -6.155939503803246e-01 + -6.153329975648972e-01 + -6.150702000825419e-01 + -6.148055662334609e-01 + -6.145391042808102e-01 + -6.142708224010276e-01 + -6.140007288452335e-01 + -6.137288318637003e-01 + -6.134551394181680e-01 + -6.131796597624336e-01 + -6.129024013716257e-01 + -6.126233721919256e-01 + -6.123425803085120e-01 + -6.120600340492245e-01 + -6.117757414641359e-01 + -6.114897106263320e-01 + -6.112019497520672e-01 + -6.109124671249474e-01 + -6.106212708396386e-01 + -6.103283687668304e-01 + -6.100337690977098e-01 + -6.097374799911853e-01 + -6.094395093910078e-01 + -6.091398653642833e-01 + -6.088385561024087e-01 + -6.085355898356886e-01 + -6.082309742898534e-01 + -6.079247173286892e-01 + -6.076168274335947e-01 + -6.073073125569597e-01 + -6.069961804304546e-01 + -6.066834389911383e-01 + -6.063690964791788e-01 + -6.060531610171422e-01 + -6.057356401425160e-01 + -6.054165419100932e-01 + -6.050958745176729e-01 + -6.047736455801236e-01 + -6.044498630713394e-01 + -6.041245351487873e-01 + -6.037976695799242e-01 + -6.034692741523835e-01 + -6.031393567367498e-01 + -6.028079252312161e-01 + -6.024749874110954e-01 + -6.021405510415967e-01 + -6.018046242357692e-01 + -6.014672148046522e-01 + -6.011283302871711e-01 + -6.007879786053585e-01 + -6.004461675660895e-01 + -6.001029047914931e-01 + -5.997581981158361e-01 + -5.994120553308485e-01 + -5.990644841299763e-01 + -5.987154923482985e-01 + -5.983650876334137e-01 + -5.980132773866208e-01 + -5.976600693793478e-01 + -5.973054714798371e-01 + -5.969494914365839e-01 + -5.965921366499265e-01 + -5.962334146660923e-01 + -5.958733334161985e-01 + -5.955119002909084e-01 + -5.951491227180785e-01 + -5.947850085981863e-01 + -5.944195652847630e-01 + -5.940528001366523e-01 + -5.936847210858313e-01 + -5.933153356463837e-01 + -5.929446511281147e-01 + -5.925726749957585e-01 + -5.921994146776245e-01 + -5.918248776317639e-01 + -5.914490714685531e-01 + -5.910720036664838e-01 + -5.906936815737919e-01 + -5.903141124838686e-01 + -5.899333037986367e-01 + -5.895512629817115e-01 + -5.891679973989596e-01 + -5.887835143677140e-01 + -5.883978211708401e-01 + -5.880109250637769e-01 + -5.876228334774148e-01 + -5.872335538851382e-01 + -5.868430932520374e-01 + -5.864514587624869e-01 + -5.860586578978172e-01 + -5.856646978273519e-01 + -5.852695857497323e-01 + -5.848733289413443e-01 + -5.844759343592291e-01 + -5.840774091773326e-01 + -5.836777608879199e-01 + -5.832769964405170e-01 + -5.828751228494087e-01 + -5.824721474461222e-01 + -5.820680771414760e-01 + -5.816629188869962e-01 + -5.812566799399100e-01 + -5.808493672421570e-01 + -5.804409877894702e-01 + -5.800315488931496e-01 + -5.796210574409676e-01 + -5.792095201878247e-01 + -5.787969440874288e-01 + -5.783833362006023e-01 + -5.779687035684036e-01 + -5.775530530727403e-01 + -5.771363914738108e-01 + -5.767187255974381e-01 + -5.763000625573806e-01 + -5.758804091701334e-01 + -5.754597721176316e-01 + -5.750381583420940e-01 + -5.746155745735496e-01 + -5.741920274506414e-01 + -5.737675239291555e-01 + -5.733420708457294e-01 + -5.729156748671344e-01 + -5.724883426083383e-01 + -5.720600807791615e-01 + -5.716308961653045e-01 + -5.712007954599810e-01 + -5.707697852850102e-01 + -5.703378722272431e-01 + -5.699050629355835e-01 + -5.694713639828742e-01 + -5.690367818720566e-01 + -5.686013232951076e-01 + -5.681649948578290e-01 + -5.677278029894381e-01 + -5.672897541841998e-01 + -5.668508549514105e-01 + -5.664111117923113e-01 + -5.659705312703284e-01 + -5.655291198708162e-01 + -5.650868839171556e-01 + -5.646438297396282e-01 + -5.641999637601537e-01 + -5.637552925169101e-01 + -5.633098223391846e-01 + -5.628635595080524e-01 + -5.624165104013904e-01 + -5.619686812129524e-01 + -5.615200782186582e-01 + -5.610707080464411e-01 + -5.606205767194389e-01 + -5.601696901727680e-01 + -5.597180550627928e-01 + -5.592656777039943e-01 + -5.588125640834879e-01 + -5.583587201544192e-01 + -5.579041522213387e-01 + -5.574488666805684e-01 + -5.569928694014434e-01 + -5.565361664370690e-01 + -5.560787640439252e-01 + -5.556206683493946e-01 + -5.551618853326125e-01 + -5.547024208857518e-01 + -5.542422809960397e-01 + -5.537814718214656e-01 + -5.533199995563840e-01 + -5.528578699065892e-01 + -5.523950887012330e-01 + -5.519316620139326e-01 + -5.514675956847120e-01 + -5.510028956792654e-01 + -5.505375681122012e-01 + -5.500716185256428e-01 + -5.496050526609592e-01 + -5.491378767346244e-01 + -5.486700964364043e-01 + -5.482017173715276e-01 + -5.477327453596050e-01 + -5.472631862648399e-01 + -5.467930458019109e-01 + -5.463223294669347e-01 + -5.458510432287255e-01 + -5.453791929736209e-01 + -5.449067840000759e-01 + -5.444338218966174e-01 + -5.439603123952597e-01 + -5.434862611282417e-01 + -5.430116738240560e-01 + -5.425365561436022e-01 + -5.420609134355956e-01 + -5.415847511352451e-01 + -5.411080748136494e-01 + -5.406308901525899e-01 + -5.401532026526268e-01 + -5.396750177177082e-01 + -5.391963408895222e-01 + -5.387171774649071e-01 + -5.382375326730170e-01 + -5.377574122460893e-01 + -5.372768215524759e-01 + -5.367957656717409e-01 + -5.363142502106338e-01 + -5.358322806031541e-01 + -5.353498619928253e-01 + -5.348669995911720e-01 + -5.343836987381033e-01 + -5.338999648334884e-01 + -5.334158029976483e-01 + -5.329312184607295e-01 + -5.324462166033185e-01 + -5.319608023124877e-01 + -5.314749807283198e-01 + -5.309887574568507e-01 + -5.305021373834015e-01 + -5.300151254489873e-01 + -5.295277270550899e-01 + -5.290399471460621e-01 + -5.285517906644440e-01 + -5.280632628772480e-01 + -5.275743687930865e-01 + -5.270851133485194e-01 + -5.265955015863814e-01 + -5.261055383771222e-01 + -5.256152286457003e-01 + -5.251245776074701e-01 + -5.246335901741112e-01 + -5.241422711040373e-01 + -5.236506252667474e-01 + -5.231586574906982e-01 + -5.226663726352896e-01 + -5.221737757253558e-01 + -5.216808715970676e-01 + -5.211876649797802e-01 + -5.206941607382560e-01 + -5.202003635069621e-01 + -5.197062778404161e-01 + -5.192119087096243e-01 + -5.187172609937223e-01 + -5.182223393579939e-01 + -5.177271482807863e-01 + -5.172316923981860e-01 + -5.167359765148800e-01 + -5.162400053033168e-01 + -5.157437833544083e-01 + -5.152473152222008e-01 + -5.147506055161414e-01 + -5.142536587718484e-01 + -5.137564794429633e-01 + -5.132590721404230e-01 + -5.127614414610688e-01 + -5.122635918987255e-01 + -5.117655278912282e-01 + -5.112672538418732e-01 + -5.107687741570007e-01 + -5.102700934318621e-01 + -5.097712161511054e-01 + -5.092721464952894e-01 + -5.087728887586368e-01 + -5.082734473837237e-01 + -5.077738269224511e-01 + -5.072740315773641e-01 + -5.067740655070005e-01 + -5.062739331645890e-01 + -5.057736389339202e-01 + -5.052731870571522e-01 + -5.047725816027976e-01 + -5.042718267556677e-01 + -5.037709267723895e-01 + -5.032698858620083e-01 + -5.027687082997551e-01 + -5.022673983228978e-01 + -5.017659599138892e-01 + -5.012643972224838e-01 + -5.007627144504877e-01 + -5.002609154317549e-01 + -4.997590043686060e-01 + -4.992569856498065e-01 + -4.987548629163329e-01 + -4.982526401066572e-01 + -4.977503215591422e-01 + -4.972479113357710e-01 + -4.967454132608579e-01 + -4.962428310876830e-01 + -4.957401690448965e-01 + -4.952374311333114e-01 + -4.947346209581790e-01 + -4.942317423902789e-01 + -4.937287995168010e-01 + -4.932257964768885e-01 + -4.927227367709692e-01 + -4.922196239985463e-01 + -4.917164622446805e-01 + -4.912132554897460e-01 + -4.907100074652099e-01 + -4.902067216434476e-01 + -4.897034018910860e-01 + -4.892000521396699e-01 + -4.886966760659724e-01 + -4.881932772071136e-01 + -4.876898592264887e-01 + -4.871864261030248e-01 + -4.866829812293292e-01 + -4.861795280147214e-01 + -4.856760706607693e-01 + -4.851726126266216e-01 + -4.846691571189501e-01 + -4.841657080652781e-01 + -4.836622690168214e-01 + -4.831588433295234e-01 + -4.826554347812969e-01 + -4.821520468729375e-01 + -4.816486828820545e-01 + -4.811453462713459e-01 + -4.806420406166251e-01 + -4.801387695127853e-01 + -4.796355363816449e-01 + -4.791323444815042e-01 + -4.786291970695780e-01 + -4.781260978727254e-01 + -4.776230503252043e-01 + -4.771200574973888e-01 + -4.766171228253449e-01 + -4.761142497486388e-01 + -4.756114415594656e-01 + -4.751087014547341e-01 + -4.746060327035595e-01 + -4.741034386828772e-01 + -4.736009226286348e-01 + -4.730984877717556e-01 + -4.725961373913972e-01 + -4.720938746127822e-01 + -4.715917026117559e-01 + -4.710896247171748e-01 + -4.705876440070008e-01 + -4.700857635192827e-01 + -4.695839864443119e-01 + -4.690823160197067e-01 + -4.685807553687465e-01 + -4.680793073815264e-01 + -4.675779752270572e-01 + -4.670767621237579e-01 + -4.665756710130068e-01 + -4.660747048828000e-01 + -4.655738667469987e-01 + -4.650731595624170e-01 + -4.645725864569897e-01 + -4.640721505529091e-01 + -4.635718546060059e-01 + -4.630717014390969e-01 + -4.625716940204364e-01 + -4.620718354412200e-01 + -4.615721285948061e-01 + -4.610725762550177e-01 + -4.605731814392849e-01 + -4.600739469669758e-01 + -4.595748755034438e-01 + -4.590759700613133e-01 + -4.585772335293120e-01 + -4.580786685902926e-01 + -4.575802780165928e-01 + -4.570820645542124e-01 + -4.565840309327809e-01 + -4.560861801183225e-01 + -4.555885148408127e-01 + -4.550910375242214e-01 + -4.545937511574892e-01 + -4.540966585092119e-01 + -4.535997618400510e-01 + -4.531030641427475e-01 + -4.526065682230286e-01 + -4.521102761991050e-01 + -4.516141909960451e-01 + -4.511183155232459e-01 + -4.506226519976986e-01 + -4.501272029129367e-01 + -4.496319709853875e-01 + -4.491369589904647e-01 + -4.486421693813144e-01 + -4.481476045149588e-01 + -4.476532669406306e-01 + -4.471591591812127e-01 + -4.466652837454304e-01 + -4.461716431800687e-01 + -4.456782399407630e-01 + -4.451850764283579e-01 + -4.446921550770006e-01 + -4.441994783152980e-01 + -4.437070485743053e-01 + -4.432148683148817e-01 + -4.427229399164072e-01 + -4.422312656936392e-01 + -4.417398479943503e-01 + -4.412486891611525e-01 + -4.407577915365379e-01 + -4.402671575060104e-01 + -4.397767893532367e-01 + -4.392866892910770e-01 + -4.387968597340107e-01 + -4.383073029681190e-01 + -4.378180210879689e-01 + -4.373290163087382e-01 + -4.368402909924953e-01 + -4.363518475687816e-01 + -4.358636880897754e-01 + -4.353758146064535e-01 + -4.348882293452391e-01 + -4.344009344375495e-01 + -4.339139321041776e-01 + -4.334272247184532e-01 + -4.329408142509990e-01 + -4.324547026649819e-01 + -4.319688922210647e-01 + -4.314833850702666e-01 + -4.309981832317592e-01 + -4.305132886299618e-01 + -4.300287034407881e-01 + -4.295444298858639e-01 + -4.290604699557538e-01 + -4.285768254794737e-01 + -4.280934983815344e-01 + -4.276104909880938e-01 + -4.271278053251724e-01 + -4.266454431999359e-01 + -4.261634065006114e-01 + -4.256816972235624e-01 + -4.252003173934318e-01 + -4.247192689049444e-01 + -4.242385536499166e-01 + -4.237581735472888e-01 + -4.232781305399061e-01 + -4.227984264773803e-01 + -4.223190631457318e-01 + -4.218400424415704e-01 + -4.213613663598799e-01 + -4.208830368540804e-01 + -4.204050553934426e-01 + -4.199274237890383e-01 + -4.194501442515862e-01 + -4.189732183034799e-01 + -4.184966476302857e-01 + -4.180204343198091e-01 + -4.175445800025869e-01 + -4.170690862458790e-01 + -4.165939547960728e-01 + -4.161191875920802e-01 + -4.156447863485650e-01 + -4.151707524172177e-01 + -4.146970878143395e-01 + -4.142237944432959e-01 + -4.137508735370036e-01 + -4.132783267358912e-01 + -4.128061559031381e-01 + -4.123343628419361e-01 + -4.118629489726368e-01 + -4.113919157251955e-01 + -4.109212649651476e-01 + -4.104509983224866e-01 + -4.099811172669397e-01 + -4.095116233202941e-01 + -4.090425181490535e-01 + -4.085738034100841e-01 + -4.081054804950061e-01 + -4.076375508831032e-01 + -4.071700161671795e-01 + -4.067028779692610e-01 + -4.062361376606066e-01 + -4.057697965353959e-01 + -4.053038563335837e-01 + -4.048383186645677e-01 + -4.043731849141807e-01 + -4.039084564103662e-01 + -4.034441345521896e-01 + -4.029802208230617e-01 + -4.025167167043049e-01 + -4.020536236099443e-01 + -4.015909428779320e-01 + -4.011286758375757e-01 + -4.006668238996902e-01 + -4.002053885607212e-01 + -3.997443712200008e-01 + -3.992837731698375e-01 + -3.988235956310492e-01 + -3.983638399923307e-01 + -3.979045076012857e-01 + -3.974455996644362e-01 + -3.969871175556028e-01 + -3.965290626455606e-01 + -3.960714361699136e-01 + -3.956142393772600e-01 + -3.951574735515524e-01 + -3.947011399985758e-01 + -3.942452398354794e-01 + -3.937897742524872e-01 + -3.933347447660869e-01 + -3.928801524442024e-01 + -3.924259983101159e-01 + -3.919722839283850e-01 + -3.915190103656777e-01 + -3.910661784756895e-01 + -3.906137895655892e-01 + -3.901618450499643e-01 + -3.897103461604648e-01 + -3.892592935529919e-01 + -3.888086884121152e-01 + -3.883585322771145e-01 + -3.879088261485191e-01 + -3.874595710701922e-01 + -3.870107681556456e-01 + -3.865624182556531e-01 + -3.861145225228043e-01 + -3.856670823544472e-01 + -3.852200986473037e-01 + -3.847735723987576e-01 + -3.843275047747315e-01 + -3.838818965026468e-01 + -3.834367486164244e-01 + -3.829920625791138e-01 + -3.825478392535004e-01 + -3.821040794322394e-01 + -3.816607841213338e-01 + -3.812179544462010e-01 + -3.807755914230820e-01 + -3.803336958584798e-01 + -3.798922686597118e-01 + -3.794513108818827e-01 + -3.790108236907996e-01 + -3.785708078449803e-01 + -3.781312640889848e-01 + -3.776921934895628e-01 + -3.772535969260590e-01 + -3.768154752452403e-01 + -3.763778294637203e-01 + -3.759406605249527e-01 + -3.755039692439986e-01 + -3.750677562951532e-01 + -3.746320226741715e-01 + -3.741967693787305e-01 + -3.737619968727901e-01 + -3.733277061470673e-01 + -3.728938983949728e-01 + -3.724605741247380e-01 + -3.720277341192338e-01 + -3.715953793429491e-01 + -3.711635103186269e-01 + -3.707321278978636e-01 + -3.703012331728173e-01 + -3.698708267439747e-01 + -3.694409092657609e-01 + -3.690114815572269e-01 + -3.685825444178367e-01 + -3.681540985405766e-01 + -3.677261445488677e-01 + -3.672986832474148e-01 + -3.668717154245260e-01 + -3.664452417818624e-01 + -3.660192630178876e-01 + -3.655937798167319e-01 + -3.651687928442244e-01 + -3.647443027770453e-01 + -3.643203103057334e-01 + -3.638968161236534e-01 + -3.634738208501973e-01 + -3.630513251204181e-01 + -3.626293296340570e-01 + -3.622078349959966e-01 + -3.617868418102546e-01 + -3.613663507559756e-01 + -3.609463624258993e-01 + -3.605268773947004e-01 + -3.601078962972172e-01 + -3.596894197319259e-01 + -3.592714482724653e-01 + -3.588539824938156e-01 + -3.584370229705016e-01 + -3.580205702765974e-01 + -3.576046249835235e-01 + -3.571891876270012e-01 + -3.567742587321382e-01 + -3.563598388637925e-01 + -3.559459285557617e-01 + -3.555325283179373e-01 + -3.551196386794110e-01 + -3.547072601423791e-01 + -3.542953931967586e-01 + -3.538840383835105e-01 + -3.534731961849680e-01 + -3.530628670336623e-01 + -3.526530514374770e-01 + -3.522437498876279e-01 + -3.518349628366106e-01 + -3.514266907403692e-01 + -3.510189340474827e-01 + -3.506116931966672e-01 + -3.502049686305037e-01 + -3.497987607886671e-01 + -3.493930701015490e-01 + -3.489878969787067e-01 + -3.485832418243141e-01 + -3.481791050461803e-01 + -3.477754870557839e-01 + -3.473723882547367e-01 + -3.469698090253131e-01 + -3.465677497415546e-01 + -3.461662107762984e-01 + -3.457651925045610e-01 + -3.453646952920019e-01 + -3.449647194961470e-01 + -3.445652654674211e-01 + -3.441663335489293e-01 + -3.437679240760669e-01 + -3.433700373751777e-01 + -3.429726737709668e-01 + -3.425758335909973e-01 + -3.421795171685511e-01 + -3.417837248082679e-01 + -3.413884567949557e-01 + -3.409937134173290e-01 + -3.405994949798623e-01 + -3.402058017865278e-01 + -3.398126341027504e-01 + -3.394199921921239e-01 + -3.390278763247643e-01 + -3.386362867748747e-01 + -3.382452237948665e-01 + -3.378546876227794e-01 + -3.374646785249709e-01 + -3.370751967392704e-01 + -3.366862424749215e-01 + -3.362978159833941e-01 + -3.359099174981644e-01 + -3.355225472182573e-01 + -3.351357053498810e-01 + -3.347493921057998e-01 + -3.343636076973308e-01 + -3.339783523006465e-01 + -3.335936261089775e-01 + -3.332094293460974e-01 + -3.328257621547543e-01 + -3.324426246881435e-01 + -3.320600171619409e-01 + -3.316779397280902e-01 + -3.312963925265998e-01 + -3.309153757279716e-01 + -3.305348894653861e-01 + -3.301549338737799e-01 + -3.297755091265357e-01 + -3.293966153413448e-01 + -3.290182526215280e-01 + -3.286404211114115e-01 + -3.282631209285871e-01 + -3.278863521751846e-01 + -3.275101149642331e-01 + -3.271344093953196e-01 + -3.267592355617253e-01 + -3.263845935667566e-01 + -3.260104834866909e-01 + -3.256369053882101e-01 + -3.252638593756906e-01 + -3.248913455268053e-01 + -3.245193638907094e-01 + -3.241479145199382e-01 + -3.237769974872735e-01 + -3.234066128661293e-01 + -3.230367606622389e-01 + -3.226674409186036e-01 + -3.222986537192250e-01 + -3.219303990783447e-01 + -3.215626770146125e-01 + -3.211954875680534e-01 + -3.208288307426270e-01 + -3.204627065455645e-01 + -3.200971150001409e-01 + -3.197320561103191e-01 + -3.193675298795836e-01 + -3.190035363166080e-01 + -3.186400754009213e-01 + -3.182771471114187e-01 + -3.179147514432106e-01 + -3.175528883799994e-01 + -3.171915578921511e-01 + -3.168307599394173e-01 + -3.164704945026028e-01 + -3.161107615528752e-01 + -3.157515610223091e-01 + -3.153928928534279e-01 + -3.150347570035595e-01 + -3.146771534412893e-01 + -3.143200821026652e-01 + -3.139635429048980e-01 + -3.136075357713601e-01 + -3.132520606345795e-01 + -3.128971174301287e-01 + -3.125427060850263e-01 + -3.121888265003789e-01 + -3.118354785699559e-01 + -3.114826622209773e-01 + -3.111303773636806e-01 + -3.107786238863177e-01 + -3.104274016699458e-01 + -3.100767106006651e-01 + -3.097265505731036e-01 + -3.093769214901858e-01 + -3.090278232197667e-01 + -3.086792556057311e-01 + -3.083312185528201e-01 + -3.079837119302323e-01 + -3.076367355606988e-01 + -3.072902893338402e-01 + -3.069443731087555e-01 + -3.065989866918071e-01 + -3.062541299629242e-01 + -3.059098027895394e-01 + -3.055660049841727e-01 + -3.052227363624680e-01 + -3.048799967605036e-01 + -3.045377860349815e-01 + -3.041961039937703e-01 + -3.038549504461246e-01 + -3.035143252370935e-01 + -3.031742281687436e-01 + -3.028346590287212e-01 + -3.024956176228422e-01 + -3.021571037727226e-01 + -3.018191172891879e-01 + -3.014816579447884e-01 + -3.011447255196590e-01 + -3.008083198102758e-01 + -3.004724406299011e-01 + -3.001370877400161e-01 + -2.998022608844934e-01 + -2.994679598576119e-01 + -2.991341844426006e-01 + -2.988009344054493e-01 + -2.984682095047237e-01 + -2.981360094999961e-01 + -2.978043341494513e-01 + -2.974731831997393e-01 + -2.971425563888731e-01 + -2.968124534589884e-01 + -2.964828741880918e-01 + -2.961538183196636e-01 + -2.958252855581221e-01 + -2.954972756291568e-01 + -2.951697882690829e-01 + -2.948428232168150e-01 + -2.945163801997789e-01 + -2.941904589349781e-01 + -2.938650591314966e-01 + -2.935401805013078e-01 + -2.932158227514122e-01 + -2.928919855851135e-01 + -2.925686687341852e-01 + -2.922458719001491e-01 + -2.919235947316832e-01 + -2.916018369508684e-01 + -2.912805982685433e-01 + -2.909598783275820e-01 + -2.906396768390501e-01 + -2.903199935181080e-01 + -2.900008280188033e-01 + -2.896821799916231e-01 + -2.893640491090957e-01 + -2.890464350819901e-01 + -2.887293375718574e-01 + -2.884127562142981e-01 + -2.880966906629407e-01 + -2.877811406014185e-01 + -2.874661057081589e-01 + -2.871515856017081e-01 + -2.868375799224188e-01 + -2.865240883310016e-01 + -2.862111104788733e-01 + -2.858986460098822e-01 + -2.855866945593247e-01 + -2.852752557499685e-01 + -2.849643292069009e-01 + -2.846539145659956e-01 + -2.843440114851778e-01 + -2.840346195818731e-01 + -2.837257384382189e-01 + -2.834173676812886e-01 + -2.831095069534547e-01 + -2.828021558855677e-01 + -2.824953140398479e-01 + -2.821889810170294e-01 + -2.818831564761255e-01 + -2.815778399984394e-01 + -2.812730311718642e-01 + -2.809687296216254e-01 + -2.806649349155874e-01 + -2.803616466281044e-01 + -2.800588643758425e-01 + -2.797565877594176e-01 + -2.794548163592918e-01 + -2.791535497376408e-01 + -2.788527874670974e-01 + -2.785525291296920e-01 + -2.782527743138438e-01 + -2.779535226034689e-01 + -2.776547735631448e-01 + -2.773565267276963e-01 + -2.770587816777879e-01 + -2.767615379983304e-01 + -2.764647952191276e-01 + -2.761685529026718e-01 + -2.758728106207560e-01 + -2.755775679024411e-01 + -2.752828243050798e-01 + -2.749885793975552e-01 + -2.746948327088052e-01 + -2.744015837838452e-01 + -2.741088321749212e-01 + -2.738165773969221e-01 + -2.735248189792139e-01 + -2.732335564721672e-01 + -2.729427894286903e-01 + -2.726525173657923e-01 + -2.723627397775934e-01 + -2.720734562117226e-01 + -2.717846661993187e-01 + -2.714963692431568e-01 + -2.712085648712142e-01 + -2.709212526021956e-01 + -2.706344319386959e-01 + -2.703481024077038e-01 + -2.700622635195931e-01 + -2.697769147573882e-01 + -2.694920556414950e-01 + -2.692076856844960e-01 + -2.689238043678598e-01 + -2.686404111894522e-01 + -2.683575056539274e-01 + -2.680750872618156e-01 + -2.677931554956217e-01 + -2.675117098420171e-01 + -2.672307498062658e-01 + -2.669502748541435e-01 + -2.666702844564393e-01 + -2.663907781339158e-01 + -2.661117553567487e-01 + -2.658332155842312e-01 + -2.655551583220328e-01 + -2.652775830228958e-01 + -2.650004891310402e-01 + -2.647238761635896e-01 + -2.644477435849875e-01 + -2.641720908275026e-01 + -2.638969173638899e-01 + -2.636222226679477e-01 + -2.633480062028563e-01 + -2.630742674151484e-01 + -2.628010057675256e-01 + -2.625282207270022e-01 + -2.622559117157733e-01 + -2.619840781930577e-01 + -2.617127196438671e-01 + -2.614418354712815e-01 + -2.611714251171109e-01 + -2.609014880694051e-01 + -2.606320237300626e-01 + -2.603630315312080e-01 + -2.600945109576975e-01 + -2.598264614192900e-01 + -2.595588823272051e-01 + -2.592917731268140e-01 + -2.590251332515323e-01 + -2.587589621335352e-01 + -2.584932592080277e-01 + -2.582280238899204e-01 + -2.579632555996659e-01 + -2.576989537753064e-01 + -2.574351178135512e-01 + -2.571717471205571e-01 + -2.569088411498421e-01 + -2.566463993032694e-01 + -2.563844209736399e-01 + -2.561229055922761e-01 + -2.558618525664796e-01 + -2.556012613001373e-01 + -2.553411312228936e-01 + -2.550814617199114e-01 + -2.548222521633088e-01 + -2.545635019702024e-01 + -2.543052105636615e-01 + -2.540473773489130e-01 + -2.537900016856341e-01 + -2.535330829668063e-01 + -2.532766206092137e-01 + -2.530206140025920e-01 + -2.527650625373169e-01 + -2.525099656058339e-01 + -2.522553225882911e-01 + -2.520011328519368e-01 + -2.517473957689335e-01 + -2.514941107682424e-01 + -2.512412772287477e-01 + -2.509888944763962e-01 + -2.507369619085763e-01 + -2.504854789235316e-01 + -2.502344448942996e-01 + -2.499838591881081e-01 + -2.497337211675947e-01 + -2.494840301941274e-01 + -2.492347856449565e-01 + -2.489859868896926e-01 + -2.487376332816026e-01 + -2.484897241934407e-01 + -2.482422589923619e-01 + -2.479952370263670e-01 + -2.477486576676466e-01 + -2.475025202765806e-01 + -2.472568241776227e-01 + -2.470115687463755e-01 + -2.467667533575219e-01 + -2.465223773319763e-01 + -2.462784400223504e-01 + -2.460349407926405e-01 + -2.457918789841980e-01 + -2.455492539346369e-01 + -2.453070649885519e-01 + -2.450653115068263e-01 + -2.448239928212929e-01 + -2.445831082525867e-01 + -2.443426571517872e-01 + -2.441026388729581e-01 + -2.438630527595843e-01 + -2.436238981283992e-01 + -2.433851743000419e-01 + -2.431468806082007e-01 + -2.429090164035606e-01 + -2.426715810065056e-01 + -2.424345737214870e-01 + -2.421979939156839e-01 + -2.419618409122474e-01 + -2.417261139893861e-01 + -2.414908125047008e-01 + -2.412559357926221e-01 + -2.410214831428485e-01 + -2.407874538901436e-01 + -2.405538473594189e-01 + -2.403206628448788e-01 + -2.400878996494245e-01 + -2.398555571012901e-01 + -2.396236345489386e-01 + -2.393921312730178e-01 + -2.391610465589564e-01 + -2.389303797400675e-01 + -2.387001301206918e-01 + -2.384702970040557e-01 + -2.382408797142012e-01 + -2.380118775445049e-01 + -2.377832897784107e-01 + -2.375551157141706e-01 + -2.373273546676602e-01 + -2.371000059435911e-01 + -2.368730688080822e-01 + -2.366465425799626e-01 + -2.364204265795865e-01 + -2.361947200488833e-01 + -2.359694222928865e-01 + -2.357445326362645e-01 + -2.355200503199650e-01 + -2.352959746331166e-01 + -2.350723049000333e-01 + -2.348490404054174e-01 + -2.346261804119323e-01 + -2.344037241869703e-01 + -2.341816710532240e-01 + -2.339600202844186e-01 + -2.337387711161095e-01 + -2.335179228570654e-01 + -2.332974748051559e-01 + -2.330774262250223e-01 + -2.328577763695832e-01 + -2.326385245095027e-01 + -2.324196699339893e-01 + -2.322012119071908e-01 + -2.319831496985562e-01 + -2.317654825913850e-01 + -2.315482098501277e-01 + -2.313313307350805e-01 + -2.311148445095759e-01 + -2.308987504304352e-01 + -2.306830477605995e-01 + -2.304677357746768e-01 + -2.302528137313455e-01 + -2.300382808821649e-01 + -2.298241364802842e-01 + -2.296103797849810e-01 + -2.293970100645507e-01 + -2.291840265929282e-01 + -2.289714285853380e-01 + -2.287592152687787e-01 + -2.285473859575702e-01 + -2.283359398767679e-01 + -2.281248762354934e-01 + -2.279141943457151e-01 + -2.277038934347241e-01 + -2.274939727028900e-01 + -2.272844314557521e-01 + -2.270752689260884e-01 + -2.268664843063303e-01 + -2.266580768713468e-01 + -2.264500458690802e-01 + -2.262423905203003e-01 + -2.260351100742427e-01 + -2.258282037663114e-01 + -2.256216708189812e-01 + -2.254155104759721e-01 + -2.252097219699820e-01 + -2.250043045201664e-01 + -2.247992573593203e-01 + -2.245945797283117e-01 + -2.243902708632074e-01 + -2.241863299547213e-01 + -2.239827562297785e-01 + -2.237795489631809e-01 + -2.235767073479459e-01 + -2.233742305913306e-01 + -2.231721179508258e-01 + -2.229703686190215e-01 + -2.227689817967141e-01 + -2.225679567331226e-01 + -2.223672926389324e-01 + -2.221669887137792e-01 + -2.219670441735496e-01 + -2.217674582593449e-01 + -2.215682301863681e-01 + -2.213693591068690e-01 + -2.211708442449674e-01 + -2.209726848493868e-01 + -2.207748801280919e-01 + -2.205774292618204e-01 + -2.203803314334422e-01 + -2.201835858632719e-01 + -2.199871917828554e-01 + -2.197911484006530e-01 + -2.195954548584606e-01 + -2.194001103587555e-01 + -2.192051141421317e-01 + -2.190104654040938e-01 + -2.188161633272981e-01 + -2.186222070944807e-01 + -2.184285959003400e-01 + -2.182353289399314e-01 + -2.180424054040691e-01 + -2.178498244760091e-01 + -2.176575853577229e-01 + -2.174656872563868e-01 + -2.172741293153170e-01 + -2.170829107184905e-01 + -2.168920306960583e-01 + -2.167014883907840e-01 + -2.165112829716549e-01 + -2.163214136638050e-01 + -2.161318796380734e-01 + -2.159426800456078e-01 + -2.157538140457635e-01 + -2.155652808465100e-01 + -2.153770796378261e-01 + -2.151892095617934e-01 + -2.150016698096599e-01 + -2.148144595574814e-01 + -2.146275779264766e-01 + -2.144410241083473e-01 + -2.142547972973645e-01 + -2.140688966286696e-01 + -2.138833212713278e-01 + -2.136980704068839e-01 + -2.135131431962469e-01 + -2.133285387811797e-01 + -2.131442563097479e-01 + -2.129602949644751e-01 + -2.127766538888946e-01 + -2.125933322205115e-01 + -2.124103291538361e-01 + -2.122276438425608e-01 + -2.120452754131807e-01 + -2.118632230196053e-01 + -2.116814858091140e-01 + -2.115000629262062e-01 + -2.113189535366684e-01 + -2.111381567835468e-01 + -2.109576717897184e-01 + -2.107774976942802e-01 + -2.105976336477213e-01 + -2.104180788038142e-01 + -2.102388322993181e-01 + -2.100598932684242e-01 + -2.098812608470688e-01 + -2.097029341689293e-01 + -2.095249123662445e-01 + -2.093471945702010e-01 + -2.091697799107427e-01 + -2.089926675162990e-01 + -2.088158565139439e-01 + -2.086393460309292e-01 + -2.084631351904399e-01 + -2.082872231124364e-01 + -2.081116089361562e-01 + -2.079362917906518e-01 + -2.077612707758083e-01 + -2.075865450013143e-01 + -2.074121135936494e-01 + -2.072379756933602e-01 + -2.070641303864229e-01 + -2.068905767613353e-01 + -2.067173139701657e-01 + -2.065443411403108e-01 + -2.063716573678470e-01 + -2.061992617242399e-01 + -2.060271533191409e-01 + -2.058553312792487e-01 + -2.056837947057731e-01 + -2.055125426973447e-01 + -2.053415743575506e-01 + -2.051708887985432e-01 + -2.050004850901530e-01 + -2.048303622987561e-01 + -2.046605195888223e-01 + -2.044909560501713e-01 + -2.043216707137636e-01 + -2.041526627003059e-01 + -2.039839311053935e-01 + -2.038154749924674e-01 + -2.036472934779214e-01 + -2.034793856435669e-01 + -2.033117505334730e-01 + -2.031443872603400e-01 + -2.029772949053144e-01 + -2.028104725040109e-01 + -2.026439191805672e-01 + -2.024776340229360e-01 + -2.023116160442759e-01 + -2.021458643362796e-01 + -2.019803779931268e-01 + -2.018151560674621e-01 + -2.016501976152565e-01 + -2.014855017030598e-01 + -2.013210674074579e-01 + -2.011568937830132e-01 + -2.009929798869589e-01 + -2.008293247967324e-01 + -2.006659275530022e-01 + -2.005027871933542e-01 + -2.003399027889975e-01 + -2.001772733993469e-01 + -2.000148980648089e-01 + -1.998527758076494e-01 + -1.996909056937942e-01 + -1.995292867991256e-01 + -1.993679181495860e-01 + -1.992067987609143e-01 + -1.990459276660739e-01 + -1.988853039482369e-01 + -1.987249266385540e-01 + -1.985647947339026e-01 + -1.984049072747179e-01 + -1.982452632992336e-01 + -1.980858618372365e-01 + -1.979267019229292e-01 + -1.977677825816225e-01 + -1.976091028269504e-01 + -1.974506616652059e-01 + -1.972924581269546e-01 + -1.971344912595772e-01 + -1.969767600601777e-01 + -1.968192635197257e-01 + -1.966620006473419e-01 + -1.965049704914111e-01 + -1.963481720619957e-01 + -1.961916043159508e-01 + -1.960352662866058e-01 + -1.958791570023236e-01 + -1.957232754408347e-01 + -1.955676205736371e-01 + -1.954121914050787e-01 + -1.952569869811404e-01 + -1.951020062511957e-01 + -1.949472481743376e-01 + -1.947927118000400e-01 + -1.946383960967118e-01 + -1.944843000224015e-01 + -1.943304226010136e-01 + -1.941767627996486e-01 + -1.940233195719773e-01 + -1.938700919198527e-01 + -1.937170788011597e-01 + -1.935642791691659e-01 + -1.934116920404960e-01 + -1.932593163702813e-01 + -1.931071510855982e-01 + -1.929551951739442e-01 + -1.928034476111422e-01 + -1.926519073523142e-01 + -1.925005733427448e-01 + -1.923494445333268e-01 + -1.921985198850023e-01 + -1.920477983694047e-01 + -1.918972789099489e-01 + -1.917469604078139e-01 + -1.915968418647559e-01 + -1.914469222323946e-01 + -1.912972003977130e-01 + -1.911476753153419e-01 + -1.909983459354311e-01 + -1.908492111828614e-01 + -1.907002699990133e-01 + -1.905515213021572e-01 + -1.904029639829450e-01 + -1.902545969863778e-01 + -1.901064192518998e-01 + -1.899584296832387e-01 + -1.898106271909556e-01 + -1.896630106872466e-01 + -1.895155790805884e-01 + -1.893683312769490e-01 + -1.892212661839438e-01 + -1.890743827110565e-01 + -1.889276797429612e-01 + -1.887811561730439e-01 + -1.886348109311961e-01 + -1.884886428944042e-01 + -1.883426509295119e-01 + -1.881968339481026e-01 + -1.880511908244114e-01 + -1.879057204210790e-01 + -1.877604216379617e-01 + -1.876152933660684e-01 + -1.874703344759240e-01 + -1.873255438107255e-01 + -1.871809202288037e-01 + -1.870364626111082e-01 + -1.868921698587295e-01 + -1.867480408184902e-01 + -1.866040743068453e-01 + -1.864602692045282e-01 + -1.863166243684211e-01 + -1.861731386269271e-01 + -1.860298108368848e-01 + -1.858866398430710e-01 + -1.857436244746902e-01 + -1.856007635808946e-01 + -1.854580560069532e-01 + -1.853155005825300e-01 + -1.851730961153894e-01 + -1.850308414342497e-01 + -1.848887353926573e-01 + -1.847467767859671e-01 + -1.846049644137440e-01 + -1.844632971091011e-01 + -1.843217736893654e-01 + -1.841803929497542e-01 + -1.840391536710132e-01 + -1.838980546953601e-01 + -1.837570948428414e-01 + -1.836162728527662e-01 + -1.834755875142904e-01 + -1.833350376404768e-01 + -1.831946220322505e-01 + -1.830543394615971e-01 + -1.829141886951903e-01 + -1.827741685199023e-01 + -1.826342777082210e-01 + -1.824945150204152e-01 + -1.823548792146146e-01 + -1.822153690716325e-01 + -1.820759833652730e-01 + -1.819367208114371e-01 + -1.817975801678411e-01 + -1.816585602158104e-01 + -1.815196596908165e-01 + -1.813808773333636e-01 + -1.812422118861797e-01 + -1.811036620617022e-01 + -1.809652266051679e-01 + -1.808269042761388e-01 + -1.806886937510042e-01 + -1.805505937426575e-01 + -1.804126030135119e-01 + -1.802747202867512e-01 + -1.801369442562599e-01 + -1.799992736029428e-01 + -1.798617070428147e-01 + -1.797242432859927e-01 + -1.795868810229852e-01 + -1.794496189531687e-01 + -1.793124557648680e-01 + -1.791753901302353e-01 + -1.790384207459855e-01 + -1.789015462957270e-01 + -1.787647654308534e-01 + -1.786280768266134e-01 + -1.784914791546937e-01 + -1.783549710639760e-01 + -1.782185512276328e-01 + -1.780822183108111e-01 + -1.779459709418989e-01 + -1.778098077766785e-01 + -1.776737274641330e-01 + -1.775377286080068e-01 + -1.774018098685905e-01 + -1.772659699092733e-01 + -1.771302073149079e-01 + -1.769945207098710e-01 + -1.768589087303928e-01 + -1.767233699571559e-01 + -1.765879030229077e-01 + -1.764525065751527e-01 + -1.763171791703831e-01 + -1.761819193770276e-01 + -1.760467257991546e-01 + -1.759115970764966e-01 + -1.757765317826701e-01 + -1.756415284467932e-01 + -1.755065856823099e-01 + -1.753717020824757e-01 + -1.752368761936218e-01 + -1.751021065398607e-01 + -1.749673916979465e-01 + -1.748327302843573e-01 + -1.746981207950505e-01 + -1.745635617515470e-01 + -1.744290517410085e-01 + -1.742945892830799e-01 + -1.741601728915786e-01 + -1.740258011048612e-01 + -1.738914724532109e-01 + -1.737571854631234e-01 + -1.736229386541751e-01 + -1.734887304969607e-01 + -1.733545594900176e-01 + -1.732204242005106e-01 + -1.730863230847530e-01 + -1.729522545880760e-01 + -1.728182172369972e-01 + -1.726842095169024e-01 + -1.725502298924380e-01 + -1.724162768405715e-01 + -1.722823488297978e-01 + -1.721484443102564e-01 + -1.720145617082366e-01 + -1.718806995062888e-01 + -1.717468561876117e-01 + -1.716130301380271e-01 + -1.714792197950507e-01 + -1.713454236195253e-01 + -1.712116399980747e-01 + -1.710778673588311e-01 + -1.709441041583024e-01 + -1.708103487974718e-01 + -1.706765996579921e-01 + -1.705428551238503e-01 + -1.704091136122197e-01 + -1.702753735243250e-01 + -1.701416332337174e-01 + -1.700078910979654e-01 + -1.698741454939726e-01 + -1.697403948188084e-01 + -1.696066374424440e-01 + -1.694728717024124e-01 + -1.693390959182572e-01 + -1.692053084687103e-01 + -1.690715077127793e-01 + -1.689376919605925e-01 + -1.688038595533757e-01 + -1.686700088249354e-01 + -1.685361380772680e-01 + -1.684022456043037e-01 + -1.682683297240474e-01 + -1.681343887869522e-01 + -1.680004210524648e-01 + -1.678664247764668e-01 + -1.677323982879569e-01 + -1.675983398888628e-01 + -1.674642478421112e-01 + -1.673301203727337e-01 + -1.671959557470875e-01 + -1.670617522514179e-01 + -1.669275081523977e-01 + -1.667932216748827e-01 + -1.666588910302535e-01 + -1.665245144618999e-01 + -1.663900902261115e-01 + -1.662556165573334e-01 + -1.661210916150787e-01 + -1.659865136192500e-01 + -1.658518808205523e-01 + -1.657171913793817e-01 + -1.655824434869752e-01 + -1.654476353589082e-01 + -1.653127651402884e-01 + -1.651778310039089e-01 + -1.650428311475997e-01 + -1.649077636947325e-01 + -1.647726268054878e-01 + -1.646374186795111e-01 + -1.645021374143022e-01 + -1.643667811219276e-01 + -1.642313479645375e-01 + -1.640958360797542e-01 + -1.639602435821208e-01 + -1.638245685678920e-01 + -1.636888091342957e-01 + -1.635529633807896e-01 + -1.634170294056715e-01 + -1.632810052843662e-01 + -1.631448890987297e-01 + -1.630086789505001e-01 + -1.628723728914803e-01 + -1.627359689604526e-01 + -1.625994652175857e-01 + -1.624628597296842e-01 + -1.623261505411777e-01 + -1.621893356498993e-01 + -1.620524130844338e-01 + -1.619153808874262e-01 + -1.617782370829140e-01 + -1.616409796594323e-01 + -1.615036066042981e-01 + -1.613661159511956e-01 + -1.612285056749289e-01 + -1.610907737177844e-01 + -1.609529180644884e-01 + -1.608149367058714e-01 + -1.606768276166446e-01 + -1.605385887249777e-01 + -1.604002179666082e-01 + -1.602617132920370e-01 + -1.601230726507729e-01 + -1.599842939686116e-01 + -1.598453751511857e-01 + -1.597063141136520e-01 + -1.595671087574746e-01 + -1.594277569704320e-01 + -1.592882566576958e-01 + -1.591486057239582e-01 + -1.590088020577491e-01 + -1.588688434990599e-01 + -1.587287278927537e-01 + -1.585884531125515e-01 + -1.584480170419727e-01 + -1.583074175205078e-01 + -1.581666523320080e-01 + -1.580257193426469e-01 + -1.578846164090064e-01 + -1.577433413163997e-01 + -1.576018918624315e-01 + -1.574602658488709e-01 + -1.573184610699215e-01 + -1.571764753211041e-01 + -1.570343063929105e-01 + -1.568919520623428e-01 + -1.567494100904384e-01 + -1.566066782348808e-01 + -1.564637542612438e-01 + -1.563206358983480e-01 + -1.561773208679784e-01 + -1.560338069324230e-01 + -1.558900918260982e-01 + -1.557461732592160e-01 + -1.556020489404925e-01 + -1.554577165574562e-01 + -1.553131737930803e-01 + -1.551684183667380e-01 + -1.550234479797016e-01 + -1.548782603038531e-01 + -1.547328529851514e-01 + -1.545872236759721e-01 + -1.544413700402710e-01 + -1.542952897416497e-01 + -1.541489804232354e-01 + -1.540024397056934e-01 + -1.538556652055174e-01 + -1.537086545589509e-01 + -1.535614054107616e-01 + -1.534139153265976e-01 + -1.532661818894184e-01 + -1.531182027303390e-01 + -1.529699754364569e-01 + -1.528214975805129e-01 + -1.526727667337492e-01 + -1.525237804315604e-01 + -1.523745362382714e-01 + -1.522250317710374e-01 + -1.520752645246755e-01 + -1.519252320010430e-01 + -1.517749318043420e-01 + -1.516243614355856e-01 + -1.514735183783821e-01 + -1.513224001894709e-01 + -1.511710043669709e-01 + -1.510193283881065e-01 + -1.508673697644896e-01 + -1.507151259726486e-01 + -1.505625944726656e-01 + -1.504097727456887e-01 + -1.502566582737337e-01 + -1.501032485283334e-01 + -1.499495409558064e-01 + -1.497955329817901e-01 + -1.496412220312461e-01 + -1.494866055649385e-01 + -1.493316810218445e-01 + -1.491764458131949e-01 + -1.490208973395277e-01 + -1.488650330073115e-01 + -1.487088502321930e-01 + -1.485523464347014e-01 + -1.483955190026080e-01 + -1.482383652905619e-01 + -1.480808826638546e-01 + -1.479230685198502e-01 + -1.477649202713839e-01 + -1.476064352431592e-01 + -1.474476107543291e-01 + -1.472884441602761e-01 + -1.471289328556787e-01 + -1.469690741904771e-01 + -1.468088654425319e-01 + -1.466483039513407e-01 + -1.464873870643751e-01 + -1.463261120992439e-01 + -1.461644763685366e-01 + -1.460024771711030e-01 + -1.458401117873900e-01 + -1.456773775256328e-01 + -1.455142717039091e-01 + -1.453507916253095e-01 + -1.451869345373153e-01 + -1.450226976893540e-01 + -1.448580783981553e-01 + -1.446930739597145e-01 + -1.445276816274984e-01 + -1.443618986028185e-01 + -1.441957221635531e-01 + -1.440291496154044e-01 + -1.438621781832398e-01 + -1.436948050802729e-01 + -1.435270275400491e-01 + -1.433588428498931e-01 + -1.431902482406432e-01 + -1.430212409022274e-01 + -1.428518180756519e-01 + -1.426819769869911e-01 + -1.425117148424586e-01 + -1.423410288662751e-01 + -1.421699162875700e-01 + -1.419983743276740e-01 + -1.418264001727286e-01 + -1.416539910114460e-01 + -1.414811440470098e-01 + -1.413078564852965e-01 + -1.411341255423312e-01 + -1.409599484327010e-01 + -1.407853222965617e-01 + -1.406102443173973e-01 + -1.404347117561954e-01 + -1.402587217519092e-01 + -1.400822714659246e-01 + -1.399053581557436e-01 + -1.397279789611117e-01 + -1.395501310267636e-01 + -1.393718115937786e-01 + -1.391930178402762e-01 + -1.390137469173205e-01 + -1.388339959972602e-01 + -1.386537622953096e-01 + -1.384730430031857e-01 + -1.382918352262208e-01 + -1.381101361784074e-01 + -1.379279430940128e-01 + -1.377452530885734e-01 + -1.375620633453186e-01 + -1.373783710804598e-01 + -1.371941734475252e-01 + -1.370094676273606e-01 + -1.368242508215203e-01 + -1.366385202106050e-01 + -1.364522729969404e-01 + -1.362655063841710e-01 + -1.360782175127518e-01 + -1.358904035964053e-01 + -1.357020618930480e-01 + -1.355131895302153e-01 + -1.353237837054462e-01 + -1.351338416922692e-01 + -1.349433606493001e-01 + -1.347523377766847e-01 + -1.345607703467432e-01 + -1.343686555800670e-01 + -1.341759906744941e-01 + -1.339827728348106e-01 + -1.337889993407819e-01 + -1.335946674413715e-01 + -1.333997743261954e-01 + -1.332043172962129e-01 + -1.330082936252972e-01 + -1.328117004983506e-01 + -1.326145352418741e-01 + -1.324167951779929e-01 + -1.322184775207451e-01 + -1.320195795498869e-01 + -1.318200985870884e-01 + -1.316200319613715e-01 + -1.314193769770063e-01 + -1.312181309395709e-01 + -1.310162911860090e-01 + -1.308138550466099e-01 + -1.306108198680959e-01 + -1.304071830473609e-01 + -1.302029419358534e-01 + -1.299980938748928e-01 + -1.297926362720222e-01 + -1.295865665431496e-01 + -1.293798820977739e-01 + -1.291725803345192e-01 + -1.289646586840201e-01 + -1.287561146028010e-01 + -1.285469455426532e-01 + -1.283371489741077e-01 + -1.281267223855238e-01 + -1.279156632656401e-01 + -1.277039691180485e-01 + -1.274916374652222e-01 + -1.272786658468367e-01 + -1.270650518164191e-01 + -1.268507929437779e-01 + -1.266358868257778e-01 + -1.264203310530910e-01 + -1.262041232080785e-01 + -1.259872609212462e-01 + -1.257697418614985e-01 + -1.255515637216261e-01 + -1.253327241582721e-01 + -1.251132208452764e-01 + -1.248930515084944e-01 + -1.246722139029198e-01 + -1.244507057917270e-01 + -1.242285249387353e-01 + -1.240056691417677e-01 + -1.237821362222557e-01 + -1.235579240163024e-01 + -1.233330303641722e-01 + -1.231074531412701e-01 + -1.228811902909497e-01 + -1.226542397328350e-01 + -1.224265993926709e-01 + -1.221982672554952e-01 + -1.219692413178169e-01 + -1.217395195880366e-01 + -1.215091001037845e-01 + -1.212779809466277e-01 + -1.210461602256936e-01 + -1.208136360512357e-01 + -1.205804065871132e-01 + -1.203464700244819e-01 + -1.201118245085727e-01 + -1.198764682774020e-01 + -1.196403996473917e-01 + -1.194036168896808e-01 + -1.191661183059585e-01 + -1.189279022478277e-01 + -1.186889671039581e-01 + -1.184493112863093e-01 + -1.182089332317922e-01 + -1.179678314226111e-01 + -1.177260043675167e-01 + -1.174834505981953e-01 + -1.172401686820595e-01 + -1.169961572351099e-01 + -1.167514149230550e-01 + -1.165059404117411e-01 + -1.162597324012717e-01 + -1.160127896494247e-01 + -1.157651109471359e-01 + -1.155166951180050e-01 + -1.152675410245315e-01 + -1.150176475647009e-01 + -1.147670136740031e-01 + -1.145156383362403e-01 + -1.142635205942373e-01 + -1.140106595167702e-01 + -1.137570541746248e-01 + -1.135027037005393e-01 + -1.132476072885310e-01 + -1.129917641930528e-01 + -1.127351736954119e-01 + -1.124778350964029e-01 + -1.122197477241451e-01 + -1.119609110122396e-01 + -1.117013244465126e-01 + -1.114409874638995e-01 + -1.111798996306716e-01 + -1.109180606010562e-01 + -1.106554699641756e-01 + -1.103921273860412e-01 + -1.101280326261970e-01 + -1.098631854961093e-01 + -1.095975858385727e-01 + -1.093312335328026e-01 + -1.090641285282797e-01 + -1.087962708201439e-01 + -1.085276604456853e-01 + -1.082582975002112e-01 + -1.079881821487157e-01 + -1.077173146235393e-01 + -1.074456951780328e-01 + -1.071733241222459e-01 + -1.069002018428373e-01 + -1.066263287812705e-01 + -1.063517054311524e-01 + -1.060763323447030e-01 + -1.058002101349244e-01 + -1.055233394662503e-01 + -1.052457210591484e-01 + -1.049673557111256e-01 + -1.046882442747131e-01 + -1.044083876525769e-01 + -1.041277868396362e-01 + -1.038464428925467e-01 + -1.035643569070283e-01 + -1.032815300276830e-01 + -1.029979634724506e-01 + -1.027136585692167e-01 + -1.024286166734134e-01 + -1.021428391952729e-01 + -1.018563276648054e-01 + -1.015690836461885e-01 + -1.012811087487812e-01 + -1.009924046817547e-01 + -1.007029732152899e-01 + -1.004128161832720e-01 + -1.001219355157012e-01 + -9.983033318947014e-02 + -9.953801123789381e-02 + -9.924497181065381e-02 + -9.895121710675164e-02 + -9.865674937610269e-02 + -9.836157098451072e-02 + -9.806568435550449e-02 + -9.776909196067303e-02 + -9.747179635418697e-02 + -9.717380018935569e-02 + -9.687510621844479e-02 + -9.657571724434684e-02 + -9.627563612429342e-02 + -9.597486578738405e-02 + -9.567340928055243e-02 + -9.537126972794066e-02 + -9.506845031433311e-02 + -9.476495428226560e-02 + -9.446078497156121e-02 + -9.415594584893169e-02 + -9.385044042616277e-02 + -9.354427227181475e-02 + -9.323744504129654e-02 + -9.292996250412121e-02 + -9.262182850819620e-02 + -9.231304696021701e-02 + -9.200362185853247e-02 + -9.169355729461479e-02 + -9.138285746240042e-02 + -9.107152658866502e-02 + -9.075956898041553e-02 + -9.044698911371001e-02 + -9.013379148226228e-02 + -8.981998063663986e-02 + -8.950556130228651e-02 + -8.919053824746566e-02 + -8.887491629233073e-02 + -8.855870039130662e-02 + -8.824189556852820e-02 + -8.792450692262105e-02 + -8.760653968435946e-02 + -8.728799913331863e-02 + -8.696889060279582e-02 + -8.664921956673162e-02 + -8.632899159487925e-02 + -8.600821233560418e-02 + -8.568688750540578e-02 + -8.536502290181033e-02 + -8.504262441718942e-02 + -8.471969805751253e-02 + -8.439624991228166e-02 + -8.407228614692197e-02 + -8.374781300392073e-02 + -8.342283682562317e-02 + -8.309736406563741e-02 + -8.277140122021633e-02 + -8.244495489588025e-02 + -8.211803185128563e-02 + -8.179063884910803e-02 + -8.146278271646745e-02 + -8.113447041766966e-02 + -8.080570906396743e-02 + -8.047650581443898e-02 + -8.014686781276095e-02 + -7.981680240792462e-02 + -7.948631704884632e-02 + -7.915541916537581e-02 + -7.882411634928695e-02 + -7.849241630024972e-02 + -7.816032674199919e-02 + -7.782785552351333e-02 + -7.749501058768483e-02 + -7.716179991262426e-02 + -7.682823159405887e-02 + -7.649431382724470e-02 + -7.616005485767063e-02 + -7.582546305035900e-02 + -7.549054686796573e-02 + -7.515531479419155e-02 + -7.481977542018518e-02 + -7.448393744841550e-02 + -7.414780964186224e-02 + -7.381140084542565e-02 + -7.347471999131863e-02 + -7.313777608894645e-02 + -7.280057821330421e-02 + -7.246313551410177e-02 + -7.212545725749871e-02 + -7.178755277686714e-02 + -7.144943146088469e-02 + -7.111110278446059e-02 + -7.077257629326426e-02 + -7.043386160590276e-02 + -7.009496843792977e-02 + -6.975590657859251e-02 + -6.941668588038064e-02 + -6.907731623736572e-02 + -6.873780762436528e-02 + -6.839817013263073e-02 + -6.805841390025939e-02 + -6.771854911509186e-02 + -6.737858603607710e-02 + -6.703853501790490e-02 + -6.669840646610939e-02 + -6.635821080020089e-02 + -6.601795854049251e-02 + -6.567766029345612e-02 + -6.533732672933422e-02 + -6.499696856244534e-02 + -6.465659655384894e-02 + -6.431622152906083e-02 + -6.397585437117004e-02 + -6.363550601749234e-02 + -6.329518746487584e-02 + -6.295490977261127e-02 + -6.261468405232740e-02 + -6.227452144919842e-02 + -6.193443314782195e-02 + -6.159443038613483e-02 + -6.125452449175782e-02 + -6.091472682299209e-02 + -6.057504875634485e-02 + -6.023550170757411e-02 + -5.989609715178513e-02 + -5.955684662483948e-02 + -5.921776169220818e-02 + -5.887885392543541e-02 + -5.854013491232313e-02 + -5.820161634720029e-02 + -5.786330995026137e-02 + -5.752522743033200e-02 + -5.718738052319666e-02 + -5.684978102238282e-02 + -5.651244078415603e-02 + -5.617537160914179e-02 + -5.583858533460867e-02 + -5.550209389764803e-02 + -5.516590918758542e-02 + -5.483004310132478e-02 + -5.449450760284144e-02 + -5.415931467224975e-02 + -5.382447627285256e-02 + -5.349000433169809e-02 + -5.315591089010736e-02 + -5.282220800586091e-02 + -5.248890761304951e-02 + -5.215602173570349e-02 + -5.182356244333215e-02 + -5.149154173522678e-02 + -5.115997163165661e-02 + -5.082886416999516e-02 + -5.049823136495267e-02 + -5.016808523392447e-02 + -4.983843779429676e-02 + -4.950930104255546e-02 + -4.918068697018742e-02 + -4.885260756307724e-02 + -4.852507478656667e-02 + -4.819810059193260e-02 + -4.787169691654435e-02 + -4.754587567726797e-02 + -4.722064876717863e-02 + -4.689602805511645e-02 + -4.657202539266345e-02 + -4.624865260469157e-02 + -4.592592148337583e-02 + -4.560384379218360e-02 + -4.528243126408893e-02 + -4.496169559628109e-02 + -4.464164845065374e-02 + -4.432230145577399e-02 + -4.400366620090135e-02 + -4.368575422918912e-02 + -4.336857704568598e-02 + -4.305214611378011e-02 + -4.273647284209545e-02 + -4.242156859535529e-02 + -4.210744469338823e-02 + -4.179411239819886e-02 + -4.148158291999365e-02 + -4.116986741259326e-02 + -4.085897697584740e-02 + -4.054892265423421e-02 + -4.023971542131075e-02 + -3.993136619382251e-02 + -3.962388583123071e-02 + -3.931728511726146e-02 + -3.901157477126550e-02 + -3.870676544906659e-02 + -3.840286772630143e-02 + -3.809989211312446e-02 + -3.779784905219891e-02 + -3.749674889469364e-02 + -3.719660192057603e-02 + -3.689741833904486e-02 + -3.659920826728452e-02 + -3.630198174507707e-02 + -3.600574873447752e-02 + -3.571051910326670e-02 + -3.541630263528623e-02 + -3.512310902944081e-02 + -3.483094789226868e-02 + -3.453982873906083e-02 + -3.424976098893085e-02 + -3.396075396966352e-02 + -3.367281691959212e-02 + -3.338595897696143e-02 + -3.310018917253339e-02 + -3.281551644593574e-02 + -3.253194963945627e-02 + -3.224949747985180e-02 + -3.196816859917648e-02 + -3.168797153104005e-02 + -3.140891468854379e-02 + -3.113100638362195e-02 + -3.085425482400445e-02 + -3.057866810270934e-02 + -3.030425420554928e-02 + -3.003102099769914e-02 + -2.975897623664901e-02 + -2.948812757423692e-02 + -2.921848253324668e-02 + -2.895004852283206e-02 + -2.868283284398848e-02 + -2.841684266935389e-02 + -2.815208505554401e-02 + -2.788856694668657e-02 + -2.762629515277662e-02 + -2.736527636581735e-02 + -2.710551716378724e-02 + -2.684702399109470e-02 + -2.658980317107453e-02 + -2.633386090723251e-02 + -2.607920326420626e-02 + -2.582583618939347e-02 + -2.557376551170278e-02 + -2.532299691452088e-02 + -2.507353595904179e-02 + -2.482538808499682e-02 + -2.457855859507147e-02 + -2.433305266394129e-02 + -2.408887533435539e-02 + -2.384603151841623e-02 + -2.360452600145211e-02 + -2.336436343201402e-02 + -2.312554832609387e-02 + -2.288808507226996e-02 + -2.265197791904223e-02 + -2.241723098404469e-02 + -2.218384825955722e-02 + -2.195183359578144e-02 + -2.172119070564114e-02 + -2.149192317903017e-02 + -2.126403446983029e-02 + -2.103752788800763e-02 + -2.081240661670658e-02 + -2.058867370373213e-02 + -2.036633206151460e-02 + -2.014538447470562e-02 + -1.992583358220780e-02 + -1.970768189350454e-02 + -1.949093179354288e-02 + -1.927558551890179e-02 + -1.906164517346932e-02 + -1.884911273803516e-02 + -1.863799005561765e-02 + -1.842827883084014e-02 + -1.821998063919436e-02 + -1.801309692347624e-02 + -1.780762899421212e-02 + -1.760357803216378e-02 + -1.740094507853177e-02 + -1.719973104881908e-02 + -1.699993673354589e-02 + -1.680156278321953e-02 + -1.660460972117107e-02 + -1.640907794332531e-02 + -1.621496771146485e-02 + -1.602227916433384e-02 + -1.583101231371427e-02 + -1.564116703862817e-02 + -1.545274309883748e-02 + -1.526574012772238e-02 + -1.508015761914186e-02 + -1.489599495303444e-02 + -1.471325139301425e-02 + -1.453192606709004e-02 + -1.435201798658663e-02 + -1.417352604127694e-02 + -1.399644899327932e-02 + -1.382078549302728e-02 + -1.364653407164367e-02 + -1.347369313289915e-02 + -1.330226097046918e-02 + -1.313223576283522e-02 + -1.296361556501821e-02 + -1.279639832327015e-02 + -1.263058187022890e-02 + -1.246616392163480e-02 + -1.230314208589229e-02 + -1.214151385550256e-02 + -1.198127661271531e-02 + -1.182242763705999e-02 + -1.166496409538715e-02 + -1.150888304589847e-02 + -1.135418144553720e-02 + -1.120085614521435e-02 + -1.104890388968280e-02 + -1.089832132231912e-02 + -1.074910498405016e-02 + -1.060125131793449e-02 + -1.045475666967276e-02 + -1.030961728045836e-02 + -1.016582929759894e-02 + -1.002338877483435e-02 + -9.882291666582974e-03 + -9.742533838318660e-03 + -9.604111064390042e-03 + -9.467019020603698e-03 + -9.331253297647066e-03 + -9.196809399717920e-03 + -9.063682741005771e-03 + -8.931868652392608e-03 + -8.801362376326071e-03 + -8.672159070075210e-03 + -8.544253813557785e-03 + -8.417641603575709e-03 + -8.292317355716863e-03 + -8.168275911736944e-03 + -8.045512032694060e-03 + -7.924020397696789e-03 + -7.803795618017998e-03 + -7.684832236645861e-03 + -7.567124714929550e-03 + -7.450667444285828e-03 + -7.335454748033539e-03 + -7.221480883382279e-03 + -7.108740043647531e-03 + -6.997226348639058e-03 + -6.886933855315346e-03 + -6.777856561420043e-03 + -6.669988397445708e-03 + -6.563323236817328e-03 + -6.457854897474280e-03 + -6.353577131920331e-03 + -6.250483639666226e-03 + -6.148568068731946e-03 + -6.047824005897748e-03 + -5.948244987599076e-03 + -5.849824501347084e-03 + -5.752555978513681e-03 + -5.656432807264865e-03 + -5.561448332218353e-03 + -5.467595840626229e-03 + -5.374868578311233e-03 + -5.283259752176146e-03 + -5.192762521793490e-03 + -5.103370006626977e-03 + -5.015075286629148e-03 + -4.927871401782054e-03 + -4.841751355904269e-03 + -4.756708115524279e-03 + -4.672734611342882e-03 + -4.589823741835811e-03 + -4.507968371897898e-03 + -4.427161334111035e-03 + -4.347395432360992e-03 + -4.268663440821394e-03 + -4.190958105123732e-03 + -4.114272145237943e-03 + -4.038598254543495e-03 + -3.963929102325757e-03 + -3.890257335680627e-03 + -3.817575578228647e-03 + -3.745876433060482e-03 + -3.675152484340320e-03 + -3.605396296371936e-03 + -3.536600416225055e-03 + -3.468757375181376e-03 + -3.401859688514519e-03 + -3.335899857279008e-03 + -3.270870369624132e-03 + -3.206763701552144e-03 + -3.143572317880366e-03 + -3.081288673370167e-03 + -3.019905214426235e-03 + -2.959414379161780e-03 + -2.899808598243805e-03 + -2.841080297295089e-03 + -2.783221897187762e-03 + -2.726225814481816e-03 + -2.670084462721576e-03 + -2.614790253659214e-03 + -2.560335598217736e-03 + -2.506712907773368e-03 + -2.453914594305456e-03 + -2.401933071402304e-03 + -2.350760756079694e-03 + -2.300390068609935e-03 + -2.250813433830373e-03 + -2.202023282926124e-03 + -2.154012052729510e-03 + -2.106772187286276e-03 + -2.060296139752280e-03 + -2.014576371403607e-03 + -1.969605353508076e-03 + -1.925375568654703e-03 + -1.881879509874747e-03 + -1.839109683164610e-03 + -1.797058608044933e-03 + -1.755718816706420e-03 + -1.715082857166331e-03 + -1.675143293280333e-03 + -1.635892703443056e-03 + -1.597323683697497e-03 + -1.559428848712492e-03 + -1.522200830321536e-03 + -1.485632280183934e-03 + -1.449715870092584e-03 + -1.414444291041649e-03 + -1.379810256474448e-03 + -1.345806501791486e-03 + -1.312425783716085e-03 + -1.279660883530597e-03 + -1.247504606026809e-03 + -1.215949779407020e-03 + -1.184989258561609e-03 + -1.154615923413642e-03 + -1.124822679454977e-03 + -1.095602461024569e-03 + -1.066948228352973e-03 + -1.038852968864840e-03 + -1.011309701783447e-03 + -9.843114734261671e-04 + -9.578513585924095e-04 + -9.319224654553773e-04 + -9.065179311555053e-04 + -8.816309237062864e-04 + -8.572546450286300e-04 + -8.333823272254078e-04 + -8.100072355517002e-04 + -7.871226705798197e-04 + -7.647219641139264e-04 + -7.427984826268421e-04 + -7.213456295816899e-04 + -7.003568409622573e-04 + -6.798255887472549e-04 + -6.597453829223955e-04 + -6.401097674232178e-04 + -6.209123240167792e-04 + -6.021466729184949e-04 + -5.838064694784330e-04 + -5.658854088227745e-04 + -5.483772251938055e-04 + -5.312756891491387e-04 + -5.145746125261390e-04 + -4.982678466736966e-04 + -4.823492804447342e-04 + -4.668128456938979e-04 + -4.516525141842837e-04 + -4.368622962674093e-04 + -4.224362466667431e-04 + -4.083684605074392e-04 + -3.946530726767535e-04 + -3.812842634726816e-04 + -3.682562540846601e-04 + -3.555633067280622e-04 + -3.431997298953713e-04 + -3.311598733734528e-04 + -3.194381292167991e-04 + -3.080289365424697e-04 + -2.969267761950205e-04 + -2.861261725181284e-04 + -2.756216974519896e-04 + -2.654079650780054e-04 + -2.554796342192304e-04 + -2.458314117831762e-04 + -2.364580472261255e-04 + -2.273543358783364e-04 + -2.185151214599497e-04 + -2.099352907310207e-04 + -2.016097775283348e-04 + -1.935335642058985e-04 + -1.857016766078902e-04 + -1.781091887621722e-04 + -1.707512232636694e-04 + -1.636229467286454e-04 + -1.567195750846747e-04 + -1.500363727286114e-04 + -1.435686485603325e-04 + -1.373117617901810e-04 + -1.312611200233150e-04 + -1.254121759173047e-04 + -1.197604333131590e-04 + -1.143014443478538e-04 + -1.090308067994204e-04 + -1.039441703359566e-04 + -9.903723276260782e-05 + -9.430573818110019e-05 + -8.974548326970851e-05 + -8.535231263786576e-05 + -8.112211782233505e-05 + -7.705084341917009e-05 + -7.313448187442813e-05 + -6.936907333027644e-05 + -6.575071119249467e-05 + -6.227553658275769e-05 + -5.893973914900695e-05 + -5.573956203853484e-05 + -5.267129595161032e-05 + -4.973128084575291e-05 + -4.691591029739419e-05 + -4.422162539456759e-05 + -4.164491727590835e-05 + -3.918233061143914e-05 + -3.683045757249665e-05 + -3.458594118927672e-05 + -3.244547785270521e-05 + -3.040581149216269e-05 + -2.846373767254703e-05 + -2.661610506551807e-05 + -2.485980997915884e-05 + -2.319180110803831e-05 + -2.160907991664170e-05 + -2.010869565536285e-05 + -1.868775067119526e-05 + -1.734339969669114e-05 + -1.607284546059277e-05 + -1.487334444060605e-05 + -1.374220506933501e-05 + -1.267678404087566e-05 + -1.167449238525668e-05 + -1.073279264625808e-05 + -9.849195952118955e-06 + -9.021268250139948e-06 + -8.246626567968640e-06 + -7.522936912951058e-06 + -6.847920492700095e-06 + -6.219349168072015e-06 + -5.635044236568559e-06 + -5.092882470932048e-06 + -4.590790895044201e-06 + -4.126746487291455e-06 + -3.698781859286729e-06 + -3.304979520120749e-06 + -2.943472506918129e-06 + -2.612449510984778e-06 + -2.310148834845521e-06 + -2.034859929271865e-06 + -1.784927798002038e-06 + -1.558746837246077e-06 + -1.354763241038800e-06 + -1.171478551126456e-06 + -1.007443551831214e-06 + -8.612614901318561e-07 + -7.315906671893518e-07 + -6.171385461507109e-07 + -5.166657190198488e-07 + -4.289874671965282e-07 + -3.529682224298676e-07 + -2.875261969469119e-07 + -2.316338735426513e-07 + -1.843129441850586e-07 + -1.446395043479685e-07 + -1.117434618507405e-07 + -8.480406305106530e-08 + -6.305553831873507e-08 + -4.578544870958251e-08 + -3.233089380928512e-08 + -2.208448022054805e-08 + -1.449165750478591e-08 + -9.047683233008124e-09 + -5.303766739728543e-09 + -2.863504097386866e-09 + -1.380623394721897e-09 + -5.650641971230870e-10 + -1.790372475752245e-10 + -3.511990521548007e-11 + -8.192061809269277e-13 + 0.000000000000000e+00 diff --git a/examples/SPIN/test_problems/validation_nvt/in.spin.nvt_lattice b/examples/SPIN/test_problems/validation_nvt/in.spin.nvt_lattice new file mode 100644 index 0000000000000000000000000000000000000000..1d63f01d4378242a240c72b735c97c5fc2a2bb9a --- /dev/null +++ b/examples/SPIN/test_problems/validation_nvt/in.spin.nvt_lattice @@ -0,0 +1,51 @@ +# bcc iron in a 3d periodic box + +clear +units metal +atom_style spin + +dimension 3 +boundary p p p + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice bcc 2.8665 +region box block 0.0 3.0 0.0 3.0 0.0 3.0 +create_box 1 box +create_atoms 1 box + +# setting mass, mag. moments, and interactions for bcc iron + +mass 1 55.845 +set group all spin 2.2 0.0 0.0 1.0 +velocity all create 400 4928459 rot yes dist gaussian + +pair_style hybrid/overlay eam/alloy spin/exchange 4.0 spin/neel 4.0 +pair_coeff * * eam/alloy Fe_Mishin2006.eam.alloy Fe +pair_coeff * * spin/exchange exchange 3.4 0.1 0.2171 1.841 +pair_coeff * * spin/neel neel 4.0 0.02 0.0 1.841 0.0 0.0 1.0 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 +fix 2 all langevin 200.0 200.0 10.0 48279 +fix 3 all langevin/spin 0.0 0.00001 321 +fix 4 all nve/spin lattice moving +timestep 0.001 + +# compute and output options + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_tmag temp v_emag ke pe etotal +thermo 200 + +run 200000 diff --git a/examples/SPIN/test_problems/validation_nvt/in.spin.nvt_spin b/examples/SPIN/test_problems/validation_nvt/in.spin.nvt_spin new file mode 100644 index 0000000000000000000000000000000000000000..435e877bdf15577d55ce88bbbb10c5ece36c1bda --- /dev/null +++ b/examples/SPIN/test_problems/validation_nvt/in.spin.nvt_spin @@ -0,0 +1,49 @@ +# bcc iron in a 3d periodic box + +clear +units metal +atom_style spin + +dimension 3 +boundary p p p + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice bcc 2.8665 +region box block 0.0 3.0 0.0 3.0 0.0 3.0 +create_box 1 box +create_atoms 1 box + +# setting mass, mag. moments, and interactions for bcc iron + +mass 1 55.845 +set group all spin 2.2 0.0 0.0 1.0 +velocity all create 0 4928459 rot yes dist gaussian + +pair_style hybrid/overlay eam/alloy spin/exchange 3.5 +pair_coeff * * eam/alloy Fe_Mishin2006.eam.alloy Fe +pair_coeff * * spin/exchange exchange 3.4 0.1 0.2171 1.841 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 +fix 2 all langevin/spin 200.0 0.1 321 +fix 3 all nve/spin lattice moving +timestep 0.001 + +# compute and output options + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_tmag temp v_emag ke pe etotal +thermo 200 + +run 200000 diff --git a/examples/SPIN/test_problems/validation_nvt/plot_nvt.py b/examples/SPIN/test_problems/validation_nvt/plot_nvt.py new file mode 100755 index 0000000000000000000000000000000000000000..06c48b4c28143ce7ff4b010479bd6b85604f0651 --- /dev/null +++ b/examples/SPIN/test_problems/validation_nvt/plot_nvt.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +import numpy as np, pylab, tkinter +import matplotlib.pyplot as plt +from scipy.optimize import curve_fit +from decimal import * +import sys, string, os + + +argv = sys.argv +if len(argv) != 3: + print("Syntax: ./plot_precession.py res_nvt_spin.dat res_nvt_lattice.dat") + sys.exit() + +dirname = os.path.join(os.getcwd(), "Feb_07") +nvtspin_file = sys.argv[1] +nvtlatt_file = sys.argv[2] + +ts,tmags,temps = np.loadtxt(nvtspin_file,skiprows=0, usecols=(1,2,3),unpack=True) +tl,tmagl,templ = np.loadtxt(nvtlatt_file,skiprows=0, usecols=(1,2,3),unpack=True) + +fig = plt.figure(figsize=(8,8)) +ax1 = plt.subplot(2,1,1) +ax2 = plt.subplot(2,1,2) + +ax1.plot(ts, tmags, 'r-', label='Spin temp. (thermostat)') +ax1.plot(ts, temps, 'g-', label='Lattice temp.') +ax1.set_yscale("log") +ax1.set_ylabel("T (K)") +ax1.legend(loc=3) + +ax2.plot(tl, tmagl, 'r-', label='Spin temp.') +ax2.plot(tl, templ, 'g-', label='Lattice temp. (thermostat)') +ax2.set_yscale("log") +ax2.set_ylabel("T (K)") +ax2.legend(loc=3) + +plt.xlabel('Time (in ps)') +plt.legend() +plt.show() + +fig.savefig(os.path.join(os.getcwd(), "nve_spin_lattice.pdf"), bbox_inches="tight") +plt.close(fig) diff --git a/examples/SPIN/test_problems/validation_nvt/run-test-nvt.sh b/examples/SPIN/test_problems/validation_nvt/run-test-nvt.sh new file mode 100755 index 0000000000000000000000000000000000000000..d3abbbe1a552e7489dd6be04b891aed13a21ee0b --- /dev/null +++ b/examples/SPIN/test_problems/validation_nvt/run-test-nvt.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# clean old res +rm res_*.dat + +# compute NVT Spin -> Lattice +./../../../../src/lmp_serial -in in.spin.nvt_spin +in="$(grep -n Step log.lammps | awk -F ':' '{print $1}')" +en="$(grep -n Loop log.lammps | awk -F ':' '{print $1}')" +in="$(echo "$in+1" | bc -l)" +en="$(echo "$en-$in" | bc -l)" +tail -n +$in log.lammps | head -n $en > res_nvt_spin.dat + +# compute NVT Lattice -> Spin +./../../../../src/lmp_serial -in in.spin.nvt_lattice +in="$(grep -n Step log.lammps | awk -F ':' '{print $1}')" +en="$(grep -n Loop log.lammps | awk -F ':' '{print $1}')" +in="$(echo "$in+1" | bc -l)" +en="$(echo "$en-$in" | bc -l)" +tail -n +$in log.lammps | head -n $en > res_nvt_lattice.dat + +# plot results +python3 plot_nvt.py res_nvt_spin.dat res_nvt_lattice.dat diff --git a/examples/kim/in.kim-pm-property b/examples/kim/in.kim-pm-property new file mode 100644 index 0000000000000000000000000000000000000000..fea15278208392e445e4f8d68db3941340976982 --- /dev/null +++ b/examples/kim/in.kim-pm-property @@ -0,0 +1,91 @@ +# kim-property example +# +# For detailed information of this example please refer to: +# https://openkim.org/doc/evaluation/tutorial-lammps/ +# +# Description: +# +# This example is designed to calculate the cohesive energy corresponding to +# the equilibrium FCC lattice constant for +# `LJ_Shifted_Bernardes_1958MedCutoff_Ar__MO_126566794224_004` model for +# argon. The material properties computed in LAMMPS are represented as a +# standard KIM property instance format. (See +# https://openkim.org/doc/schema/properties-framework/ and +# https://lammps.sandia.gov/doc/kim_commands.html for further details). +# Then the created property instance is written to a file named results.edn +# using the `kim_property dump` commands. +# +# Requirement: +# +# This example requires LAMMPS built with the Python 3.6 or later package +# installed. See the `https://lammps.sandia.gov/doc/python.html` doc page for +# more info on building LAMMPS with the version of Python on your system. +# After successfully building LAMMPS with Python, you need to install the +# kim-property Python package, See the +# `https://lammps.sandia.gov/doc/Build_extras.html#kim` doc page for +# further details. +# +# This example requires that the KIM Portable Model (PM) +# `LJ_Shifted_Bernardes_1958MedCutoff_Ar__MO_126566794224_004` +# is installed. This can be done with the command +# `kim-api-collections-management install user LJ_Shifted_Bernardes_1958MedCutoff_Ar__MO_126566794224_004` +# If this command does not work, you may need to setup your PATH to find the utility. +# If you installed the kim-api using the LAMMPS CMake build, you can do the following +# (where the current working directory is assumed to be the LAMMPS build directory) +# source ./kim_build-prefix/bin/kim-api-activate +# If you installed the kim-api using the LAMMPS Make build, you can do the following +# (where the current working directory is assumed to be the LAMMPS src directory) +# source ../lib/kim/installed-kim-api-X.Y.Z/bin/kim-api-activate +# (where you should relplace X.Y.Z with the appropriate kim-api version number). +# +# Or, see https://openkim.org/doc/obtaining-models for alternative options. +# + +# Initialize interatomic potential (KIM model) and units +atom_style atomic + +# Set the OpenKIM model that will be used +kim_init LJ_Shifted_Bernardes_1958MedCutoff_Ar__MO_126566794224_004 metal + +# the equilibrium lattice constant for the fcc structure +variable lattice_constant equal 5.248509056866169 + +# Periodic boundary conditions along all three dimensions +boundary p p p + +# Create an FCC lattice with the lattice spacing +# using a single conventional (orthogonal) unit cell +lattice fcc ${lattice_constant} +region box block 0 1 0 1 0 1 units lattice +create_box 1 box +create_atoms 1 box +mass 1 39.948 + +# Specify the KIM interactions +kim_interactions Ar + +# Compute energy +run 0 + +# Get cohesive energy +variable natoms equal "count(all)" +variable ecohesive equal "-pe/v_natoms" + +# Create a property instance +kim_property create 1 cohesive-potential-energy-cubic-crystal + +# Set all the key-value pairs for this property instance +kim_property modify 1 key short-name source-value 1 fcc & + key species source-value 1 Ar & + key a source-value ${lattice_constant} & + source-unit angstrom & + key basis-atom-coordinates source-value 1 1:3 0.0 0.0 0.0 & + source-value 2 1:3 0.0 0.5 0.5 & + source-value 3 1:3 0.5 0.0 0.5 & + source-value 4 1:3 0.5 0.5 0.0 & + key space-group source-value Fm-3m & + key cohesive-potential-energy source-value ${ecohesive} & + source-unit eV + +# Dump the results in a file +kim_property dump "results.edn" diff --git a/lib/colvars/lepton/src/MSVC_erfc.h b/lib/colvars/lepton/src/MSVC_erfc.h index c30a8ce542157a22db338c81b261678e3258cd8f..b1cd87a2896600a8db4cb2ab9fd0459c4e117712 100644 --- a/lib/colvars/lepton/src/MSVC_erfc.h +++ b/lib/colvars/lepton/src/MSVC_erfc.h @@ -8,9 +8,13 @@ * (VC11 has _MSC_VER=1700). */ -#if defined(_MSC_VER) +#if defined(_MSC_VER) || defined(__MINGW32__) +#if !defined(M_PI) #define M_PI 3.14159265358979323846264338327950288 +#endif +#endif +#if defined(_MSC_VER) #if _MSC_VER <= 1700 // 1700 is VC11, 1800 is VC12 /*************************** * erf.cpp diff --git a/lib/gpu/.gitignore b/lib/gpu/.gitignore index 7b5254e4808cebde170a59bf91821de6a46147d9..d32b287c1e05c2d509a37324cef7f8bafeea51bb 100644 --- a/lib/gpu/.gitignore +++ b/lib/gpu/.gitignore @@ -2,6 +2,7 @@ /obj_ocl /ocl_get_devices /nvc_get_devices +/hip_get_devices /*.cubin /*_cubin.h /*_cl.h diff --git a/lib/gpu/Makefile.hip b/lib/gpu/Makefile.hip new file mode 100644 index 0000000000000000000000000000000000000000..62d34592d1974cc8b33760c87071fc75067b4a2a --- /dev/null +++ b/lib/gpu/Makefile.hip @@ -0,0 +1,148 @@ +# /* ---------------------------------------------------------------------- +# Generic Linux Makefile for HIP +# - export HIP_PLATFORM=hcc (or nvcc) before execution +# - change HIP_ARCH for your GPU +# ------------------------------------------------------------------------- */ + +# this setting should match LAMMPS Makefile +# one of LAMMPS_SMALLBIG (default), LAMMPS_BIGBIG and LAMMPS_SMALLSMALL + +LMP_INC = -DLAMMPS_SMALLBIG + +# precision for GPU calculations +# -D_SINGLE_SINGLE # Single precision for all calculations +# -D_DOUBLE_DOUBLE # Double precision for all calculations +# -D_SINGLE_DOUBLE # Accumulation of forces, etc. in double + +HIP_PRECISION = -D_SINGLE_DOUBLE + +HIP_OPTS = -O3 +HIP_HOST_OPTS = -Wno-deprecated-declarations +HIP_HOST_INCLUDE = + +# use device sort +# requires linking with hipcc and hipCUB + (rocPRIM or CUB for AMD or Nvidia respectively) +HIP_HOST_OPTS += -DUSE_HIP_DEVICE_SORT +# path to cub +HIP_HOST_INCLUDE += -I./ +# path to hipcub +HIP_HOST_INCLUDE += -I$(HIP_PATH)/../include + +# use mpi +HIP_HOST_OPTS += -DMPI_GERYON -DUCL_NO_EXIT +# this settings should match LAMMPS Makefile +MPI_COMP_OPTS = $(shell mpicxx --showme:compile) +MPI_LINK_OPTS = $(shell mpicxx --showme:link) +#MPI_COMP_OPTS += -I/usr/include/mpi -DMPICH_IGNORE_CXX_SEEK -DOMPI_SKIP_MPICXX=1 + +HIP_PATH ?= $(wildcard /opt/rocm/hip) +HIP_PLATFORM=$(shell $(HIP_PATH)/bin/hipconfig --compiler) + +ifeq (hcc,$(HIP_PLATFORM)) + HIP_OPTS += -ffast-math + # possible values: gfx803,gfx900,gfx906 + HIP_ARCH = gfx906 +else ifeq (nvcc,$(HIP_PLATFORM)) + HIP_OPTS += --use_fast_math + HIP_ARCH = -gencode arch=compute_30,code=[sm_30,compute_30] -gencode arch=compute_32,code=[sm_32,compute_32] -gencode arch=compute_35,code=[sm_35,compute_35] \ + -gencode arch=compute_50,code=[sm_50,compute_50] -gencode arch=compute_52,code=[sm_52,compute_52] -gencode arch=compute_53,code=[sm_53,compute_53]\ + -gencode arch=compute_60,code=[sm_60,compute_60] -gencode arch=compute_61,code=[sm_61,compute_61] -gencode arch=compute_62,code=[sm_62,compute_62]\ + -gencode arch=compute_70,code=[sm_70,compute_70] -gencode arch=compute_72,code=[sm_72,compute_72] -gencode arch=compute_75,code=[sm_75,compute_75] +else + $(error Specify HIP platform using 'export HIP_PLATFORM=(hcc,nvcc)') +endif + +BIN_DIR = . +OBJ_DIR = ./obj +LIB_DIR = . +AR = ar +BSH = /bin/sh + + +# /* ---------------------------------------------------------------------- +# don't change section below without need +# ------------------------------------------------------------------------- */ + +HIP_OPTS += -DUSE_HIP $(HIP_PRECISION) +HIP_GPU_OPTS += $(HIP_OPTS) -I./ + +ifeq (hcc,$(HIP_PLATFORM)) + HIP_HOST_OPTS += -fPIC + HIP_GPU_CC = $(HIP_PATH)/bin/hipcc --genco + HIP_GPU_OPTS_S = -t="$(HIP_ARCH)" -f=\" + HIP_GPU_OPTS_E = \" + HIP_KERNEL_SUFFIX = .cpp + HIP_LIBS_TARGET = export HCC_AMDGPU_TARGET := $(HIP_ARCH) + export HCC_AMDGPU_TARGET := $(HIP_ARCH) +else ifeq (nvcc,$(HIP_PLATFORM)) + HIP_GPU_CC = $(HIP_PATH)/bin/hipcc --fatbin + HIP_GPU_OPTS += $(HIP_ARCH) + HIP_GPU_SORT_ARCH = $(HIP_ARCH) + # fix nvcc can't handle -pthread flag + MPI_COMP_OPTS := $(subst -pthread,-Xcompiler -pthread,$(MPI_COMP_OPTS)) + MPI_LINK_OPTS := $(subst -pthread,-Xcompiler -pthread,$(MPI_LINK_OPTS)) +endif + +# hipcc is essential for device sort, because of hipcub is header only library and ROCm gpu code generation is deferred to the linking stage +HIP_HOST_CC = $(HIP_PATH)/bin/hipcc +HIP_HOST_OPTS += $(HIP_OPTS) $(MPI_COMP_OPTS) $(LMP_INC) +HIP_HOST_CC_CMD = $(HIP_HOST_CC) $(HIP_HOST_OPTS) $(HIP_HOST_INCLUDE) + +# sources + +ALL_H = $(wildcard ./geryon/ucl*.h) $(wildcard ./geryon/hip*.h) $(wildcard ./lal_*.h) +SRCS := $(wildcard ./lal_*.cpp) +OBJS := $(subst ./,$(OBJ_DIR)/,$(SRCS:%.cpp=%.o)) +CUS := $(wildcard lal_*.cu) +CUHS := $(filter-out pppm_cubin.h, $(CUS:lal_%.cu=%_cubin.h)) pppm_f_cubin.h pppm_d_cubin.h +CUHS := $(addprefix $(OBJ_DIR)/, $(CUHS)) + +all: $(OBJ_DIR) $(CUHS) $(LIB_DIR)/libgpu.a $(BIN_DIR)/hip_get_devices + +$(OBJ_DIR): + mkdir -p $@ + +# GPU kernels compilation + +$(OBJ_DIR)/pppm_f_cubin.h: lal_pppm.cu $(ALL_H) + @cp $< $(OBJ_DIR)/temp_pppm_f.cu$(HIP_KERNEL_SUFFIX) + $(HIP_GPU_CC) $(HIP_GPU_OPTS_S) $(HIP_GPU_OPTS) -Dgrdtyp=float -Dgrdtyp4=float4 $(HIP_GPU_OPTS_E) -o $(OBJ_DIR)/pppm_f.cubin $(OBJ_DIR)/temp_pppm_f.cu$(HIP_KERNEL_SUFFIX) + @xxd -i $(OBJ_DIR)/pppm_f.cubin $@ + @sed -i "s/[a-zA-Z0-9_]*pppm_f_cubin/pppm_f/g" $@ + @rm $(OBJ_DIR)/temp_pppm_f.cu$(HIP_KERNEL_SUFFIX) $(OBJ_DIR)/pppm_f.cubin + +$(OBJ_DIR)/pppm_d_cubin.h: lal_pppm.cu $(ALL_H) + @cp $< $(OBJ_DIR)/temp_pppm_d.cu$(HIP_KERNEL_SUFFIX) + $(HIP_GPU_CC) $(HIP_GPU_OPTS_S) $(HIP_GPU_OPTS) -Dgrdtyp=double -Dgrdtyp4=double4 $(HIP_GPU_OPTS_E) -o $(OBJ_DIR)/pppm_d.cubin $(OBJ_DIR)/temp_pppm_d.cu$(HIP_KERNEL_SUFFIX) + @xxd -i $(OBJ_DIR)/pppm_d.cubin $@ + @sed -i "s/[a-zA-Z0-9_]*pppm_d_cubin/pppm_d/g" $@ + @rm $(OBJ_DIR)/temp_pppm_d.cu$(HIP_KERNEL_SUFFIX) $(OBJ_DIR)/pppm_d.cubin + +$(OBJ_DIR)/%_cubin.h: lal_%.cu $(ALL_H) + @cp $< $(OBJ_DIR)/temp_$*.cu$(HIP_KERNEL_SUFFIX) + $(HIP_GPU_CC) $(HIP_GPU_OPTS_S) $(HIP_GPU_OPTS) $(HIP_GPU_OPTS_E) -o $(OBJ_DIR)/$*.cubin $(OBJ_DIR)/temp_$*.cu$(HIP_KERNEL_SUFFIX) + @xxd -i $(OBJ_DIR)/$*.cubin $@ + @sed -i "s/[a-zA-Z0-9_]*$*_cubin/$*/g" $@ + @rm $(OBJ_DIR)/temp_$*.cu$(HIP_KERNEL_SUFFIX) $(OBJ_DIR)/$*.cubin + +# host sources compilation + +$(OBJ_DIR)/lal_atom.o: lal_atom.cpp $(CUHS) $(ALL_H) + $(HIP_HOST_CC_CMD) -o $@ -c $< -I$(OBJ_DIR) $(HIP_GPU_SORT_ARCH) + +$(OBJ_DIR)/lal_%.o: lal_%.cpp $(CUHS) $(ALL_H) + $(HIP_HOST_CC_CMD) -o $@ -c $< -I$(OBJ_DIR) + +# libgpu building + +$(LIB_DIR)/libgpu.a: $(OBJS) + $(AR) -crs $@ $(OBJS) + printf "export HIP_PLATFORM := %s\n%s\n" "$(HIP_PLATFORM)" "$(HIP_LIBS_TARGET)" > Makefile.lammps + +# test app building + +$(BIN_DIR)/hip_get_devices: ./geryon/ucl_get_devices.cpp $(ALL_H) + $(HIP_HOST_CC_CMD) -o $@ $< -DUCL_HIP $(MPI_LINK_OPTS) + +clean: + -rm -f $(BIN_DIR)/hip_get_devices $(LIB_DIR)/libgpu.a $(OBJS) $(OBJ_DIR)/temp_* $(CUHS) diff --git a/lib/gpu/README b/lib/gpu/README index 2ef8ce9556c6d559a60848d5e4b39ce5e3961c30..dfa8dcf7ff4bef41002749540e63a60b15625b0f 100644 --- a/lib/gpu/README +++ b/lib/gpu/README @@ -67,8 +67,8 @@ library requires installing the CUDA GPU driver and CUDA toolkit for your operating system. Installation of the CUDA SDK is not necessary. In addition to the LAMMPS library, the binary nvc_get_devices will also be built. This can be used to query the names and properties of GPU -devices on your system. A Makefile for OpenCL compilation is provided, -but support for OpenCL use is not currently provided by the developers. +devices on your system. A Makefile for OpenCL and ROCm HIP compilation +is provided, but support for it is not currently provided by the developers. Details of the implementation are provided in: ---- @@ -169,6 +169,25 @@ NOTE: The system-specific setting LAMMPS_SMALLBIG (default), LAMMPS_BIGBIG, src/MAKE/Makefile.foo) should be consistent with that specified when building libgpu.a (i.e. by LMP_INC in the lib/gpu/Makefile.bar). + BUILDING FOR HIP FRAMEWORK + -------------------------------- +1. Install the latest ROCm framework (https://github.com/RadeonOpenCompute/ROCm). +2. GPU sorting requires installing hipcub +(https://github.com/ROCmSoftwarePlatform/hipCUB). The HIP CUDA-backend +additionally requires cub (https://nvlabs.github.io/cub). Download and +extract the cub directory to lammps/lib/gpu/ or specify an appropriate +path in lammps/lib/gpu/Makefile.hip. +3. In Makefile.hip it is possible to specify the target platform via +export HIP_PLATFORM=hcc or HIP_PLATFORM=nvcc as well as the target +architecture (gfx803, gfx900, gfx906 etc.) +4. If your MPI implementation does not support `mpicxx --showme` command, +it is required to specify the corresponding MPI compiler and linker flags +in lammps/lib/gpu/Makefile.hip and in lammps/src/MAKE/OPTIONS/Makefile.hip. +5. Building the GPU library (libgpu.a): + cd lammps/lib/gpu; make -f Makefile.hip -j +6. Building the LAMMPS executable (lmp_hip): + cd ../../src; make hip -j + EXAMPLE CONVENTIONAL BUILD PROCESS -------------------------------- diff --git a/lib/gpu/geryon/hip_device.h b/lib/gpu/geryon/hip_device.h new file mode 100644 index 0000000000000000000000000000000000000000..93f38d28bbf773ace806e7746bc598a715859ce5 --- /dev/null +++ b/lib/gpu/geryon/hip_device.h @@ -0,0 +1,519 @@ +/* ----------------------------------------------------------------------- + Copyright (2009) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the Simplified BSD License. + ----------------------------------------------------------------------- */ + +#ifndef HIP_DEVICE +#define HIP_DEVICE + + +#include +#include +#include +#include +#include +#include "hip_macros.h" +#include "ucl_types.h" + +namespace ucl_hip { + +// -------------------------------------------------------------------------- +// - COMMAND QUEUE STUFF +// -------------------------------------------------------------------------- +typedef hipStream_t command_queue; + +inline void ucl_sync(hipStream_t &stream) { + CU_SAFE_CALL(hipStreamSynchronize(stream)); +} + +struct NVDProperties { + int device_id; + std::string name; + int major; + int minor; + CUDA_INT_TYPE totalGlobalMem; + int multiProcessorCount; + + int maxThreadsPerBlock; + int maxThreadsDim[3]; + int maxGridSize[3]; + int sharedMemPerBlock; + int totalConstantMemory; + int SIMDWidth; + int memPitch; + int regsPerBlock; + int clockRate; + int textureAlign; + + int kernelExecTimeoutEnabled; + int integrated; + int canMapHostMemory; + int concurrentKernels; + int ECCEnabled; + int computeMode; +}; + +/// Class for looking at device properties +/** \note Calls to change the device outside of the class results in incorrect + * behavior + * \note There is no error checking for indexing past the number of devices **/ +class UCL_Device { + public: + /// Collect properties for every GPU on the node + /** \note You must set the active GPU with set() before using the device **/ + inline UCL_Device(); + + inline ~UCL_Device(); + + /// Returns 1 (For compatibility with OpenCL) + inline int num_platforms() { return 1; } + + /// Return a string with name and info of the current platform + inline std::string platform_name() + { return "HIP platform"; } + + /// Delete any contexts/data and set the platform number to be used + inline int set_platform(const int pid); + + /// Return the number of devices that support CUDA + inline int num_devices() { return _properties.size(); } + + /// Set the CUDA device to the specified device number + /** A context and default command queue will be created for the device + * Returns UCL_SUCCESS if successful or UCL_ERROR if the device could not + * be allocated for use. clear() is called to delete any contexts and + * associated data from previous calls to set(). **/ + inline int set(int num); + + /// Delete any context and associated data stored from a call to set() + inline void clear(); + + /// Get the current device number + inline int device_num() { return _device; } + + /// Returns the default stream for the current device + inline command_queue & cq() { return cq(0); } + + /// Returns the stream indexed by i + inline command_queue & cq(const int i) { return _cq[i]; } + + /// Block until all commands in the default stream have completed + inline void sync() { sync(0); } + + /// Block until all commands in the specified stream have completed + inline void sync(const int i) { ucl_sync(cq(i)); } + + /// Get the number of command queues currently available on device + inline int num_queues() + { return _cq.size(); } + + /// Add a stream for device computations + inline void push_command_queue() { + _cq.push_back(hipStream_t()); + CU_SAFE_CALL(hipStreamCreateWithFlags(&_cq.back(),0)); + } + + /// Remove a stream for device computations + /** \note You cannot delete the default stream **/ + inline void pop_command_queue() { + if (_cq.size()<2) return; + CU_SAFE_CALL_NS(hipStreamDestroy(_cq.back())); + _cq.pop_back(); + } + + /// Set the default command queue (by default this is the null stream) + /** \param i index of the command queue (as added by push_command_queue()) + If i is 0, the default command queue is set to the null stream **/ + inline void set_command_queue(const int i) { + if (i==0) _cq[0]=0; + else _cq[0]=_cq[i]; + } + + /// Get the current CUDA device name + inline std::string name() { return name(_device); } + /// Get the CUDA device name + inline std::string name(const int i) + { return std::string(_properties[i].name); } + + /// Get a string telling the type of the current device + inline std::string device_type_name() { return device_type_name(_device); } + /// Get a string telling the type of the device + inline std::string device_type_name(const int i) { return "GPU"; } + + /// Get current device type (UCL_CPU, UCL_GPU, UCL_ACCELERATOR, UCL_DEFAULT) + inline int device_type() { return device_type(_device); } + /// Get device type (UCL_CPU, UCL_GPU, UCL_ACCELERATOR, UCL_DEFAULT) + inline int device_type(const int i) { return UCL_GPU; } + + /// Returns true if host memory is efficiently addressable from device + inline bool shared_memory() { return shared_memory(_device); } + /// Returns true if host memory is efficiently addressable from device + inline bool shared_memory(const int i) { return device_type(i)==UCL_CPU; } + + /// Returns true if double precision is support for the current device + inline bool double_precision() { return double_precision(_device); } + /// Returns true if double precision is support for the device + inline bool double_precision(const int i) {return arch(i)>=1.3;} + + /// Get the number of compute units on the current device + inline unsigned cus() { return cus(_device); } + /// Get the number of compute units + inline unsigned cus(const int i) + { return _properties[i].multiProcessorCount; } + + /// Get the number of cores in the current device + inline unsigned cores() { return cores(_device); } + /// Get the number of cores + inline unsigned cores(const int i) + { if (arch(i)<2.0) return _properties[i].multiProcessorCount*8; + else if (arch(i)<2.1) return _properties[i].multiProcessorCount*32; + else if (arch(i)<3.0) return _properties[i].multiProcessorCount*48; + else return _properties[i].multiProcessorCount*192; } + + /// Get the gigabytes of global memory in the current device + inline double gigabytes() { return gigabytes(_device); } + /// Get the gigabytes of global memory + inline double gigabytes(const int i) + { return static_cast(_properties[i].totalGlobalMem)/1073741824; } + + /// Get the bytes of global memory in the current device + inline size_t bytes() { return bytes(_device); } + /// Get the bytes of global memory + inline size_t bytes(const int i) { return _properties[i].totalGlobalMem; } + + // Get the gigabytes of free memory in the current device + inline double free_gigabytes() { return free_gigabytes(_device); } + // Get the gigabytes of free memory + inline double free_gigabytes(const int i) + { return static_cast(free_bytes(i))/1073741824; } + + // Get the bytes of free memory in the current device + inline size_t free_bytes() { return free_bytes(_device); } + // Get the bytes of free memory + inline size_t free_bytes(const int i) { + CUDA_INT_TYPE dfree, dtotal; + CU_SAFE_CALL_NS(hipMemGetInfo(&dfree, &dtotal)); + return static_cast(dfree); + } + + /// Return the GPGPU compute capability for current device + inline double arch() { return arch(_device); } + /// Return the GPGPU compute capability + inline double arch(const int i) + { return static_cast(_properties[i].minor)/10+_properties[i].major;} + + /// Clock rate in GHz for current device + inline double clock_rate() { return clock_rate(_device); } + /// Clock rate in GHz + inline double clock_rate(const int i) + { return _properties[i].clockRate*1e-6;} + + /// Get the maximum number of threads per block + inline size_t group_size() { return group_size(_device); } + /// Get the maximum number of threads per block + inline size_t group_size(const int i) + { return _properties[i].maxThreadsPerBlock; } + + /// Return the maximum memory pitch in bytes for current device + inline size_t max_pitch() { return max_pitch(_device); } + /// Return the maximum memory pitch in bytes + inline size_t max_pitch(const int i) { return _properties[i].memPitch; } + + /// Returns false if accelerator cannot be shared by multiple processes + /** If it cannot be determined, true is returned **/ + inline bool sharing_supported() { return sharing_supported(_device); } + /// Returns false if accelerator cannot be shared by multiple processes + /** If it cannot be determined, true is returned **/ + inline bool sharing_supported(const int i) + { return (_properties[i].computeMode == hipComputeModeDefault); } + + /// True if splitting device into equal subdevices supported + inline bool fission_equal() + { return fission_equal(_device); } + /// True if splitting device into equal subdevices supported + inline bool fission_equal(const int i) + { return false; } + /// True if splitting device into subdevices by specified counts supported + inline bool fission_by_counts() + { return fission_by_counts(_device); } + /// True if splitting device into subdevices by specified counts supported + inline bool fission_by_counts(const int i) + { return false; } + /// True if splitting device into subdevices by affinity domains supported + inline bool fission_by_affinity() + { return fission_by_affinity(_device); } + /// True if splitting device into subdevices by affinity domains supported + inline bool fission_by_affinity(const int i) + { return false; } + + /// Maximum number of subdevices allowed from device fission + inline int max_sub_devices() + { return max_sub_devices(_device); } + /// Maximum number of subdevices allowed from device fission + inline int max_sub_devices(const int i) + { return 0; } + + /// List all devices along with all properties + inline void print_all(std::ostream &out); + + /// Select the platform that has accelerators (for compatibility with OpenCL) + inline int set_platform_accelerator(int pid=-1) { return UCL_SUCCESS; } + + inline int load_module(const void* program, hipModule_t& module, std::string *log=NULL){ + auto it = _loaded_modules.emplace(program, hipModule_t()); + if(!it.second){ + module = it.first->second; + return UCL_SUCCESS; + } + const unsigned int num_opts=2; + hipJitOption options[num_opts]; + void *values[num_opts]; + + // set up size of compilation log buffer + options[0] = hipJitOptionInfoLogBufferSizeBytes; + values[0] = (void *)(int)10240; + // set up pointer to the compilation log buffer + options[1] = hipJitOptionInfoLogBuffer; + char clog[10240] = { 0 }; + values[1] = clog; + + hipError_t err=hipModuleLoadDataEx(&module,program,num_opts, options,(void **)values); + + if (log!=NULL) + *log=std::string(clog); + + if (err != hipSuccess) { + #ifndef UCL_NO_EXIT + std::cerr << std::endl + << "----------------------------------------------------------\n" + << " UCL Error: Error compiling PTX Program...\n" + << "----------------------------------------------------------\n"; + std::cerr << log << std::endl; + #endif + _loaded_modules.erase(it.first); + return UCL_COMPILE_ERROR; + } + it.first->second = module; + return UCL_SUCCESS; + } + private: + std::unordered_map _loaded_modules; + int _device, _num_devices; + std::vector _properties; + std::vector _cq; + hipDevice_t _cu_device; +}; + +// Grabs the properties for all devices +UCL_Device::UCL_Device() { + CU_SAFE_CALL_NS(hipInit(0)); + CU_SAFE_CALL_NS(hipGetDeviceCount(&_num_devices)); + for (int i=0; i<_num_devices; ++i) { + hipDevice_t dev; + CU_SAFE_CALL_NS(hipDeviceGet(&dev,i)); + int major, minor; + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&major, hipDeviceAttributeComputeCapabilityMajor, dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&minor, hipDeviceAttributeComputeCapabilityMinor, dev)); + if (major==9999) + continue; + + NVDProperties prop; + prop.device_id = i; + prop.major=major; + prop.minor=minor; + + char namecstr[1024]; + CU_SAFE_CALL_NS(hipDeviceGetName(namecstr,1024,dev)); + prop.name=namecstr; + + CU_SAFE_CALL_NS(hipDeviceTotalMem(&prop.totalGlobalMem,dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.multiProcessorCount, hipDeviceAttributeMultiprocessorCount, dev)); + + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.maxThreadsPerBlock, hipDeviceAttributeMaxThreadsPerBlock, dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.maxThreadsDim[0], hipDeviceAttributeMaxBlockDimX, dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.maxThreadsDim[1], hipDeviceAttributeMaxBlockDimY, dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.maxThreadsDim[2], hipDeviceAttributeMaxBlockDimZ, dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.maxGridSize[0], hipDeviceAttributeMaxGridDimX, dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.maxGridSize[1], hipDeviceAttributeMaxGridDimY, dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.maxGridSize[2], hipDeviceAttributeMaxGridDimZ, dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.sharedMemPerBlock, hipDeviceAttributeMaxSharedMemoryPerBlock, dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.totalConstantMemory, hipDeviceAttributeTotalConstantMemory, dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.SIMDWidth, hipDeviceAttributeWarpSize, dev)); + //CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.memPitch, CU_DEVICE_ATTRIBUTE_MAX_PITCH, dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.regsPerBlock, hipDeviceAttributeMaxRegistersPerBlock, dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.clockRate, hipDeviceAttributeClockRate, dev)); + //CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.textureAlign, CU_DEVICE_ATTRIBUTE_TEXTURE_ALIGNMENT, dev)); + + //#if CUDA_VERSION >= 2020 + //CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.kernelExecTimeoutEnabled, CU_DEVICE_ATTRIBUTE_KERNEL_EXEC_TIMEOUT,dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.integrated, hipDeviceAttributeIntegrated, dev)); + //CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.canMapHostMemory, CU_DEVICE_ATTRIBUTE_CAN_MAP_HOST_MEMORY, dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.computeMode, hipDeviceAttributeComputeMode,dev)); + //#endif + //#if CUDA_VERSION >= 3010 + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.concurrentKernels, hipDeviceAttributeConcurrentKernels, dev)); + //CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.ECCEnabled, CU_DEVICE_ATTRIBUTE_ECC_ENABLED, dev)); + //#endif + + _properties.push_back(prop); + } + _device=-1; + _cq.push_back(hipStream_t()); + _cq.back()=0; +} + +UCL_Device::~UCL_Device() { + clear(); +} + +int UCL_Device::set_platform(const int pid) { + clear(); + #ifdef UCL_DEBUG + assert(pid-1) { + for (int i=1; i= 2020 + int driver_version; + hipDriverGetVersion(&driver_version); + out << "Driver Version: " + << driver_version/1000 << "." << driver_version%100 + << std::endl; + //#endif + + if (num_devices() == 0) + out << "There is no device supporting HIP\n"; + for (int i=0; i= 2000 + out << " Number of compute units/multiprocessors: " + << _properties[i].multiProcessorCount << std::endl; + out << " Number of cores: " + << cores(i) << std::endl; + //#endif + out << " Total amount of constant memory: " + << _properties[i].totalConstantMemory << " bytes\n"; + out << " Total amount of local/shared memory per block: " + << _properties[i].sharedMemPerBlock << " bytes\n"; + out << " Total number of registers available per block: " + << _properties[i].regsPerBlock << std::endl; + out << " Warp size: " + << _properties[i].SIMDWidth << std::endl; + out << " Maximum number of threads per block: " + << _properties[i].maxThreadsPerBlock << std::endl; + out << " Maximum group size (# of threads per block) " + << _properties[i].maxThreadsDim[0] << " x " + << _properties[i].maxThreadsDim[1] << " x " + << _properties[i].maxThreadsDim[2] << std::endl; + out << " Maximum item sizes (# threads for each dim) " + << _properties[i].maxGridSize[0] << " x " + << _properties[i].maxGridSize[1] << " x " + << _properties[i].maxGridSize[2] << std::endl; + //out << " Maximum memory pitch: " + // << max_pitch(i) << " bytes\n"; + //out << " Texture alignment: " + // << _properties[i].textureAlign << " bytes\n"; + out << " Clock rate: " + << clock_rate(i) << " GHz\n"; + //#if CUDA_VERSION >= 2020 + //out << " Run time limit on kernels: "; + //if (_properties[i].kernelExecTimeoutEnabled) + // out << "Yes\n"; + //else + // out << "No\n"; + out << " Integrated: "; + if (_properties[i].integrated) + out << "Yes\n"; + else + out << "No\n"; + //out << " Support host page-locked memory mapping: "; + //if (_properties[i].canMapHostMemory) + // out << "Yes\n"; + //else + // out << "No\n"; + out << " Compute mode: "; + if (_properties[i].computeMode == hipComputeModeDefault) + out << "Default\n"; // multiple threads can use device +//#if CUDA_VERSION >= 8000 +// else if (_properties[i].computeMode == hipComputeModeExclusiveProcess) +//#else + else if (_properties[i].computeMode == hipComputeModeExclusive) +//#endif + out << "Exclusive\n"; // only thread can use device + else if (_properties[i].computeMode == hipComputeModeProhibited) + out << "Prohibited\n"; // no thread can use device + //#if CUDART_VERSION >= 4000 + else if (_properties[i].computeMode == hipComputeModeExclusiveProcess) + out << "Exclusive Process\n"; // multiple threads 1 process + //#endif + else + out << "Unknown\n"; + //#endif + //#if CUDA_VERSION >= 3010 + out << " Concurrent kernel execution: "; + if (_properties[i].concurrentKernels) + out << "Yes\n"; + else + out << "No\n"; + //out << " Device has ECC support enabled: "; + //if (_properties[i].ECCEnabled) + // out << "Yes\n"; + //else + // out << "No\n"; + //#endif + } +} + +} + +#endif diff --git a/lib/gpu/geryon/hip_kernel.h b/lib/gpu/geryon/hip_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..654eb44772371461d32078057f91be6289870c1b --- /dev/null +++ b/lib/gpu/geryon/hip_kernel.h @@ -0,0 +1,298 @@ +/* ----------------------------------------------------------------------- + Copyright (2010) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the Simplified BSD License. + ----------------------------------------------------------------------- */ + +#ifndef HIP_KERNEL +#define HIP_KERNEL + + +#include +#include "hip_device.h" +#include +#include +#include + +namespace ucl_hip { + +class UCL_Texture; +template class UCL_D_Vec; +template class UCL_D_Mat; +template class UCL_Vector; +template class UCL_Matrix; +#define UCL_MAX_KERNEL_ARGS 256 + +/// Class storing 1 or more kernel functions from a single string or file +class UCL_Program { + UCL_Device* _device_ptr; + public: + inline UCL_Program(UCL_Device &device) { _device_ptr = &device; _cq=device.cq(); } + inline UCL_Program(UCL_Device &device, const void *program, + const char *flags="", std::string *log=NULL) { + _device_ptr = &device; _cq=device.cq(); + init(device); + load_string(program,flags,log); + } + + inline ~UCL_Program() {} + + /// Initialize the program with a device + inline void init(UCL_Device &device) { _device_ptr = &device; _cq=device.cq(); } + + /// Clear any data associated with program + /** \note Must call init() after each clear **/ + inline void clear() { } + + /// Load a program from a file and compile with flags + inline int load(const char *filename, const char *flags="", std::string *log=NULL) { + std::ifstream in(filename); + if (!in || in.is_open()==false) { + #ifndef UCL_NO_EXIT + std::cerr << "UCL Error: Could not open kernel file: " + << filename << std::endl; + UCL_GERYON_EXIT; + #endif + return UCL_FILE_NOT_FOUND; + } + + std::string program((std::istreambuf_iterator(in)), + std::istreambuf_iterator()); + in.close(); + return load_string(program.c_str(),flags,log); + } + + /// Load a program from a string and compile with flags + inline int load_string(const void *program, const char *flags="", std::string *log=NULL) { + return _device_ptr->load_module(program, _module, log); + } + + friend class UCL_Kernel; + private: + hipModule_t _module; + hipStream_t _cq; + friend class UCL_Texture; +}; + +/// Class for dealing with CUDA Driver kernels +class UCL_Kernel { + public: + UCL_Kernel() : _dimensions(1), _num_args(0) { + _num_blocks[0]=0; + } + + UCL_Kernel(UCL_Program &program, const char *function) : + _dimensions(1), _num_args(0) { + _num_blocks[0]=0; + set_function(program,function); + _cq=program._cq; + } + + ~UCL_Kernel() {} + + /// Clear any function associated with the kernel + inline void clear() { } + + /// Get the kernel function from a program + /** \ret UCL_ERROR_FLAG (UCL_SUCCESS, UCL_FILE_NOT_FOUND, UCL_ERROR) **/ + inline int set_function(UCL_Program &program, const char *function) { + hipError_t err=hipModuleGetFunction(&_kernel,program._module,function); + if (err!=hipSuccess) { + #ifndef UCL_NO_EXIT + std::cerr << "UCL Error: Could not find function: " << function + << " in program.\n"; + UCL_GERYON_EXIT; + #endif + return UCL_FUNCTION_NOT_FOUND; + } + _cq=program._cq; + return UCL_SUCCESS; + } + + /// Set the kernel argument. + /** If not a device pointer, this must be repeated each time the argument + * changes + * \note To set kernel parameter i (i>0), parameter i-1 must be set **/ + template + inline void set_arg(const unsigned index, const dtype * const arg) { + if (index==_num_args) + add_arg(arg); + else if (index<_num_args){ + assert(0==1); // not implemented + } + else + assert(0==1); // Must add kernel parameters in sequential order + } + + /// Set a geryon container as a kernel argument. + template + inline void set_arg(const UCL_D_Vec * const arg) + { set_arg(&arg->begin()); } + + /// Set a geryon container as a kernel argument. + template + inline void set_arg(const UCL_D_Mat * const arg) + { set_arg(&arg->begin()); } + + /// Set a geryon container as a kernel argument. + template + inline void set_arg(const UCL_Vector * const arg) + { set_arg(&arg->device.begin()); } + + /// Set a geryon container as a kernel argument. + template + inline void set_arg(const UCL_Matrix * const arg) + { set_arg(&arg->device.begin()); } + + /// Add a kernel argument. + inline void add_arg(const hipDeviceptr_t* const arg) { + add_arg((void**)arg); + } + + /// Add a kernel argument. + template + inline void add_arg(const dtype* const arg) { + const auto old_size = _hip_kernel_args.size(); + const auto aligned_size = (old_size+alignof(dtype)-1) & ~(alignof(dtype)-1); + const auto arg_size = sizeof(dtype); + _hip_kernel_args.resize(aligned_size + arg_size); + *((dtype*)(&_hip_kernel_args[aligned_size])) = *arg; + _num_args++; + if (_num_args>UCL_MAX_KERNEL_ARGS) assert(0==1); + } + + /// Add a geryon container as a kernel argument. + template + inline void add_arg(const UCL_D_Vec * const arg) + { add_arg(&arg->begin()); } + + /// Add a geryon container as a kernel argument. + template + inline void add_arg(const UCL_D_Mat * const arg) + { add_arg(&arg->begin()); } + + /// Add a geryon container as a kernel argument. + template + inline void add_arg(const UCL_Vector * const arg) + { add_arg(&arg->device.begin()); } + + /// Add a geryon container as a kernel argument. + template + inline void add_arg(const UCL_Matrix * const arg) + { add_arg(&arg->device.begin()); } + + /// Set the number of thread blocks and the number of threads in each block + /** \note This should be called before any arguments have been added + \note The default command queue is used for the kernel execution **/ + inline void set_size(const size_t num_blocks, const size_t block_size) { + _dimensions=1; + _num_blocks[0]=num_blocks; + _num_blocks[1]=1; + _num_blocks[2]=1; + + _block_size[0]=block_size; + _block_size[1]=1; + _block_size[2]=1; + } + + /// Set the number of thread blocks and the number of threads in each block + /** \note This should be called before any arguments have been added + \note The default command queue for the kernel is changed to cq **/ + inline void set_size(const size_t num_blocks, const size_t block_size, + command_queue &cq) + { _cq=cq; set_size(num_blocks,block_size); } + + /// Set the number of thread blocks and the number of threads in each block + /** \note This should be called before any arguments have been added + \note The default command queue is used for the kernel execution **/ + inline void set_size(const size_t num_blocks_x, const size_t num_blocks_y, + const size_t block_size_x, const size_t block_size_y) { + _dimensions=2; + _num_blocks[0]=num_blocks_x; + _num_blocks[1]=num_blocks_y; + _num_blocks[2]=1; + + _block_size[0]=block_size_x; + _block_size[1]=block_size_y; + _block_size[2]=1; + } + + /// Set the number of thread blocks and the number of threads in each block + /** \note This should be called before any arguments have been added + \note The default command queue for the kernel is changed to cq **/ + inline void set_size(const size_t num_blocks_x, const size_t num_blocks_y, + const size_t block_size_x, const size_t block_size_y, + command_queue &cq) + {_cq=cq; set_size(num_blocks_x, num_blocks_y, block_size_x, block_size_y);} + + /// Set the number of thread blocks and the number of threads in each block + /** \note This should be called before any arguments have been added + \note The default command queue is used for the kernel execution **/ + inline void set_size(const size_t num_blocks_x, const size_t num_blocks_y, + const size_t block_size_x, + const size_t block_size_y, const size_t block_size_z) { + _dimensions=2; + _num_blocks[0]=num_blocks_x; + _num_blocks[1]=num_blocks_y; + _num_blocks[2]=1; + + _block_size[0]=block_size_x; + _block_size[1]=block_size_y; + _block_size[2]=block_size_z; + } + + /// Set the number of thread blocks and the number of threads in each block + /** \note This should be called before any arguments have been added + \note The default command queue is used for the kernel execution **/ + inline void set_size(const size_t num_blocks_x, const size_t num_blocks_y, + const size_t block_size_x, const size_t block_size_y, + const size_t block_size_z, command_queue &cq) { + _cq=cq; + set_size(num_blocks_x, num_blocks_y, block_size_x, block_size_y, + block_size_z); + } + + /// Run the kernel in the default command queue + inline void run() { + size_t args_size = _hip_kernel_args.size(); + void *config[] = { + HIP_LAUNCH_PARAM_BUFFER_POINTER, (void*)_hip_kernel_args.data(), + HIP_LAUNCH_PARAM_BUFFER_SIZE, &args_size, + HIP_LAUNCH_PARAM_END + }; + const auto res = hipModuleLaunchKernel(_kernel,_num_blocks[0],_num_blocks[1], + _num_blocks[2],_block_size[0],_block_size[1], + _block_size[2],0,_cq, NULL, config); + CU_SAFE_CALL(res); +//#endif + } + + /// Clear any arguments associated with the kernel + inline void clear_args() { + _num_args=0; + _hip_kernel_args.clear(); + } + + /// Return the default command queue/stream associated with this data + inline command_queue & cq() { return _cq; } + /// Change the default command queue associated with matrix + inline void cq(command_queue &cq_in) { _cq=cq_in; } + #include "ucl_arg_kludge.h" + + private: + hipFunction_t _kernel; + hipStream_t _cq; + unsigned _dimensions; + unsigned _num_blocks[3]; + unsigned _num_args; + friend class UCL_Texture; + + unsigned _block_size[3]; + std::vector _hip_kernel_args; +}; + +} // namespace + +#endif + diff --git a/lib/gpu/geryon/hip_macros.h b/lib/gpu/geryon/hip_macros.h new file mode 100644 index 0000000000000000000000000000000000000000..9c9971b89638d08cbe444b1c5a3617bdf6c11b3b --- /dev/null +++ b/lib/gpu/geryon/hip_macros.h @@ -0,0 +1,83 @@ +#ifndef HIP_MACROS_H +#define HIP_MACROS_H + +#include +#include +#include + +//#if CUDA_VERSION >= 3020 +#define CUDA_INT_TYPE size_t +//#else +//#define CUDA_INT_TYPE unsigned +//#endif + +#ifdef MPI_GERYON +#include "mpi.h" +#define NVD_GERYON_EXIT do { \ + int is_final; \ + MPI_Finalized(&is_final); \ + if (!is_final) \ + MPI_Abort(MPI_COMM_WORLD,-1); \ + } while(0) +#else +#define NVD_GERYON_EXIT assert(0==1) +#endif + +#ifndef UCL_GERYON_EXIT +#define UCL_GERYON_EXIT NVD_GERYON_EXIT +#endif + +#ifdef UCL_DEBUG +#define UCL_SYNC_DEBUG +#define UCL_DESTRUCT_CHECK +#endif + +#ifndef UCL_NO_API_CHECK + +#define CU_SAFE_CALL_NS( call ) do { \ + hipError_t err = call; \ + if( hipSuccess != err) { \ + fprintf(stderr, "HIP runtime error %d in call at file '%s' in line %i.\n", \ + err, __FILE__, __LINE__ ); \ + NVD_GERYON_EXIT; \ + } } while (0) + +#ifdef UCL_SYNC_DEBUG + +#define CU_SAFE_CALL( call ) do { \ + CU_SAFE_CALL_NS( call ); \ + hipError_t err=hipCtxSynchronize(); \ + if( hipSuccess != err) { \ + fprintf(stderr, "HIP runtime error %d in file '%s' in line %i.\n", \ + err, __FILE__, __LINE__ ); \ + NVD_GERYON_EXIT; \ + } } while (0) + +#else + +#define CU_SAFE_CALL( call ) CU_SAFE_CALL_NS( call ) + +#endif + +#else // not DEBUG + +// void macros for performance reasons +#define CU_SAFE_CALL_NS( call ) call +#define CU_SAFE_CALL( call) call + +#endif + +#ifdef UCL_DESTRUCT_CHECK + +#define CU_DESTRUCT_CALL( call) CU_SAFE_CALL( call) +#define CU_DESTRUCT_CALL_NS( call) CU_SAFE_CALL_NS( call) + +#else + +#define CU_DESTRUCT_CALL( call) call +#define CU_DESTRUCT_CALL_NS( call) call + +#endif + +#endif + diff --git a/lib/gpu/geryon/hip_mat.h b/lib/gpu/geryon/hip_mat.h new file mode 100644 index 0000000000000000000000000000000000000000..d9bbb4e5218f940fcab9791b4945562c7fb2e03e --- /dev/null +++ b/lib/gpu/geryon/hip_mat.h @@ -0,0 +1,43 @@ +/* ----------------------------------------------------------------------- + Copyright (2010) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the Simplified BSD License. + ----------------------------------------------------------------------- */ + +/*! \file */ + +#ifndef HIP_MAT_H +#define HIP_MAT_H + + +#include +#include "hip_memory.h" + +/// Namespace for CUDA Driver routines +namespace ucl_hip { + +#define _UCL_MAT_ALLOW +#define _UCL_DEVICE_PTR_MAT +#include "ucl_basemat.h" +#include "ucl_h_vec.h" +#include "ucl_h_mat.h" +#include "ucl_d_vec.h" +#include "ucl_d_mat.h" +#include "ucl_s_obj_help.h" +#include "ucl_vector.h" +#include "ucl_matrix.h" +#undef _UCL_DEVICE_PTR_MAT +#undef _UCL_MAT_ALLOW + +#define UCL_COPY_ALLOW +#include "ucl_copy.h" +#undef UCL_COPY_ALLOW + +#define UCL_PRINT_ALLOW +#include "ucl_print.h" +#undef UCL_PRINT_ALLOW + +} // namespace ucl_cudadr + +#endif diff --git a/lib/gpu/geryon/hip_memory.h b/lib/gpu/geryon/hip_memory.h new file mode 100644 index 0000000000000000000000000000000000000000..13f60ad93963ecbfcbdc5ab6e44dcd39161bfffb --- /dev/null +++ b/lib/gpu/geryon/hip_memory.h @@ -0,0 +1,279 @@ +/* ----------------------------------------------------------------------- + Copyright (2010) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the Simplified BSD License. + ----------------------------------------------------------------------- */ + +#ifndef HIP_MEMORY_H +#define HIP_MEMORY_H + + +#include +#include +#include +#include +#include "hip_macros.h" +#include "hip_device.h" +#include "ucl_types.h" + +namespace ucl_hip { + +// -------------------------------------------------------------------------- +// - API Specific Types +// -------------------------------------------------------------------------- +//typedef dim3 ucl_kernel_dim; + +#ifdef __HIP_PLATFORM_NVCC__ +typedef enum hipArray_Format { + HIP_AD_FORMAT_UNSIGNED_INT8 = 0x01, + HIP_AD_FORMAT_UNSIGNED_INT16 = 0x02, + HIP_AD_FORMAT_UNSIGNED_INT32 = 0x03, + HIP_AD_FORMAT_SIGNED_INT8 = 0x08, + HIP_AD_FORMAT_SIGNED_INT16 = 0x09, + HIP_AD_FORMAT_SIGNED_INT32 = 0x0a, + HIP_AD_FORMAT_HALF = 0x10, + HIP_AD_FORMAT_FLOAT = 0x20 +}hipArray_Format; +#endif + +// -------------------------------------------------------------------------- +// - API SPECIFIC DEVICE POINTERS +// -------------------------------------------------------------------------- +typedef hipDeviceptr_t device_ptr; + +// -------------------------------------------------------------------------- +// - HOST MEMORY ALLOCATION ROUTINES +// -------------------------------------------------------------------------- +template +inline int _host_alloc(mat_type &mat, copy_type &cm, const size_t n, + const enum UCL_MEMOPT kind, const enum UCL_MEMOPT kind2){ + hipError_t err=hipSuccess; + if (kind==UCL_NOT_PINNED) + *(mat.host_ptr())=(typename mat_type::data_type*)malloc(n); + else if (kind==UCL_WRITE_ONLY) + err=hipHostMalloc((void **)mat.host_ptr(),n,hipHostMallocWriteCombined); + else + err=hipHostMalloc((void **)mat.host_ptr(),n,hipHostMallocDefault); + if (err!=hipSuccess || *(mat.host_ptr())==NULL) + return UCL_MEMORY_ERROR; + mat.cq()=cm.cq(); + return UCL_SUCCESS; +} + +template +inline int _host_alloc(mat_type &mat, UCL_Device &dev, const size_t n, + const enum UCL_MEMOPT kind, const enum UCL_MEMOPT kind2){ + hipError_t err=hipSuccess; + if (kind==UCL_NOT_PINNED) + *(mat.host_ptr())=(typename mat_type::data_type*)malloc(n); + else if (kind==UCL_WRITE_ONLY) + err=hipHostMalloc((void **)mat.host_ptr(),n,hipHostMallocWriteCombined); + else + err=hipHostMalloc((void **)mat.host_ptr(),n,hipHostMallocDefault); + if (err!=hipSuccess || *(mat.host_ptr())==NULL) + return UCL_MEMORY_ERROR; + mat.cq()=dev.cq(); + return UCL_SUCCESS; +} + +template +inline void _host_free(mat_type &mat) { + if (mat.kind()==UCL_VIEW) + return; + else if (mat.kind()!=UCL_NOT_PINNED) + CU_DESTRUCT_CALL(hipHostFree(mat.begin())); + else + free(mat.begin()); +} + +template +inline int _host_resize(mat_type &mat, const size_t n) { + _host_free(mat); + hipError_t err=hipSuccess; + if (mat.kind()==UCL_NOT_PINNED) + *(mat.host_ptr())=(typename mat_type::data_type*)malloc(n); + else if (mat.kind()==UCL_WRITE_ONLY) + err=hipHostMalloc((void **)mat.host_ptr(),n,hipHostMallocWriteCombined); + else + err=hipHostMalloc((void **)mat.host_ptr(),n,hipHostMallocDefault); + if (err!=hipSuccess || *(mat.host_ptr())==NULL) + return UCL_MEMORY_ERROR; + return UCL_SUCCESS; +} + +// -------------------------------------------------------------------------- +// - DEVICE MEMORY ALLOCATION ROUTINES +// -------------------------------------------------------------------------- +template +inline int _device_alloc(mat_type &mat, copy_type &cm, const size_t n, + const enum UCL_MEMOPT kind) { + hipError_t err=hipMalloc((void**)&mat.cbegin(),n); + if (err!=hipSuccess) + return UCL_MEMORY_ERROR; + mat.cq()=cm.cq(); + return UCL_SUCCESS; +} + +template +inline int _device_alloc(mat_type &mat, UCL_Device &dev, const size_t n, + const enum UCL_MEMOPT kind) { + hipError_t err=hipMalloc((void**)&mat.cbegin(),n); + if (err!=hipSuccess) + return UCL_MEMORY_ERROR; + mat.cq()=dev.cq(); + return UCL_SUCCESS; +} + +template +inline int _device_alloc(mat_type &mat, copy_type &cm, const size_t rows, + const size_t cols, size_t &pitch, + const enum UCL_MEMOPT kind) { + hipError_t err; + size_t upitch; + err=hipMallocPitch((void**)&mat.cbegin(),&upitch, + cols*sizeof(typename mat_type::data_type),rows); + pitch=static_cast(upitch); + if (err!=hipSuccess) + return UCL_MEMORY_ERROR; + mat.cq()=cm.cq(); + return UCL_SUCCESS; +} + +template +inline int _device_alloc(mat_type &mat, UCL_Device &d, const size_t rows, + const size_t cols, size_t &pitch, + const enum UCL_MEMOPT kind) { + hipError_t err; + size_t upitch; + err=hipMallocPitch((void**)&mat.cbegin(),&upitch, + cols*sizeof(typename mat_type::data_type),rows); + pitch=static_cast(upitch); + if (err!=hipSuccess) + return UCL_MEMORY_ERROR; + mat.cq()=d.cq(); + return UCL_SUCCESS; +} + +template +inline void _device_free(mat_type &mat) { + if (mat.kind()!=UCL_VIEW){ + CU_DESTRUCT_CALL(hipFree((void*)mat.cbegin())); + } +} + +template +inline int _device_resize(mat_type &mat, const size_t n) { + _device_free(mat); + hipError_t err=hipMalloc((void**)&mat.cbegin(),n); + if (err!=hipSuccess) + return UCL_MEMORY_ERROR; + return UCL_SUCCESS; +} + +template +inline int _device_resize(mat_type &mat, const size_t rows, + const size_t cols, size_t &pitch) { + _device_free(mat); + hipError_t err; + size_t upitch; + err=hipMallocPitch((void**)&mat.cbegin(),&upitch, + cols*sizeof(typename mat_type::data_type),rows); + pitch=static_cast(upitch); + if (err!=hipSuccess) + return UCL_MEMORY_ERROR; + return UCL_SUCCESS; +} + +inline void _device_view(hipDeviceptr_t *ptr, hipDeviceptr_t &in) { + *ptr=in; +} + +template +inline void _device_view(hipDeviceptr_t *ptr, numtyp *in) { + *ptr=0; +} + +inline void _device_view(hipDeviceptr_t *ptr, hipDeviceptr_t &in, + const size_t offset, const size_t numsize) { + *ptr=(hipDeviceptr_t)(((char*)in)+offset*numsize); +} + +template +inline void _device_view(hipDeviceptr_t *ptr, numtyp *in, + const size_t offset, const size_t numsize) { + *ptr=0; +} + +// -------------------------------------------------------------------------- +// - DEVICE IMAGE ALLOCATION ROUTINES +// -------------------------------------------------------------------------- +template +inline void _device_image_alloc(mat_type &mat, copy_type &cm, const size_t rows, + const size_t cols) { + assert(0==1); +} + +template +inline void _device_image_alloc(mat_type &mat, UCL_Device &d, const size_t rows, + const size_t cols) { + assert(0==1); +} + +template +inline void _device_image_free(mat_type &mat) { + assert(0==1); +} + +// -------------------------------------------------------------------------- +// - ZERO ROUTINES +// -------------------------------------------------------------------------- +inline void _host_zero(void *ptr, const size_t n) { + memset(ptr,0,n); +} + +template +inline void _device_zero(mat_type &mat, const size_t n, command_queue &cq) { + CU_SAFE_CALL(hipMemsetAsync((void*)mat.cbegin(),0,n,cq)); +} + + +// -------------------------------------------------------------------------- +// - MEMCPY ROUTINES +// -------------------------------------------------------------------------- + + +template +hipMemcpyKind _memcpy_kind(mat1 &dst, const mat2 &src){ + assert(mat1::MEM_TYPE < 2 && mat2::MEM_TYPE < 2); + return (hipMemcpyKind)((1 - mat2::MEM_TYPE)*2 + (1 - mat1::MEM_TYPE)); +} + +template +inline void ucl_mv_cpy(mat1 &dst, const mat2 &src, const size_t n) { + CU_SAFE_CALL(hipMemcpy((void*)dst.begin(), (void*)src.begin(), n, _memcpy_kind(dst, src))); +} + +template +inline void ucl_mv_cpy(mat1 &dst, const mat2 &src, const size_t n, hipStream_t &cq) { + CU_SAFE_CALL(hipMemcpyAsync((void*)dst.begin(), (void*)src.begin(), n, _memcpy_kind(dst, src), cq)); +} + +template +inline void ucl_mv_cpy(mat1 &dst, const size_t dpitch, const mat2 &src, + const size_t spitch, const size_t cols, + const size_t rows) { + CU_SAFE_CALL(hipMemcpy2D((void*)dst.begin(), dpitch, (void*)src.begin(), spitch, cols, rows, _memcpy_kind(dst, src))); +} + +template +inline void ucl_mv_cpy(mat1 &dst, const size_t dpitch, const mat2 &src, + const size_t spitch, const size_t cols, + const size_t rows,hipStream_t &cq) { + CU_SAFE_CALL(hipMemcpy2DAsync((void*)dst.begin(), dpitch, (void*)src.begin(), spitch, cols, rows, _memcpy_kind(dst, src), cq)); +} + +} // namespace ucl_cudart + +#endif + diff --git a/lib/gpu/geryon/hip_texture.h b/lib/gpu/geryon/hip_texture.h new file mode 100644 index 0000000000000000000000000000000000000000..e7aa4e1461e25c2c0cdcbee35761d64832bc6b8c --- /dev/null +++ b/lib/gpu/geryon/hip_texture.h @@ -0,0 +1,113 @@ +/* ----------------------------------------------------------------------- + Copyright (2010) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the Simplified BSD License. + ----------------------------------------------------------------------- */ + +#ifndef HIP_TEXTURE +#define HIP_TEXTURE + + +#include +#include "hip_kernel.h" +#include "hip_mat.h" + +namespace ucl_hip { + +#ifdef __HIP_PLATFORM_NVCC__ +inline hipError_t hipModuleGetTexRef(CUtexref* texRef, hipModule_t hmod, const char* name){ + return hipCUResultTohipError(cuModuleGetTexRef(texRef, hmod, name)); +} +inline hipError_t hipTexRefSetFormat(CUtexref tex, hipArray_Format fmt, int NumPackedComponents) { + return hipCUResultTohipError(cuTexRefSetFormat(tex, (CUarray_format)fmt, NumPackedComponents )); +} +inline hipError_t hipTexRefSetAddress(size_t* offset, CUtexref tex, hipDeviceptr_t devPtr, size_t size) { + return hipCUResultTohipError(cuTexRefSetAddress(offset, tex, devPtr, size)); +} +#endif + +/// Class storing a texture reference +class UCL_Texture { + public: + UCL_Texture() {} + ~UCL_Texture() {} + /// Construct with a specified texture reference + inline UCL_Texture(UCL_Program &prog, const char *texture_name) + { get_texture(prog,texture_name); } + /// Set the texture reference for this object + inline void get_texture(UCL_Program &prog, const char *texture_name) + { + #ifdef __HIP_PLATFORM_NVCC__ + CU_SAFE_CALL(hipModuleGetTexRef(&_tex, prog._module, texture_name)); + #else + size_t _global_var_size; + CU_SAFE_CALL(hipModuleGetGlobal(&_device_ptr_to_global_var, &_global_var_size, prog._module, texture_name)); + #endif + } + + /// Bind a float array where each fetch grabs a vector of length numel + template + inline void bind_float(UCL_D_Vec &vec, const unsigned numel) + { _bind_float(vec,numel); } + + /// Bind a float array where each fetch grabs a vector of length numel + template + inline void bind_float(UCL_D_Mat &vec, const unsigned numel) + { _bind_float(vec,numel); } + + /// Bind a float array where each fetch grabs a vector of length numel + template + inline void bind_float(UCL_Vector &vec, const unsigned numel) + { _bind_float(vec.device,numel); } + + /// Bind a float array where each fetch grabs a vector of length numel + template + inline void bind_float(UCL_Matrix &vec, const unsigned numel) + { _bind_float(vec.device,numel); } + + /// Unbind the texture reference from the memory allocation + inline void unbind() { } + + /// Make a texture reference available to kernel + inline void allow(UCL_Kernel &kernel) { + //#if CUDA_VERSION < 4000 + //CU_SAFE_CALL(cuParamSetTexRef(kernel._kernel, CU_PARAM_TR_DEFAULT, _tex)); + //#endif + } + + private: +#ifdef __HIP_PLATFORM_NVCC__ + CUtexref _tex; +#else + void* _device_ptr_to_global_var; +#endif + friend class UCL_Kernel; + + template + inline void _bind_float(mat_typ &vec, const unsigned numel) { + #ifdef UCL_DEBUG + assert(numel!=0 && numel<5); + #endif + +#ifdef __HIP_PLATFORM_NVCC__ + if (vec.element_size()==sizeof(float)) + CU_SAFE_CALL(hipTexRefSetFormat(_tex, HIP_AD_FORMAT_FLOAT, numel)); + else { + if (numel>2) + CU_SAFE_CALL(hipTexRefSetFormat(_tex, HIP_AD_FORMAT_SIGNED_INT32, numel)); + else + CU_SAFE_CALL(hipTexRefSetFormat(_tex,HIP_AD_FORMAT_SIGNED_INT32,numel*2)); + } + CU_SAFE_CALL(hipTexRefSetAddress(NULL, _tex, vec.cbegin(), vec.numel()*vec.element_size())); +#else + void* data_ptr = (void*)vec.cbegin(); + CU_SAFE_CALL(hipMemcpyHtoD(hipDeviceptr_t(_device_ptr_to_global_var), &data_ptr, sizeof(void*))); +#endif + } +}; + +} // namespace + +#endif + diff --git a/lib/gpu/geryon/hip_timer.h b/lib/gpu/geryon/hip_timer.h new file mode 100644 index 0000000000000000000000000000000000000000..3be0b8cfd665d5d630f45fdb160845fc15ac7a44 --- /dev/null +++ b/lib/gpu/geryon/hip_timer.h @@ -0,0 +1,107 @@ +/* ----------------------------------------------------------------------- + Copyright (2010) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the Simplified BSD License. + ----------------------------------------------------------------------- */ + +#ifndef HIP_TIMER_H +#define HIP_TIMER_H + + +#include +#include "hip_macros.h" +#include "hip_device.h" + +namespace ucl_hip { + +/// Class for timing CUDA Driver events +class UCL_Timer { + public: + inline UCL_Timer() : _total_time(0.0f), _initialized(false) { } + inline UCL_Timer(UCL_Device &dev) : _total_time(0.0f), _initialized(false) + { init(dev); } + + inline ~UCL_Timer() { clear(); } + + /// Clear any data associated with timer + /** \note init() must be called to reuse timer after a clear() **/ + inline void clear() { + if (_initialized) { + CU_DESTRUCT_CALL(hipEventDestroy(start_event)); + CU_DESTRUCT_CALL(hipEventDestroy(stop_event)); + _initialized=false; + _total_time=0.0; + } + } + + /// Initialize default command queue for timing + inline void init(UCL_Device &dev) { init(dev, dev.cq()); } + + /// Initialize command queue for timing + inline void init(UCL_Device &dev, command_queue &cq) { + clear(); + _cq=cq; + _initialized=true; + CU_SAFE_CALL( hipEventCreateWithFlags(&start_event,0) ); + CU_SAFE_CALL( hipEventCreateWithFlags(&stop_event,0) ); + } + + /// Start timing on command queue + inline void start() { CU_SAFE_CALL(hipEventRecord(start_event,_cq)); } + + /// Stop timing on command queue + inline void stop() { CU_SAFE_CALL(hipEventRecord(stop_event,_cq)); } + + /// Block until the start event has been reached on device + inline void sync_start() + { CU_SAFE_CALL(hipEventSynchronize(start_event)); } + + /// Block until the stop event has been reached on device + inline void sync_stop() + { CU_SAFE_CALL(hipEventSynchronize(stop_event)); } + + /// Set the time elapsed to zero (not the total_time) + inline void zero() { + CU_SAFE_CALL(hipEventRecord(start_event,_cq)); + CU_SAFE_CALL(hipEventRecord(stop_event,_cq)); + } + + /// Set the total time to zero + inline void zero_total() { _total_time=0.0; } + + /// Add time from previous start and stop to total + /** Forces synchronization **/ + inline double add_to_total() + { double t=time(); _total_time+=t; return t/1000.0; } + + /// Add a user specified time to the total (ms) + inline void add_time_to_total(const double t) { _total_time+=t; } + + /// Return the time (ms) of last start to stop - Forces synchronization + inline double time() { + float timer; + CU_SAFE_CALL(hipEventSynchronize(stop_event)); + CU_SAFE_CALL( hipEventElapsedTime(&timer,start_event,stop_event) ); + return timer; + } + + /// Return the time (s) of last start to stop - Forces synchronization + inline double seconds() { return time()/1000.0; } + + /// Return the total time in ms + inline double total_time() { return _total_time; } + + /// Return the total time in seconds + inline double total_seconds() { return _total_time/1000.0; } + + private: + hipEvent_t start_event, stop_event; + hipStream_t _cq; + double _total_time; + bool _initialized; +}; + +} // namespace + +#endif diff --git a/lib/gpu/geryon/ucl_get_devices.cpp b/lib/gpu/geryon/ucl_get_devices.cpp index 1fa758fb462fbea69014d7d9d647a797cc33c055..b8dfc6f7b12a7c8a8e140fd5b4f7f92618884e53 100644 --- a/lib/gpu/geryon/ucl_get_devices.cpp +++ b/lib/gpu/geryon/ucl_get_devices.cpp @@ -36,6 +36,11 @@ using namespace ucl_cudadr; using namespace ucl_cudart; #endif +#ifdef UCL_HIP +#include "hip_device.h" +using namespace ucl_hip; +#endif + int main(int argc, char** argv) { UCL_Device cop; std::cout << "Found " << cop.num_platforms() << " platform(s).\n"; diff --git a/lib/gpu/lal_answer.cpp b/lib/gpu/lal_answer.cpp index aa6d33d33492890080ed09661a6d715939d3a7c4..95d40c0d0a765ea69bc22808e776ac58dd88f5fb 100644 --- a/lib/gpu/lal_answer.cpp +++ b/lib/gpu/lal_answer.cpp @@ -179,13 +179,15 @@ double AnswerT::energy_virial(double *eatom, double **vatom, if (_eflag) { for (int i=0; i<_inum; i++) evdwl+=engv[i]; - if (_ef_atom) - if (_ilist==NULL) + if (_ef_atom) { + if (_ilist==NULL) { for (int i=0; i<_inum; i++) eatom[i]+=engv[i]; - else + } else { for (int i=0; i<_inum; i++) eatom[_ilist[i]]+=engv[i]; + } + } vstart=_inum; } if (_vflag) { @@ -193,7 +195,7 @@ double AnswerT::energy_virial(double *eatom, double **vatom, for (int j=0; j<6; j++) { for (int i=vstart; i +#include +#endif + namespace LAMMPS_AL { #define AtomT Atom @@ -70,6 +75,26 @@ bool AtomT::alloc(const int nall) { } #endif + #ifdef USE_HIP_DEVICE_SORT + if (_gpu_nbor==1) { + size_t temp_storage_bytes = 0; + if(hipSuccess != hipcub::DeviceRadixSort::SortPairs(nullptr, temp_storage_bytes, sort_out_keys, sort_out_keys, sort_out_values, sort_out_values, _max_atoms)) + return false; + if(sort_out_size < _max_atoms){ + if (sort_out_keys ) hipFree(sort_out_keys); + if (sort_out_values) hipFree(sort_out_values); + hipMalloc(&sort_out_keys , _max_atoms * sizeof(unsigned)); + hipMalloc(&sort_out_values, _max_atoms * sizeof(int )); + sort_out_size = _max_atoms; + } + if(temp_storage_bytes > sort_temp_storage_size){ + if(sort_temp_storage) hipFree(sort_temp_storage); + hipMalloc(&sort_temp_storage, temp_storage_bytes); + sort_temp_storage_size = temp_storage_bytes; + } + } + #endif + // --------------------------- Device allocations int gpu_bytes=0; success=success && (x.alloc(_max_atoms*4,*dev,UCL_WRITE_ONLY, @@ -184,6 +209,27 @@ bool AtomT::add_fields(const bool charge, const bool rot, return false; } #endif + + #ifdef USE_HIP_DEVICE_SORT + if (_gpu_nbor==1) { + size_t temp_storage_bytes = 0; + if(hipSuccess != hipcub::DeviceRadixSort::SortPairs(nullptr, temp_storage_bytes, sort_out_keys, sort_out_keys, sort_out_values, sort_out_values, _max_atoms)) + return false; + if(sort_out_size < _max_atoms){ + if (sort_out_keys ) hipFree(sort_out_keys); + if (sort_out_values) hipFree(sort_out_values); + hipMalloc(&sort_out_keys , _max_atoms * sizeof(unsigned)); + hipMalloc(&sort_out_values, _max_atoms * sizeof(int )); + sort_out_size = _max_atoms; + } + if(temp_storage_bytes > sort_temp_storage_size){ + if(sort_temp_storage) hipFree(sort_temp_storage); + hipMalloc(&sort_temp_storage, temp_storage_bytes); + sort_temp_storage_size = temp_storage_bytes; + } + } + #endif + success=success && (dev_particle_id.alloc(_max_atoms,*dev, UCL_READ_ONLY)==UCL_SUCCESS); gpu_bytes+=dev_particle_id.row_bytes(); @@ -275,6 +321,19 @@ void AtomT::clear_resize() { if (_gpu_nbor==1) cudppDestroyPlan(sort_plan); #endif + #ifdef USE_HIP_DEVICE_SORT + if (_gpu_nbor==1) { + if(sort_out_keys) hipFree(sort_out_keys); + if(sort_out_values) hipFree(sort_out_values); + if(sort_temp_storage) hipFree(sort_temp_storage); + sort_out_keys = nullptr; + sort_out_values = nullptr; + sort_temp_storage = nullptr; + sort_temp_storage_size = 0; + sort_out_size = 0; + } + #endif + if (_gpu_nbor==2) { host_particle_id.clear(); host_cell_id.clear(); @@ -326,6 +385,22 @@ void AtomT::sort_neighbor(const int num_atoms) { UCL_GERYON_EXIT; } #endif + + #ifdef USE_HIP_DEVICE_SORT + if(sort_out_size < num_atoms){ + printf("AtomT::sort_neighbor: invalid temp buffer size\n"); + UCL_GERYON_EXIT; + } + if(hipSuccess != hipcub::DeviceRadixSort::SortPairs(sort_temp_storage, sort_temp_storage_size, (unsigned *)dev_cell_id.begin(), sort_out_keys, (int *)dev_particle_id.begin(), sort_out_values, num_atoms)){ + printf("AtomT::sort_neighbor: DeviceRadixSort error\n"); + UCL_GERYON_EXIT; + } + if(hipSuccess != hipMemcpy((unsigned *)dev_cell_id.begin(), sort_out_keys , num_atoms*sizeof(unsigned), hipMemcpyDeviceToDevice) || + hipSuccess != hipMemcpy((int *) dev_particle_id.begin(), sort_out_values, num_atoms*sizeof(int ), hipMemcpyDeviceToDevice)){ + printf("AtomT::sort_neighbor: copy output error\n"); + UCL_GERYON_EXIT; + } + #endif } #ifdef GPU_CAST diff --git a/lib/gpu/lal_atom.cu b/lib/gpu/lal_atom.cu index 28ff31c5660fb0690e5309a869857ddfe49f800d..99c76ba6251ec8d7bd87ade6eebf543a7440fcf8 100644 --- a/lib/gpu/lal_atom.cu +++ b/lib/gpu/lal_atom.cu @@ -13,7 +13,7 @@ // email : brownw@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_preprocessor.h" #endif diff --git a/lib/gpu/lal_atom.h b/lib/gpu/lal_atom.h index 57880d7ca976c029ab7006b9c2a5035da8018cfc..e39740d6c8293eb5099850171031269c24861a5e 100644 --- a/lib/gpu/lal_atom.h +++ b/lib/gpu/lal_atom.h @@ -29,6 +29,11 @@ using namespace ucl_opencl; #include "geryon/nvc_mat.h" #include "geryon/nvc_kernel.h" using namespace ucl_cudart; +#elif defined(USE_HIP) +#include "geryon/hip_timer.h" +#include "geryon/hip_mat.h" +#include "geryon/hip_kernel.h" +using namespace ucl_hip; #else #include "geryon/nvd_timer.h" #include "geryon/nvd_mat.h" @@ -477,6 +482,14 @@ class Atom { CUDPPConfiguration sort_config; CUDPPHandle sort_plan; #endif + + #ifdef USE_HIP_DEVICE_SORT + unsigned* sort_out_keys = nullptr; + int* sort_out_values = nullptr; + void* sort_temp_storage = nullptr; + size_t sort_temp_storage_size = 0; + size_t sort_out_size = 0; + #endif }; } diff --git a/lib/gpu/lal_aux_fun1.h b/lib/gpu/lal_aux_fun1.h index 47a216ff6f1c6fa84eefec11a48ad79e3e697804..5b7150d950fbd43e75921ee3d26c3fa690d591b8 100644 --- a/lib/gpu/lal_aux_fun1.h +++ b/lib/gpu/lal_aux_fun1.h @@ -13,7 +13,7 @@ // email : brownw@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_preprocessor.h" #endif diff --git a/lib/gpu/lal_base_atomic.h b/lib/gpu/lal_base_atomic.h index da17bd928db3506f3199fa3c34c7f587293e0df6..c97f42c50e66f6df6fccf44610fad2a94ef8c217 100644 --- a/lib/gpu/lal_base_atomic.h +++ b/lib/gpu/lal_base_atomic.h @@ -24,6 +24,8 @@ #include "geryon/ocl_texture.h" #elif defined(USE_CUDART) #include "geryon/nvc_texture.h" +#elif defined(USE_HIP) +#include "geryon/hip_texture.h" #else #include "geryon/nvd_texture.h" #endif diff --git a/lib/gpu/lal_base_charge.h b/lib/gpu/lal_base_charge.h index 29554ebfd5acd9423ea817db3e1b39293486a548..b6d3e9e3f8e82179497ed7f5dd22c6d83d5192ec 100644 --- a/lib/gpu/lal_base_charge.h +++ b/lib/gpu/lal_base_charge.h @@ -25,6 +25,8 @@ #include "geryon/ocl_texture.h" #elif defined(USE_CUDART) #include "geryon/nvc_texture.h" +#elif defined(USE_HIP) +#include "geryon/hip_texture.h" #else #include "geryon/nvd_texture.h" #endif diff --git a/lib/gpu/lal_base_dipole.h b/lib/gpu/lal_base_dipole.h index eecdcd3aab06e302fc8f997dab04405e72aee6ce..856b69b56b617767cb3be58b325201b5bf92da1e 100644 --- a/lib/gpu/lal_base_dipole.h +++ b/lib/gpu/lal_base_dipole.h @@ -23,6 +23,8 @@ #ifdef USE_OPENCL #include "geryon/ocl_texture.h" +#elif defined(USE_HIP) +#include "geryon/hip_texture.h" #else #include "geryon/nvd_texture.h" #endif diff --git a/lib/gpu/lal_base_dpd.h b/lib/gpu/lal_base_dpd.h index b46de2197dcd0d384b7931d85e7ab6516d9b80d4..5d1573c1a9e7e54d260719bedf584aa8b89e0d32 100644 --- a/lib/gpu/lal_base_dpd.h +++ b/lib/gpu/lal_base_dpd.h @@ -23,6 +23,8 @@ #ifdef USE_OPENCL #include "geryon/ocl_texture.h" +#elif defined(USE_HIP) +#include "geryon/hip_texture.h" #else #include "geryon/nvd_texture.h" #endif diff --git a/lib/gpu/lal_base_ellipsoid.h b/lib/gpu/lal_base_ellipsoid.h index d4bdd9e96989c823767034039923dac6b5e264db..dc1e624a2fe83acffb0d4d2fdc634cc54662e9b0 100644 --- a/lib/gpu/lal_base_ellipsoid.h +++ b/lib/gpu/lal_base_ellipsoid.h @@ -24,6 +24,8 @@ #include "geryon/ocl_texture.h" #elif defined(USE_CUDART) #include "geryon/nvc_texture.h" +#elif defined(USE_HIP) +#include "geryon/hip_texture.h" #else #include "geryon/nvd_texture.h" #endif diff --git a/lib/gpu/lal_base_three.h b/lib/gpu/lal_base_three.h index 96ca928edc0429ee8a10e1efd8ad318377b285cf..38af78b78ece17e14b4ef7b2cd38f83b3c3ecd37 100644 --- a/lib/gpu/lal_base_three.h +++ b/lib/gpu/lal_base_three.h @@ -24,6 +24,8 @@ #include "geryon/ocl_texture.h" #elif defined(USE_CUDART) #include "geryon/nvc_texture.h" +#elif defined(USE_HIP) +#include "geryon/hip_texture.h" #else #include "geryon/nvd_texture.h" #endif diff --git a/lib/gpu/lal_beck.cu b/lib/gpu/lal_beck.cu index 7d72128b5fba5145c96bfb9a517382c6d2d22a20..bdfa57a0cefb0d9a6f7b145110abd26a33ec4e89 100644 --- a/lib/gpu/lal_beck.cu +++ b/lib/gpu/lal_beck.cu @@ -13,12 +13,12 @@ // email : nguyentd@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_born.cu b/lib/gpu/lal_born.cu index 0ca7fea5febbac35b0e0feb07d2faa8c163a50aa..6e1d7d95a02d372121a01ebb78a94b04e56a4fb5 100644 --- a/lib/gpu/lal_born.cu +++ b/lib/gpu/lal_born.cu @@ -13,12 +13,12 @@ // email : nguyentd@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_born_coul_long.cu b/lib/gpu/lal_born_coul_long.cu index 71e5e0ae502e8847bc006788deaf82da9477868d..441ce4beb5bce75edb11c226bbc2aa7e6d922fe4 100644 --- a/lib/gpu/lal_born_coul_long.cu +++ b/lib/gpu/lal_born_coul_long.cu @@ -13,15 +13,15 @@ // email : nguyentd@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_born_coul_long_cs.cu b/lib/gpu/lal_born_coul_long_cs.cu index b3e79d9ec841afd0d31cf64b5af1c74f9469a811..f4b6da2d0d4c8be8ddc4db8bc8e7d5e827ce879d 100644 --- a/lib/gpu/lal_born_coul_long_cs.cu +++ b/lib/gpu/lal_born_coul_long_cs.cu @@ -13,15 +13,16 @@ // email : ndactrung@gmail.com // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" + #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_born_coul_wolf.cu b/lib/gpu/lal_born_coul_wolf.cu index 2c2249feeb7a34bdc899195aad60284aef51551a..e34367e18ba3c631d58f4eb05ab902e29c632ac8 100644 --- a/lib/gpu/lal_born_coul_wolf.cu +++ b/lib/gpu/lal_born_coul_wolf.cu @@ -13,15 +13,15 @@ // email : nguyentd@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_born_coul_wolf_cs.cu b/lib/gpu/lal_born_coul_wolf_cs.cu index 847387bfe8c1adbe2e1f3f2a3ed21964d08c5989..1a024207363eaed2c111f84833314571f4c70bd0 100644 --- a/lib/gpu/lal_born_coul_wolf_cs.cu +++ b/lib/gpu/lal_born_coul_wolf_cs.cu @@ -13,15 +13,15 @@ // email : ndactrung@gmail.com // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_buck.cu b/lib/gpu/lal_buck.cu index c1e1c7d7e251bfad2c1e0ffe71abc9a1bd1f9580..c23186f2d8fdbfb766a13e2abbede83c181ab67a 100644 --- a/lib/gpu/lal_buck.cu +++ b/lib/gpu/lal_buck.cu @@ -13,12 +13,12 @@ // email : nguyentd@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_buck_coul.cu b/lib/gpu/lal_buck_coul.cu index 6f0d414825562f4999ef627313ec1f8f81b31255..2282532f4a69c6c5708e2e8b11e783bcc06dd332 100644 --- a/lib/gpu/lal_buck_coul.cu +++ b/lib/gpu/lal_buck_coul.cu @@ -13,15 +13,15 @@ // email : nguyentd@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_buck_coul_long.cu b/lib/gpu/lal_buck_coul_long.cu index da3237a31fd41cd38f70ea7f0064c1cb05e9c787..469c23557154c8e1c8fe75fee3d787f2d14c19ab 100644 --- a/lib/gpu/lal_buck_coul_long.cu +++ b/lib/gpu/lal_buck_coul_long.cu @@ -13,15 +13,15 @@ // email : nguyentd@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_charmm_long.cu b/lib/gpu/lal_charmm_long.cu index 244131f8338fab3254ca7a494b2318ec0abbde39..a7977070574bdb8136885d9415bd3eaa1f12b578 100644 --- a/lib/gpu/lal_charmm_long.cu +++ b/lib/gpu/lal_charmm_long.cu @@ -13,15 +13,15 @@ // email : brownw@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_colloid.cu b/lib/gpu/lal_colloid.cu index 28a9809b192aa2328cdcfabb95c0a2cf425fab02..437faff25bdec909a612958028b8dfe718d0fb3f 100644 --- a/lib/gpu/lal_colloid.cu +++ b/lib/gpu/lal_colloid.cu @@ -13,12 +13,12 @@ // email : nguyentd@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_coul.cu b/lib/gpu/lal_coul.cu index 503e674c817cd90cf64177df47d411a445cd6898..21d849bb6fe0512b1409571a44ca28daaca2412b 100644 --- a/lib/gpu/lal_coul.cu +++ b/lib/gpu/lal_coul.cu @@ -13,15 +13,15 @@ // email : ndtrung@umich.edu // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_coul_debye.cu b/lib/gpu/lal_coul_debye.cu index 464a1b18def652095e85ab7c87735d365d40a5d5..ab8bc5b961b01a01cb9eb3f0ce278c8dcb5f2976 100644 --- a/lib/gpu/lal_coul_debye.cu +++ b/lib/gpu/lal_coul_debye.cu @@ -13,15 +13,15 @@ // email : ndtrung@umich.edu // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_coul_dsf.cu b/lib/gpu/lal_coul_dsf.cu index 82c44cd382e5f65d1adec3e1b816e86afba68d4c..147ac685522eb6c3cdfe6d422969fc49b63431b6 100644 --- a/lib/gpu/lal_coul_dsf.cu +++ b/lib/gpu/lal_coul_dsf.cu @@ -13,15 +13,15 @@ // email : nguyentd@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_coul_long.cu b/lib/gpu/lal_coul_long.cu index 365195e00c5809e0c48d6cc20b915fbb02860e67..f97a03962903a94c2bd9d905e0af8cc9c90bb466 100644 --- a/lib/gpu/lal_coul_long.cu +++ b/lib/gpu/lal_coul_long.cu @@ -13,15 +13,15 @@ // email : a.kohlmeyer@temple.edu // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_coul_long_cs.cu b/lib/gpu/lal_coul_long_cs.cu index 3c34666131d2d363c03bc6f684bd9b573b54c043..14791579442fb4943b695a4c2c7c102bdf2c0d06 100644 --- a/lib/gpu/lal_coul_long_cs.cu +++ b/lib/gpu/lal_coul_long_cs.cu @@ -13,15 +13,15 @@ // email : ndactrung@gmail.com // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_device.cpp b/lib/gpu/lal_device.cpp index 5bd306ea5b45be9c5830fafbfd1fea1b9ec2eee5..aaf74ed28c314bed0152667b43294ffe0055cae2 100644 --- a/lib/gpu/lal_device.cpp +++ b/lib/gpu/lal_device.cpp @@ -268,7 +268,7 @@ int DeviceT::init(Answer &ans, const bool charge, gpu_nbor=1; else if (_gpu_mode==Device::GPU_HYB_NEIGH) gpu_nbor=2; - #ifndef USE_CUDPP + #if !defined(USE_CUDPP) && !defined(USE_HIP_DEVICE_SORT) if (gpu_nbor==1) gpu_nbor=2; #endif @@ -341,7 +341,7 @@ int DeviceT::init_nbor(Neighbor *nbor, const int nlocal, gpu_nbor=1; else if (_gpu_mode==Device::GPU_HYB_NEIGH) gpu_nbor=2; - #ifndef USE_CUDPP + #if !defined(USE_CUDPP) && !defined(USE_HIP_DEVICE_SORT) if (gpu_nbor==1) gpu_nbor=2; #endif @@ -712,7 +712,7 @@ int DeviceT::compile_kernels() { gpu_lib_data.update_host(false); _ptx_arch=static_cast(gpu_lib_data[0])/100.0; - #ifndef USE_OPENCL + #if !(defined(USE_OPENCL) || defined(USE_HIP)) if (_ptx_arch>gpu->arch() || floor(_ptx_arch)arch())) return -4; #endif diff --git a/lib/gpu/lal_device.cu b/lib/gpu/lal_device.cu index 37d07588458ef4a69325f53d3e838dd1b19ab71b..afc7a0b988777e6c8cf9e045c1739b95dd8ae97a 100644 --- a/lib/gpu/lal_device.cu +++ b/lib/gpu/lal_device.cu @@ -13,7 +13,7 @@ // email : brownw@ornl.gov // *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_preprocessor.h" #endif diff --git a/lib/gpu/lal_dipole_lj.cu b/lib/gpu/lal_dipole_lj.cu index 745bdb7f274e133a7a8df75fc21bd33cbf88afbc..8ea49e7f60c2bc3c4c00e531cfa7effa713d2f47 100644 --- a/lib/gpu/lal_dipole_lj.cu +++ b/lib/gpu/lal_dipole_lj.cu @@ -13,16 +13,16 @@ // email : nguyentd@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; -texture mu_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); +_texture( mu_tex,float4); #else -texture pos_tex; -texture q_tex; -texture mu_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); +_texture_2d( mu_tex,int4); #endif #else diff --git a/lib/gpu/lal_dipole_lj_sf.cu b/lib/gpu/lal_dipole_lj_sf.cu index 9847e84823da2cc76a55a8585cd25d2cbd642648..9d753d9b63553b5feaf341b3628287d3b93cead6 100644 --- a/lib/gpu/lal_dipole_lj_sf.cu +++ b/lib/gpu/lal_dipole_lj_sf.cu @@ -13,17 +13,17 @@ // email : nguyentd@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; -texture mu_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); +_texture( mu_tex,float4); #else -texture pos_tex; -texture q_tex; -texture mu_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); +_texture_2d( mu_tex,int4); #endif #else diff --git a/lib/gpu/lal_dipole_long_lj.cu b/lib/gpu/lal_dipole_long_lj.cu index f888dece9b513001b4af9309ebc76e7b30f060d2..95c3b7a3dbb1a1a61b4ea5a0d058815635b0e9f7 100644 --- a/lib/gpu/lal_dipole_long_lj.cu +++ b/lib/gpu/lal_dipole_long_lj.cu @@ -13,16 +13,16 @@ // email : nguyentd@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; -texture mu_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); +_texture( mu_tex,float4); #else -texture pos_tex; -texture q_tex; -texture mu_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); +_texture_2d( mu_tex,int4); #endif #else diff --git a/lib/gpu/lal_dpd.cu b/lib/gpu/lal_dpd.cu index 462401ad7007bc927fce4f5ca86a8553e21c0f71..d97f430f7741032e210bbf09e6570a92111ea2ff 100644 --- a/lib/gpu/lal_dpd.cu +++ b/lib/gpu/lal_dpd.cu @@ -13,14 +13,14 @@ // email : nguyentd@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture vel_tex; +_texture( pos_tex,float4); +_texture( vel_tex,float4); #else -texture pos_tex; -texture vel_tex; +_texture_2d( pos_tex,int4); +_texture_2d( vel_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_eam.cu b/lib/gpu/lal_eam.cu index 13440b7d45a8e1594afffa69f5b53475ad2ca3f7..9427b1832f0c892488fa4ae171375e76b838bb6a 100644 --- a/lib/gpu/lal_eam.cu +++ b/lib/gpu/lal_eam.cu @@ -13,27 +13,27 @@ // email : brownw@ornl.gov nguyentd@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture fp_tex; -texture rhor_sp1_tex; -texture rhor_sp2_tex; -texture frho_sp1_tex; -texture frho_sp2_tex; -texture z2r_sp1_tex; -texture z2r_sp2_tex; +_texture( pos_tex,float4); +_texture( fp_tex,float); +_texture( rhor_sp1_tex,float4); +_texture( rhor_sp2_tex,float4); +_texture( frho_sp1_tex,float4); +_texture( frho_sp2_tex,float4); +_texture( z2r_sp1_tex,float4); +_texture( z2r_sp2_tex,float4); #else -texture pos_tex; -texture fp_tex; -texture rhor_sp1_tex; -texture rhor_sp2_tex; -texture frho_sp1_tex; -texture frho_sp2_tex; -texture z2r_sp1_tex; -texture z2r_sp2_tex; +_texture( pos_tex,int4); +_texture( fp_tex,int2); +_texture( rhor_sp1_tex,int4); +_texture( rhor_sp2_tex,int4); +_texture( frho_sp1_tex,int4); +_texture( frho_sp2_tex,int4); +_texture( z2r_sp1_tex,int4); +_texture( z2r_sp2_tex,int4); #endif #else diff --git a/lib/gpu/lal_ellipsoid_extra.h b/lib/gpu/lal_ellipsoid_extra.h index 71668f5e0280d4b0359c28a61404e912d1874d3d..e6122c740450143e25b505534a9aecb4c2c877dd 100644 --- a/lib/gpu/lal_ellipsoid_extra.h +++ b/lib/gpu/lal_ellipsoid_extra.h @@ -18,12 +18,14 @@ enum{SPHERE_SPHERE,SPHERE_ELLIPSE,ELLIPSE_SPHERE,ELLIPSE_ELLIPSE}; -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex, quat_tex; +_texture( pos_tex, float4); +_texture( quat_tex,float4); #else -texture pos_tex, quat_tex; +_texture_2d( pos_tex,int4); +_texture_2d( quat_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_ellipsoid_nbor.cu b/lib/gpu/lal_ellipsoid_nbor.cu index cac77f5dd38fb786cd4cbbec0089258a17cedb2b..e6eedc715914ce0228af88138db4b740b254506e 100644 --- a/lib/gpu/lal_ellipsoid_nbor.cu +++ b/lib/gpu/lal_ellipsoid_nbor.cu @@ -13,12 +13,12 @@ // email : brownw@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_preprocessor.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_gauss.cu b/lib/gpu/lal_gauss.cu index 98e71ea413bf356a9fe8c982584ef5ce9522a2d7..f9d374153745ef12c3880f8c8c8b9e27246f8171 100644 --- a/lib/gpu/lal_gauss.cu +++ b/lib/gpu/lal_gauss.cu @@ -13,12 +13,12 @@ // email : nguyentd@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_gayberne.cu b/lib/gpu/lal_gayberne.cu index cd1ee59fc66c25d4a08e974007dbabf966f757b2..5c035da004585340e07c1c6502623bb022665831 100644 --- a/lib/gpu/lal_gayberne.cu +++ b/lib/gpu/lal_gayberne.cu @@ -13,7 +13,7 @@ // email : brownw@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_ellipsoid_extra.h" #endif diff --git a/lib/gpu/lal_gayberne_lj.cu b/lib/gpu/lal_gayberne_lj.cu index 7925b72784ab4002d3bab679855586d81e354cef..eb9c797dc774415dc0adbb15b71355ea4838d695 100644 --- a/lib/gpu/lal_gayberne_lj.cu +++ b/lib/gpu/lal_gayberne_lj.cu @@ -13,7 +13,7 @@ // email : brownw@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_ellipsoid_extra.h" #endif diff --git a/lib/gpu/lal_lj.cu b/lib/gpu/lal_lj.cu index 5838ac95cf130c2f67b3e9676704d2878a755f0a..716346a83db689884ff1d2fd70a9aa48c527d571 100644 --- a/lib/gpu/lal_lj.cu +++ b/lib/gpu/lal_lj.cu @@ -13,12 +13,12 @@ // email : brownw@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_lj96.cu b/lib/gpu/lal_lj96.cu index 8dd63ef9203f206fc452089ceebd13890176c11e..aa06caa4aed804beb13c205753be120278886eb5 100644 --- a/lib/gpu/lal_lj96.cu +++ b/lib/gpu/lal_lj96.cu @@ -13,12 +13,12 @@ // email : brownw@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_lj_class2_long.cu b/lib/gpu/lal_lj_class2_long.cu index 41ceca35d74d0ef8a6ffcfc47ea53116c734bbbd..4e1bf9c1f73ca39ef741b1ea7a365ab980d2bc4f 100644 --- a/lib/gpu/lal_lj_class2_long.cu +++ b/lib/gpu/lal_lj_class2_long.cu @@ -13,15 +13,15 @@ // email : brownw@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_lj_coul.cu b/lib/gpu/lal_lj_coul.cu index 5c7f0da46fd066eb7573b4a74345240a0c5c0b37..cd72f72d977778e871861e7832e0dddb93752515 100644 --- a/lib/gpu/lal_lj_coul.cu +++ b/lib/gpu/lal_lj_coul.cu @@ -13,15 +13,15 @@ // email : brownw@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_lj_coul_debye.cu b/lib/gpu/lal_lj_coul_debye.cu index 91b105b3da672b608aa11a73306e2c512f2baf98..40b704662326c2a0bc00638b4b1bfddd3817536a 100644 --- a/lib/gpu/lal_lj_coul_debye.cu +++ b/lib/gpu/lal_lj_coul_debye.cu @@ -13,15 +13,15 @@ // email : nguyentd@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_lj_coul_long.cu b/lib/gpu/lal_lj_coul_long.cu index 0e25bb2dbc15a6781bbc6202da87d52163593baf..6a09cc4b75cc458f935beab1aab665c347c063a6 100644 --- a/lib/gpu/lal_lj_coul_long.cu +++ b/lib/gpu/lal_lj_coul_long.cu @@ -13,15 +13,15 @@ // email : brownw@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_lj_coul_msm.cu b/lib/gpu/lal_lj_coul_msm.cu index 3f73c6f47d6615f1a57505ada7a81976e8d9f996..c8eaa47b3d8fb5faaad79d4a12b4c42f72892a2e 100644 --- a/lib/gpu/lal_lj_coul_msm.cu +++ b/lib/gpu/lal_lj_coul_msm.cu @@ -13,19 +13,19 @@ // email : nguyentd@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; -texture gcons_tex; -texture dgcons_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); +_texture( gcons_tex,float); +_texture( dgcons_tex,float); #else -texture pos_tex; -texture q_tex; -texture gcons_tex; -texture dgcons_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); +_texture( gcons_tex,int2); +_texture( dgcons_tex,int2); #endif #else diff --git a/lib/gpu/lal_lj_cubic.cu b/lib/gpu/lal_lj_cubic.cu index 683c6b2aacd2882e623e9adcbfc4748e079d3f18..b6a0768a363a46bb9683c25be7ff31b72f540f13 100644 --- a/lib/gpu/lal_lj_cubic.cu +++ b/lib/gpu/lal_lj_cubic.cu @@ -13,12 +13,12 @@ // email : ndactrung@gmail.com // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_lj_dsf.cu b/lib/gpu/lal_lj_dsf.cu index 323576fe779dca0a46eb0464f15cfb948a10c475..2475743ccc2106e08d5d119021e7393c81ef390f 100644 --- a/lib/gpu/lal_lj_dsf.cu +++ b/lib/gpu/lal_lj_dsf.cu @@ -13,15 +13,15 @@ // email : brownw@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_lj_expand.cu b/lib/gpu/lal_lj_expand.cu index 9281ad27bd67b9c1fe9f4332a31791a3cbfb7902..44968355886c376344700fdaf2192ac155236e98 100644 --- a/lib/gpu/lal_lj_expand.cu +++ b/lib/gpu/lal_lj_expand.cu @@ -13,13 +13,13 @@ // email : ibains@nvidia.com // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else diff --git a/lib/gpu/lal_lj_expand_coul_long.cu b/lib/gpu/lal_lj_expand_coul_long.cu index aa8f02be8c521134f80201e873ad5636b33783e4..e9de9bab27df61c0fb7d56a26cacf8312074ee0e 100644 --- a/lib/gpu/lal_lj_expand_coul_long.cu +++ b/lib/gpu/lal_lj_expand_coul_long.cu @@ -13,15 +13,15 @@ // email : ndactrung@gmail.com // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_lj_gromacs.cu b/lib/gpu/lal_lj_gromacs.cu index 93dc3d945619b7c14f32642fffffbaff608ad3f1..dcef79dc90a6b75ca60ef2fe800304c4f4683ddb 100644 --- a/lib/gpu/lal_lj_gromacs.cu +++ b/lib/gpu/lal_lj_gromacs.cu @@ -13,13 +13,13 @@ // email : nguyentd@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else diff --git a/lib/gpu/lal_lj_sdk.cu b/lib/gpu/lal_lj_sdk.cu index 01b2cdd18d287b14737984e9167e67df2b5a1487..a11b1c78870c067d8a298a8f0eea0e26f4d817ea 100644 --- a/lib/gpu/lal_lj_sdk.cu +++ b/lib/gpu/lal_lj_sdk.cu @@ -13,12 +13,12 @@ // email : brownw@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_lj_sdk_long.cu b/lib/gpu/lal_lj_sdk_long.cu index 5ff64b22540cd48b7f85a5a3eaf8c0ebefeb9e66..e28fa19db4ffe7d20e0814a49b3af9f54373c38c 100644 --- a/lib/gpu/lal_lj_sdk_long.cu +++ b/lib/gpu/lal_lj_sdk_long.cu @@ -13,15 +13,15 @@ // email : brownw@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_lj_tip4p_long.cpp b/lib/gpu/lal_lj_tip4p_long.cpp index f60f162d7be20ae380349c2d5cbb7901a4602a78..5dd082df33bf8b80ca3238b0b2e5d09ee24d34b4 100644 --- a/lib/gpu/lal_lj_tip4p_long.cpp +++ b/lib/gpu/lal_lj_tip4p_long.cpp @@ -23,7 +23,7 @@ const char *lj_tip4p=0; #include "lal_lj_tip4p_long.h" #include -using namespace LAMMPS_AL; +namespace LAMMPS_AL { #define LJTIP4PLongT LJ_TIP4PLong extern Device device; @@ -381,6 +381,5 @@ int** LJTIP4PLongT::compute(const int ago, const int inum_full, } - - template class LJ_TIP4PLong; +} diff --git a/lib/gpu/lal_lj_tip4p_long.cu b/lib/gpu/lal_lj_tip4p_long.cu index c1011964332b644f501f919c7c5f81326d507dbb..66c9a6071e708d3c3e894110e7925cea03a211ed 100644 --- a/lib/gpu/lal_lj_tip4p_long.cu +++ b/lib/gpu/lal_lj_tip4p_long.cu @@ -13,7 +13,7 @@ // email : thevsevak@gmail.com // *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifdef LAMMPS_SMALLBIG @@ -27,11 +27,11 @@ #define tagint int #endif #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_mie.cu b/lib/gpu/lal_mie.cu index 33018566eb2f018cd672711960bd2691ce61d7b3..e2ede4d3a16abda9aad6d8ffed90ded480e9c14e 100644 --- a/lib/gpu/lal_mie.cu +++ b/lib/gpu/lal_mie.cu @@ -13,12 +13,12 @@ // email : nguyentd@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_morse.cu b/lib/gpu/lal_morse.cu index 0a14071d1978f4faf81b807983916153ec295419..7e4e0e54fa4e254290226e64828412a6f534a60f 100644 --- a/lib/gpu/lal_morse.cu +++ b/lib/gpu/lal_morse.cu @@ -13,13 +13,13 @@ // email : brownw@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else diff --git a/lib/gpu/lal_neighbor_cpu.cu b/lib/gpu/lal_neighbor_cpu.cu index d005eb9f9738a72ea4ba284cf570df2a93f7e900..29141a8b90a500a9284742661386aa9476240a5d 100644 --- a/lib/gpu/lal_neighbor_cpu.cu +++ b/lib/gpu/lal_neighbor_cpu.cu @@ -13,7 +13,7 @@ // email : brownw@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_preprocessor.h" #endif diff --git a/lib/gpu/lal_neighbor_gpu.cu b/lib/gpu/lal_neighbor_gpu.cu index 83692a24e4c04bfd9d1441c2a49b5da01fc9813e..8a2b603217a406518792769e1f2c47292adbc6f8 100644 --- a/lib/gpu/lal_neighbor_gpu.cu +++ b/lib/gpu/lal_neighbor_gpu.cu @@ -14,7 +14,7 @@ // email : penwang@nvidia.com, brownw@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_preprocessor.h" #ifdef LAMMPS_SMALLBIG #define tagint int @@ -27,9 +27,9 @@ #define tagint int #endif #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif __kernel void calc_cell_id(const numtyp4 *restrict pos, diff --git a/lib/gpu/lal_neighbor_shared.h b/lib/gpu/lal_neighbor_shared.h index 834ee8406d34a1fa7d4c709f46b6e6b423a82906..5cfc4e47678359b36818ebca814c948e6a430cd5 100644 --- a/lib/gpu/lal_neighbor_shared.h +++ b/lib/gpu/lal_neighbor_shared.h @@ -24,6 +24,10 @@ using namespace ucl_opencl; #include "geryon/nvc_kernel.h" #include "geryon/nvc_texture.h" using namespace ucl_cudart; +#elif defined(USE_HIP) +#include "geryon/hip_kernel.h" +#include "geryon/hip_texture.h" +using namespace ucl_hip; #else #include "geryon/nvd_kernel.h" #include "geryon/nvd_texture.h" diff --git a/lib/gpu/lal_pppm.cu b/lib/gpu/lal_pppm.cu index 24636b9a9303251f1f8daf31c13c81e2786df429..6a7408c7207287023b094e05664490e5f6a04e06 100644 --- a/lib/gpu/lal_pppm.cu +++ b/lib/gpu/lal_pppm.cu @@ -13,15 +13,15 @@ // email : brownw@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_preprocessor.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif // Allow PPPM to compile without atomics for NVIDIA 1.0 cards, error diff --git a/lib/gpu/lal_pppm.h b/lib/gpu/lal_pppm.h index 65af1575134b3bdaa8f51672a985d9f40d405c58..499aa265d305809bac1e39902c0c26b22b8edb68 100644 --- a/lib/gpu/lal_pppm.h +++ b/lib/gpu/lal_pppm.h @@ -23,6 +23,8 @@ #include "geryon/ocl_texture.h" #elif defined(USE_CUDART) #include "geryon/nvc_texture.h" +#elif defined(USE_HIP) +#include "geryon/hip_texture.h" #else #include "geryon/nvd_texture.h" #endif diff --git a/lib/gpu/lal_precision.h b/lib/gpu/lal_precision.h index d5b1b9b6c05fb7bda08c7acef1e15917a07f2436..7f82ba18aa92d033f5aca5be60fcda278e9c480b 100644 --- a/lib/gpu/lal_precision.h +++ b/lib/gpu/lal_precision.h @@ -24,9 +24,11 @@ struct _lgpu_int2 { int x; int y; }; +#ifndef USE_HIP #ifndef int2 #define int2 _lgpu_int2 #endif +#endif struct _lgpu_float2 { float x; float y; diff --git a/lib/gpu/lal_preprocessor.h b/lib/gpu/lal_preprocessor.h index 566a451c21b83c34090fcc8a8a150fe42d4eef17..cd95355ee40f5951aea8f525011940eacc37096c 100644 --- a/lib/gpu/lal_preprocessor.h +++ b/lib/gpu/lal_preprocessor.h @@ -1,4 +1,4 @@ -// ************************************************************************** +// ************************************************************************** // preprocessor.cu // ------------------- // W. Michael Brown (ORNL) @@ -60,6 +60,150 @@ // //*************************************************************************/ +#define _texture(name, type) texture name +#define _texture_2d(name, type) texture name + +// ------------------------------------------------------------------------- +// HIP DEFINITIONS +// ------------------------------------------------------------------------- + +#ifdef USE_HIP + #include + #ifdef __HIP_PLATFORM_HCC__ + #define mul24(x, y) __mul24(x, y) + #undef _texture + #undef _texture_2d + #define _texture(name, type) __device__ type* name + #define _texture_2d(name, type) __device__ type* name + #endif + #define GLOBAL_ID_X threadIdx.x+mul24(blockIdx.x,blockDim.x) + #define GLOBAL_ID_Y threadIdx.y+mul24(blockIdx.y,blockDim.y) + #define GLOBAL_SIZE_X mul24(gridDim.x,blockDim.x); + #define GLOBAL_SIZE_Y mul24(gridDim.y,blockDim.y); + #define THREAD_ID_X threadIdx.x + #define THREAD_ID_Y threadIdx.y + #define BLOCK_ID_X blockIdx.x + #define BLOCK_ID_Y blockIdx.y + #define BLOCK_SIZE_X blockDim.x + #define BLOCK_SIZE_Y blockDim.y + #define __kernel extern "C" __global__ + #ifdef __local + #undef __local + #endif + #define __local __shared__ + #define __global + #define restrict __restrict__ + #define atom_add atomicAdd + #define ucl_inline static __inline__ __device__ + + #define THREADS_PER_ATOM 4 + #define THREADS_PER_CHARGE 8 + #define BLOCK_NBOR_BUILD 128 + #define BLOCK_PAIR 256 + #define BLOCK_BIO_PAIR 256 + #define BLOCK_ELLIPSE 128 + #define MAX_SHARED_TYPES 11 + + #ifdef _SINGLE_SINGLE + ucl_inline double shfl_xor(double var, int laneMask, int width) { + #ifdef __HIP_PLATFORM_HCC__ + return __shfl_xor(var, laneMask, width); + #else + return __shfl_xor_sync(0xffffffff, var, laneMask, width); + #endif + } + #else + ucl_inline double shfl_xor(double var, int laneMask, int width) { + int2 tmp; + tmp.x = __double2hiint(var); + tmp.y = __double2loint(var); + #ifdef __HIP_PLATFORM_HCC__ + tmp.x = __shfl_xor(tmp.x,laneMask,width); + tmp.y = __shfl_xor(tmp.y,laneMask,width); + #else + tmp.x = __shfl_xor_sync(0xffffffff, tmp.x,laneMask,width); + tmp.y = __shfl_xor_sync(0xffffffff, tmp.y,laneMask,width); + #endif + return __hiloint2double(tmp.x,tmp.y); + } + #endif + + #ifdef __HIP_PLATFORM_HCC__ + #define ARCH 600 + #define WARP_SIZE 64 + #endif + + #ifdef __HIP_PLATFORM_NVCC__ + #define ARCH __CUDA_ARCH__ + #define WARP_SIZE 32 + #endif + + #define fast_mul(X,Y) (X)*(Y) + + #define MEM_THREADS WARP_SIZE + #define PPPM_BLOCK_1D 64 + #define BLOCK_CELL_2D 8 + #define BLOCK_CELL_ID 128 + #define MAX_BIO_SHARED_TYPES 128 + + #ifdef __HIP_PLATFORM_NVCC__ + #ifdef _DOUBLE_DOUBLE + #define fetch4(ans,i,pos_tex) { \ + int4 xy = tex1Dfetch(pos_tex,i*2); \ + int4 zt = tex1Dfetch(pos_tex,i*2+1); \ + ans.x=__hiloint2double(xy.y, xy.x); \ + ans.y=__hiloint2double(xy.w, xy.z); \ + ans.z=__hiloint2double(zt.y, zt.x); \ + ans.w=__hiloint2double(zt.w, zt.z); \ + } + #define fetch(ans,i,q_tex) { \ + int2 qt = tex1Dfetch(q_tex,i); \ + ans=__hiloint2double(qt.y, qt.x); \ + } + #else + #define fetch4(ans,i,pos_tex) ans=tex1Dfetch(pos_tex, i); + #define fetch(ans,i,q_tex) ans=tex1Dfetch(q_tex,i); + #endif + #else + #ifdef _DOUBLE_DOUBLE + #define fetch4(ans,i,pos_tex) (ans=*(((double4*)pos_tex) + i)) + #define fetch(ans,i,q_tex) (ans=*(((double *) q_tex) + i)) + #else + #define fetch4(ans,i,pos_tex) (ans=*(((float4*)pos_tex) + i)) + #define fetch(ans,i,q_tex) (ans=*(((float *) q_tex) + i)) + #endif + #endif + + #ifdef _DOUBLE_DOUBLE + #define ucl_exp exp + #define ucl_powr pow + #define ucl_atan atan + #define ucl_cbrt cbrt + #define ucl_ceil ceil + #define ucl_abs fabs + #define ucl_rsqrt rsqrt + #define ucl_sqrt sqrt + #define ucl_recip(x) ((numtyp)1.0/(x)) + + #else + #define ucl_atan atanf + #define ucl_cbrt cbrtf + #define ucl_ceil ceilf + #define ucl_abs fabsf + #define ucl_recip(x) ((numtyp)1.0/(x)) + #define ucl_rsqrt rsqrtf + #define ucl_sqrt sqrtf + + #ifdef NO_HARDWARE_TRANSCENDENTALS + #define ucl_exp expf + #define ucl_powr powf + #else + #define ucl_exp __expf + #define ucl_powr __powf + #endif + #endif +#endif + // ------------------------------------------------------------------------- // CUDA DEFINITIONS // ------------------------------------------------------------------------- diff --git a/lib/gpu/lal_re_squared.cu b/lib/gpu/lal_re_squared.cu index e238734074849da8db2760635571e74a25835915..cd525a1ade69beff2931a13dacc48ac9fd3a35b8 100644 --- a/lib/gpu/lal_re_squared.cu +++ b/lib/gpu/lal_re_squared.cu @@ -13,7 +13,7 @@ // email : brownw@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_ellipsoid_extra.h" #endif diff --git a/lib/gpu/lal_re_squared_lj.cu b/lib/gpu/lal_re_squared_lj.cu index d69dae2461ff699646bd5760f20d36674b1f9ad6..b3c44febe71e6cc12056f70dfa64836295f6caa0 100644 --- a/lib/gpu/lal_re_squared_lj.cu +++ b/lib/gpu/lal_re_squared_lj.cu @@ -13,7 +13,7 @@ // email : brownw@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_ellipsoid_extra.h" #endif diff --git a/lib/gpu/lal_soft.cu b/lib/gpu/lal_soft.cu index 831b986725a135355158ce439308a6c6abaa1284..bccfa85ed5010d480583898c262999fd1b97c317 100644 --- a/lib/gpu/lal_soft.cu +++ b/lib/gpu/lal_soft.cu @@ -13,12 +13,12 @@ // email : nguyentd@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_sw.cu b/lib/gpu/lal_sw.cu index 3b6de5a683d603e74f53b8c2364942fd0f83f2cf..de36d29efbd616385bbd87886222c45a413bb972 100644 --- a/lib/gpu/lal_sw.cu +++ b/lib/gpu/lal_sw.cu @@ -13,19 +13,19 @@ // email : brownw@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture sw1_tex; -texture sw2_tex; -texture sw3_tex; +_texture( pos_tex,float4); +_texture( sw1_tex,float4); +_texture( sw2_tex,float4); +_texture( sw3_tex,float4); #else -texture pos_tex; -texture sw1_tex; -texture sw2_tex; -texture sw3_tex; +_texture_2d( pos_tex,int4); +_texture( sw1_tex,int4); +_texture( sw2_tex,int4); +_texture( sw3_tex,int4); #endif #else diff --git a/lib/gpu/lal_table.cu b/lib/gpu/lal_table.cu index 971b56d96e5b4955acd687f60a1110e1fa645552..8c0b52e05f51daa74316064968a9444a97c8ca37 100644 --- a/lib/gpu/lal_table.cu +++ b/lib/gpu/lal_table.cu @@ -13,12 +13,12 @@ // email : nguyentd@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_tersoff.cu b/lib/gpu/lal_tersoff.cu index 2e29ca721b00d7c4a8563bce6f997c5d3a055ad7..d57efaf15c7c1bda41a271ad205296f0f351533d 100644 --- a/lib/gpu/lal_tersoff.cu +++ b/lib/gpu/lal_tersoff.cu @@ -13,23 +13,23 @@ // email : ndactrung@gmail.com // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_tersoff_extra.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture ts1_tex; -texture ts2_tex; -texture ts3_tex; -texture ts4_tex; -texture ts5_tex; +_texture( pos_tex,float4); +_texture( ts1_tex,float4); +_texture( ts2_tex,float4); +_texture( ts3_tex,float4); +_texture( ts4_tex,float4); +_texture( ts5_tex,float4); #else -texture pos_tex; -texture ts1_tex; -texture ts2_tex; -texture ts3_tex; -texture ts4_tex; -texture ts5_tex; +_texture_2d( pos_tex,int4); +_texture( ts1_tex,int4); +_texture( ts2_tex,int4); +_texture( ts3_tex,int4); +_texture( ts4_tex,int4); +_texture( ts5_tex,int4); #endif #else diff --git a/lib/gpu/lal_tersoff_extra.h b/lib/gpu/lal_tersoff_extra.h index 47d16678f088be320463ca4f6dca4299fee1dd82..7ee29751b7014d7c34c5a0bde286e3aa1b3b9755 100644 --- a/lib/gpu/lal_tersoff_extra.h +++ b/lib/gpu/lal_tersoff_extra.h @@ -16,7 +16,7 @@ #ifndef LAL_TERSOFF_EXTRA_H #define LAL_TERSOFF_EXTRA_H -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #else #endif diff --git a/lib/gpu/lal_tersoff_mod.cu b/lib/gpu/lal_tersoff_mod.cu index c85f5e08ca4821b84bf607bdbf8b0bd9ad74954c..da284f39ee2549c106c65931580393ef0059f0d6 100644 --- a/lib/gpu/lal_tersoff_mod.cu +++ b/lib/gpu/lal_tersoff_mod.cu @@ -13,23 +13,23 @@ // email : ndactrung@gmail.com // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_tersoff_mod_extra.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture ts1_tex; -texture ts2_tex; -texture ts3_tex; -texture ts4_tex; -texture ts5_tex; +_texture( pos_tex,float4); +_texture( ts1_tex,float4); +_texture( ts2_tex,float4); +_texture( ts3_tex,float4); +_texture( ts4_tex,float4); +_texture( ts5_tex,float4); #else -texture pos_tex; -texture ts1_tex; -texture ts2_tex; -texture ts3_tex; -texture ts4_tex; -texture ts5_tex; +_texture_2d( pos_tex,int4); +_texture( ts1_tex,int4); +_texture( ts2_tex,int4); +_texture( ts3_tex,int4); +_texture( ts4_tex,int4); +_texture( ts5_tex,int4); #endif #else diff --git a/lib/gpu/lal_tersoff_mod_extra.h b/lib/gpu/lal_tersoff_mod_extra.h index a130d98488d8a281233d1587c7a123a3889b5816..fb658cb0da1d3e1167fdc7f416ef944ff3ccc953 100644 --- a/lib/gpu/lal_tersoff_mod_extra.h +++ b/lib/gpu/lal_tersoff_mod_extra.h @@ -16,7 +16,7 @@ #ifndef LAL_TERSOFF_MOD_EXTRA_H #define LAL_TERSOFF_MOD_EXTRA_H -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #else #endif diff --git a/lib/gpu/lal_tersoff_zbl.cu b/lib/gpu/lal_tersoff_zbl.cu index b574a529c019437c6275a50b8c80d88121a140f2..a170715f57719a68ad515f0d98738c152bfbd7f8 100644 --- a/lib/gpu/lal_tersoff_zbl.cu +++ b/lib/gpu/lal_tersoff_zbl.cu @@ -13,25 +13,25 @@ // email : ndactrung@gmail.com // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_tersoff_zbl_extra.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture ts1_tex; -texture ts2_tex; -texture ts3_tex; -texture ts4_tex; -texture ts5_tex; -texture ts6_tex; +_texture( pos_tex,float4); +_texture( ts1_tex,float4); +_texture( ts2_tex,float4); +_texture( ts3_tex,float4); +_texture( ts4_tex,float4); +_texture( ts5_tex,float4); +_texture( ts6_tex,float4); #else -texture pos_tex; -texture ts1_tex; -texture ts2_tex; -texture ts3_tex; -texture ts4_tex; -texture ts5_tex; -texture ts6_tex; +_texture_2d( pos_tex,int4); +_texture( ts1_tex,int4); +_texture( ts2_tex,int4); +_texture( ts3_tex,int4); +_texture( ts4_tex,int4); +_texture( ts5_tex,int4); +_texture( ts6_tex,int4); #endif #else diff --git a/lib/gpu/lal_tersoff_zbl_extra.h b/lib/gpu/lal_tersoff_zbl_extra.h index 32c05a3716b19719e33922391d81608d639cb89a..9e5bcb10b44a8e320578949a77bff891eabbe691 100644 --- a/lib/gpu/lal_tersoff_zbl_extra.h +++ b/lib/gpu/lal_tersoff_zbl_extra.h @@ -16,7 +16,7 @@ #ifndef LAL_TERSOFF_ZBL_EXTRA_H #define LAL_TERSOFF_ZBL_EXTRA_H -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #else #endif diff --git a/lib/gpu/lal_ufm.cu b/lib/gpu/lal_ufm.cu index 51c4df3b5b9cfac3516ece739aebb0e7b3afd123..33d0f3c956fb16ae2c711f8666613a546571a542 100644 --- a/lib/gpu/lal_ufm.cu +++ b/lib/gpu/lal_ufm.cu @@ -15,12 +15,12 @@ dekoning@ifi.unicamp.br ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_vashishta.cu b/lib/gpu/lal_vashishta.cu index 0da46c3b534cc29cba600851db0dda05a2cf38da..d13bc659e75a478d14033292b1532a8f1b044936 100644 --- a/lib/gpu/lal_vashishta.cu +++ b/lib/gpu/lal_vashishta.cu @@ -13,23 +13,23 @@ // email : andershaf@gmail.com // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture param1_tex; -texture param2_tex; -texture param3_tex; -texture param4_tex; -texture param5_tex; +_texture( pos_tex,float4); +_texture( param1_tex,float4); +_texture( param2_tex,float4); +_texture( param3_tex,float4); +_texture( param4_tex,float4); +_texture( param5_tex,float4); #else -texture pos_tex; -texture param1_tex; -texture param2_tex; -texture param3_tex; -texture param4_tex; -texture param5_tex; +_texture_2d( pos_tex,int4); +_texture( param1_tex,int4); +_texture( param2_tex,int4); +_texture( param3_tex,int4); +_texture( param4_tex,int4); +_texture( param5_tex,int4); #endif #else diff --git a/lib/gpu/lal_yukawa.cu b/lib/gpu/lal_yukawa.cu index a8d637ec97e7d92b95726175a21f17b4d2e58b63..5237549b0aa503b08cbfd651bc60eb27683e2866 100644 --- a/lib/gpu/lal_yukawa.cu +++ b/lib/gpu/lal_yukawa.cu @@ -13,12 +13,12 @@ // email : nguyentd@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_yukawa_colloid.cu b/lib/gpu/lal_yukawa_colloid.cu index a3cbbbc11ccecd22eb596f1d591a09d4f12d7c38..8c006a09be262f087ebf408ee6309becd357e06c 100644 --- a/lib/gpu/lal_yukawa_colloid.cu +++ b/lib/gpu/lal_yukawa_colloid.cu @@ -13,15 +13,15 @@ // email : nguyentd@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture rad_tex; +_texture( pos_tex,float4); +_texture( rad_tex,float); #else -texture pos_tex; -texture rad_tex; +_texture_2d( pos_tex,int4); +_texture( rad_tex,int2); #endif #else diff --git a/lib/gpu/lal_zbl.cu b/lib/gpu/lal_zbl.cu index 33c850e1347f4062edb6de2b8f60a0f90d405680..fbedfe2de2341352511ba7726c269324d24e58bb 100644 --- a/lib/gpu/lal_zbl.cu +++ b/lib/gpu/lal_zbl.cu @@ -13,12 +13,12 @@ // email : ndactrung@gmail.com // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/kokkos/BUILD.md b/lib/kokkos/BUILD.md new file mode 100644 index 0000000000000000000000000000000000000000..84a1bc3b3788c2ba99c7b2ab4c992108737188db --- /dev/null +++ b/lib/kokkos/BUILD.md @@ -0,0 +1,323 @@ +![Kokkos](https://avatars2.githubusercontent.com/u/10199860?s=200&v=4) + +# Installing and Using Kokkos + +## Kokkos Philosophy +Kokkos provides a modern CMake style build system. +As C++ continues to develop for C++20 and beyond, CMake is likely to provide the most robust support +for C++. Applications heavily leveraging Kokkos are strongly encouraged to use a CMake build system. + +You can either use Kokkos as an installed package (encouraged) or use Kokkos in-tree in your project. +Modern CMake is exceedingly simple at a high-level (with the devil in the details). +Once Kokkos is installed In your `CMakeLists.txt` simply use: +```` +find_package(Kokkos REQUIRED) +```` +Then for every executable or library in your project: +```` +target_link_libraries(myTarget Kokkos::kokkos) +```` +That's it! There is no checking Kokkos preprocessor, compiler, or linker flags. +Kokkos propagates all the necessary flags to your project. +This means not only is linking to Kokkos easy, but Kokkos itself can actually configure compiler and linker flags for *your* +project. If building in-tree, there is no `find_package` and you link with `target_link_libraries(kokkos)`. + + +## Configuring CMake +A very basic installation is done with: +```` +cmake ${srcdir} \ + -DCMAKE_CXX_COMPILER=g++ \ + -DCMAKE_INSTALL_PREFIX=${my_install_folder} +```` +which builds and installed a default Kokkos when you run `make install`. +There are numerous device backends, options, and architecture-specific optimizations that can be configured, e.g. +```` +cmake ${srcdir} \ + -DCMAKE_CXX_COMPILER=g++ \ + -DCMAKE_INSTALL_PREFIX=${my_install_folder} \ + -DKokkos_ENABLE_OPENMP=On +```` +which activates the OpenMP backend. All of the options controlling device backends, options, architectures, and third-party libraries (TPLs) are given below. + +## Spack +An alternative to manually building with the CMake is to use the Spack package manager. +To do so, download the `kokkos-spack` git repo and add to the package list: +```` +spack repo add $path-to-kokkos-spack +```` +A basic installation would be done as: +```` +spack install kokkos +```` +Spack allows options and and compilers to be tuned in the install command. +```` +spack install kokkos@3.0 %gcc@7.3.0 +openmp +```` +This example illustrates the three most common parameters to Spack: +* Variants: specified with, e.g. `+openmp`, this activates (or deactivates with, e.g. `~openmp`) certain options. +* Version: immediately following `kokkos` the `@version` can specify a particular Kokkos to build +* Compiler: a default compiler will be chosen if not specified, but an exact compiler version can be given with the `%`option. + +For a complete list of Kokkos options, run: +```` +spack info kokkos +```` + +#### Spack Development +Spack currently installs packages to a location determined by a unique hash. This hash name is not really "human readable". +Generally, Spack usage should never really require you to reference the computer-generated unique install folder. +If you must know, you can locate Spack Kokkos installations with: +```` +spack find -p kokkos ... +```` +where `...` is the unique spec identifying the particular Kokkos configuration and version. + +A better way to use Spack for doing Kokkos development is the DIY feature of Spack. +If you wish to develop Kokkos itself, go to the Kokkos source folder: +```` +spack diy -u cmake kokkos@diy ... +```` +where `...` is a Spack spec identifying the exact Kokkos configuration. +This then creates a `spack-build` directory where you can run `make`. + +If doing development on a downstream project, you can do almost exactly the same thing. +```` +spack diy -u cmake ${myproject}@${myversion} ... ^kokkos... +```` +where the `...` are the specs for your project and the desired Kokkos configuration. +Again, a `spack-build` directory will be created where you can run `make`. + +Spack has a few idiosyncracies that make building outside of Spack annoying related to Spack forcing use of a compiler wrapper. This can be worked around by having a `-DSpack_WORKAROUND=On` given your CMake. Then add the block of code to your CMakeLists.txt: + +```` +if (Spack_WORKAROUND) + set(SPACK_CXX $ENV{SPACK_CXX}) + if(SPACK_CXX) + set(CMAKE_CXX_COMPILER ${SPACK_CXX} CACHE STRING "the C++ compiler" FORCE) + set(ENV{CXX} ${SPACK_CXX}) + endif() +endif() +```` + +# Kokkos Keyword Listing + +## Device Backends +Device backends can be enabled by specifying `-DKokkos_ENABLE_X`. + +* Kokkos_ENABLE_CUDA + * Whether to build CUDA backend + * BOOL Default: OFF +* Kokkos_ENABLE_HPX + * Whether to build HPX backend (experimental) + * BOOL Default: OFF +* Kokkos_ENABLE_OPENMP + * Whether to build OpenMP backend + * BOOL Default: OFF +* Kokkos_ENABLE_PTHREAD + * Whether to build Pthread backend + * BOOL Default: OFF +* Kokkos_ENABLE_SERIAL + * Whether to build serial backend + * BOOL Default: ON + +## Enable Options +Options can be enabled by specifying `-DKokkos_ENABLE_X`. + +* Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION + * Whether to aggressively vectorize loops + * BOOL Default: OFF +* Kokkos_ENABLE_COMPILER_WARNINGS + * Whether to print all compiler warnings + * BOOL Default: OFF +* Kokkos_ENABLE_CUDA_CONSTEXPR + * Whether to activate experimental relaxed constexpr functions + * BOOL Default: OFF +* Kokkos_ENABLE_CUDA_LAMBDA + * Whether to activate experimental lambda features + * BOOL Default: OFF +* Kokkos_ENABLE_CUDA_LDG_INTRINSIC + * Whether to use CUDA LDG intrinsics + * BOOL Default: OFF +* Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE + * Whether to enable relocatable device code (RDC) for CUDA + * BOOL Default: OFF +* Kokkos_ENABLE_CUDA_UVM + * Whether to use unified memory (UM) by default for CUDA + * BOOL Default: OFF +* Kokkos_ENABLE_DEBUG + * Whether to activate extra debug features - may increase compile times + * BOOL Default: OFF +* Kokkos_ENABLE_DEBUG_BOUNDS_CHECK + * Whether to use bounds checking - will increase runtime + * BOOL Default: OFF +* Kokkos_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK + * Debug check on dual views + * BOOL Default: OFF +* Kokkos_ENABLE_DEPRECATED_CODE + * Whether to enable deprecated code + * BOOL Default: OFF +* Kokkos_ENABLE_HPX_ASYNC_DISPATCH + * Whether HPX supports asynchronous dispatch + * BOOL Default: OFF +* Kokkos_ENABLE_LARGE_MEM_TESTS + * Whether to perform extra large memory tests + * BOOL_Default: OFF +* Kokkos_ENABLE_PROFILING + * Whether to create bindings for profiling tools + * BOOL Default: ON +* Kokkos_ENABLE_PROFILING_LOAD_PRINT + * Whether to print information about which profiling tools gotloaded + * BOOL Default: OFF +* Kokkos_ENABLE_TESTS + * Whether to build serial backend + * BOOL Default: OFF + +## Other Options +* Kokkos_CXX_STANDARD + * The C++ standard for Kokkos to use: c++11, c++14, c++17, or c++20. This should be given in CMake style as 11, 14, 17, or 20. + * STRING Default: 11 + +## Third-party Libraries (TPLs) +The following options control enabling TPLs: +* Kokkos_ENABLE_HPX + * Whether to enable the HPX library + * BOOL Default: OFF +* Kokkos_ENABLE_HWLOC + * Whether to enable the HWLOC library + * BOOL Default: Off +* Kokkos_ENABLE_LIBNUMA + * Whether to enable the LIBNUMA library + * BOOL Default: Off +* Kokkos_ENABLE_MEMKIND + * Whether to enable the MEMKIND library + * BOOL Default: Off +* Kokkos_ENABLE_LIBDL + * Whether to enable the LIBDL library + * BOOL Default: On +* Kokkos_ENABLE_LIBRT + * Whether to enable the LIBRT library + * BOOL Default: Off + +The following options control finding and configuring non-CMake TPLs: +* Kokkos_CUDA_DIR or CUDA_ROOT + * Location of CUDA install prefix for libraries + * PATH Default: +* Kokkos_HWLOC_DIR or HWLOC_ROOT + * Location of HWLOC install prefix + * PATH Default: +* Kokkos_LIBNUMA_DIR or LIBNUMA_ROOT + * Location of LIBNUMA install prefix + * PATH Default: +* Kokkos_MEMKIND_DIR or MEMKIND_ROOT + * Location of MEMKIND install prefix + * PATH Default: +* Kokkos_LIBDL_DIR or LIBDL_ROOT + * Location of LIBDL install prefix + * PATH Default: +* Kokkos_LIBRT_DIR or LIBRT_ROOT + * Location of LIBRT install prefix + * PATH Default: + +The following options control `find_package` paths for CMake-based TPLs: +* HPX_DIR or HPX_ROOT + * Location of HPX prefix (ROOT) or CMake config file (DIR) + * PATH Default: + +## Architecture Keywords +Architecture-specific optimizations can be enabled by specifying `-DKokkos_ARCH_X`. + +* Kokkos_ARCH_AMDAVX + * Whether to optimize for the AMDAVX architecture + * BOOL Default: OFF +* Kokkos_ARCH_ARMV80 + * Whether to optimize for the ARMV80 architecture + * BOOL Default: OFF +* Kokkos_ARCH_ARMV81 + * Whether to optimize for the ARMV81 architecture + * BOOL Default: OFF +* Kokkos_ARCH_ARMV8_THUNDERX + * Whether to optimize for the ARMV8_THUNDERX architecture + * BOOL Default: OFF +* Kokkos_ARCH_ARMV8_TX2 + * Whether to optimize for the ARMV8_TX2 architecture + * BOOL Default: OFF +* Kokkos_ARCH_BDW + * Whether to optimize for the BDW architecture + * BOOL Default: OFF +* Kokkos_ARCH_BGQ + * Whether to optimize for the BGQ architecture + * BOOL Default: OFF +* Kokkos_ARCH_EPYC + * Whether to optimize for the EPYC architecture + * BOOL Default: OFF +* Kokkos_ARCH_HSW + * Whether to optimize for the HSW architecture + * BOOL Default: OFF +* Kokkos_ARCH_KEPLER30 + * Whether to optimize for the KEPLER30 architecture + * BOOL Default: OFF +* Kokkos_ARCH_KEPLER32 + * Whether to optimize for the KEPLER32 architecture + * BOOL Default: OFF +* Kokkos_ARCH_KEPLER35 + * Whether to optimize for the KEPLER35 architecture + * BOOL Default: OFF +* Kokkos_ARCH_KEPLER37 + * Whether to optimize for the KEPLER37 architecture + * BOOL Default: OFF +* Kokkos_ARCH_KNC + * Whether to optimize for the KNC architecture + * BOOL Default: OFF +* Kokkos_ARCH_KNL + * Whether to optimize for the KNL architecture + * BOOL Default: OFF +* Kokkos_ARCH_MAXWELL50 + * Whether to optimize for the MAXWELL50 architecture + * BOOL Default: OFF +* Kokkos_ARCH_MAXWELL52 + * Whether to optimize for the MAXWELL52 architecture + * BOOL Default: OFF +* Kokkos_ARCH_MAXWELL53 + * Whether to optimize for the MAXWELL53 architecture + * BOOL Default: OFF +* Kokkos_ARCH_PASCAL60 + * Whether to optimize for the PASCAL60 architecture + * BOOL Default: OFF +* Kokkos_ARCH_PASCAL61 + * Whether to optimize for the PASCAL61 architecture + * BOOL Default: OFF +* Kokkos_ARCH_POWER7 + * Whether to optimize for the POWER7 architecture + * BOOL Default: OFF +* Kokkos_ARCH_POWER8 + * Whether to optimize for the POWER8 architecture + * BOOL Default: OFF +* Kokkos_ARCH_POWER9 + * Whether to optimize for the POWER9 architecture + * BOOL Default: OFF +* Kokkos_ARCH_SKX + * Whether to optimize for the SKX architecture + * BOOL Default: OFF +* Kokkos_ARCH_SNB + * Whether to optimize for the SNB architecture + * BOOL Default: OFF +* Kokkos_ARCH_TURING75 + * Whether to optimize for the TURING75 architecture + * BOOL Default: OFF +* Kokkos_ARCH_VOLTA70 + * Whether to optimize for the VOLTA70 architecture + * BOOL Default: OFF +* Kokkos_ARCH_VOLTA72 + * Whether to optimize for the VOLTA72 architecture + * BOOL Default: OFF +* Kokkos_ARCH_WSM + * Whether to optimize for the WSM architecture + * BOOL Default: OFF + +##### [LICENSE](https://github.com/kokkos/kokkos/blob/devel/LICENSE) + +[![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) + +Under the terms of Contract DE-NA0003525 with NTESS, +the U.S. Government retains certain rights in this software. diff --git a/lib/kokkos/CHANGELOG.md b/lib/kokkos/CHANGELOG.md index 8d196e2c35abf8c6a8109a3a2f84cc781de67925..1d54b4c21defdcf9b19579aeef28bdfad119c94f 100644 --- a/lib/kokkos/CHANGELOG.md +++ b/lib/kokkos/CHANGELOG.md @@ -1,5 +1,45 @@ # Change Log +## [3.0.00](https://github.com/kokkos/kokkos/tree/3.0.00) (2020-01-27) +[Full Changelog](https://github.com/kokkos/kokkos/compare/2.9.00...3.0.00) + +**Implemented enhancements:** + +- BuildSystem: Standalone Modern CMake Support [\#2104](https://github.com/kokkos/kokkos/issues/2104) +- StyleFormat: ClangFormat Style [\#2157](https://github.com/kokkos/kokkos/issues/2157) +- Documentation: Document build system and CMake philosophy [\#2263](https://github.com/kokkos/kokkos/issues/2263) +- BuildSystem: Add Alias with Namespace Kokkos:: to Interal Libraries [\#2530](https://github.com/kokkos/kokkos/issues/2530) +- BuildSystem: Universal Kokkos find\_package [\#2099](https://github.com/kokkos/kokkos/issues/2099) +- BuildSystem: Dropping support for Kokkos\_{DEVICES,OPTIONS,ARCH} in CMake [\#2329](https://github.com/kokkos/kokkos/issues/2329) +- BuildSystem: Set Kokkos\_DEVICES and Kokkos\_ARCH variables in exported CMake configuration [\#2193](https://github.com/kokkos/kokkos/issues/2193) +- BuildSystem: Drop support for CUDA 7 and CUDA 8 [\#2489](https://github.com/kokkos/kokkos/issues/2489) +- BuildSystem: Drop CMake option SEPARATE\_TESTS [\#2266](https://github.com/kokkos/kokkos/issues/2266) +- BuildSystem: Support expt-relaxed-constexpr same as expt-extended-lambda [\#2411](https://github.com/kokkos/kokkos/issues/2411) +- BuildSystem: Add Xnvlink to command line options allowed in nvcc\_wrapper [\#2197](https://github.com/kokkos/kokkos/issues/2197) +- BuildSystem: Install Kokkos config files and target files to lib/cmake/Kokkos [\#2162](https://github.com/kokkos/kokkos/issues/2162) +- BuildSystem: nvcc\_wrappers and c++ 14 [\#2035](https://github.com/kokkos/kokkos/issues/2035) +- BuildSystem: Kokkos version major/version minor \(Feature request\) [\#1930](https://github.com/kokkos/kokkos/issues/1930) +- BuildSystem: CMake namespaces \(and other modern cmake cleanup\) [\#1924](https://github.com/kokkos/kokkos/issues/1924) +- BuildSystem: Remove capability to install Kokkos via GNU Makefiles [\#2332](https://github.com/kokkos/kokkos/issues/2332) +- Documentation: Remove PDF ProgrammingGuide in Kokkos replace with link [\#2244](https://github.com/kokkos/kokkos/issues/2244) +- View: Add Method to Resize View without Initialization [\#2048](https://github.com/kokkos/kokkos/issues/2048) +- Vector: implement “insert†method for Kokkos\_Vector \(as a serial function on host\) [\#2437](https://github.com/kokkos/kokkos/issues/2437) + +**Fixed bugs:** + +- ParallelScan: Kokkos::parallel\scan fix race condition seen in inter-block fence [\#2681](https://github.com/kokkos/kokkos/issues/2681) +- OffsetView: Kokkos::OffsetView missing constructor which takes pointer [\#2247](https://github.com/kokkos/kokkos/issues/2247) +- OffsetView: Kokkos::OffsetView: allow offset=0 [\#2246](https://github.com/kokkos/kokkos/issues/2246) +- DeepCopy: Missing DeepCopy instrumentation in Kokkos [\#2522](https://github.com/kokkos/kokkos/issues/2522) +- nvcc\_wrapper: --host-only fails with multiple -W\* flags [\#2484](https://github.com/kokkos/kokkos/issues/2484) +- nvcc\_wrapper: taking first -std option is counterintuitive [\#2553](https://github.com/kokkos/kokkos/issues/2553) +- Subview: Error taking subviews of views with static_extents of min rank [\#2448](https://github.com/kokkos/kokkos/issues/2448) +- TeamPolicy: reducers with valuetypes without += broken on CUDA [\#2410](https://github.com/kokkos/kokkos/issues/2410) +- Libs: Fix inconsistency of Kokkos library names in Kokkos and Trilinos [\#1902](https://github.com/kokkos/kokkos/issues/1902) +- Complex: operator\>\> for complex\ uses std::ostream, not std::istream [\#2313](https://github.com/kokkos/kokkos/issues/2313) +- Macros: Restrict not honored for non-intel compilers [\#1922](https://github.com/kokkos/kokkos/issues/1922) + + ## [2.9.00](https://github.com/kokkos/kokkos/tree/2.9.00) (2019-06-24) [Full Changelog](https://github.com/kokkos/kokkos/compare/2.8.00...2.9.00) diff --git a/lib/kokkos/CMakeLists.txt b/lib/kokkos/CMakeLists.txt index 236f523aec21e30e731937c7d5118ec3d2e2a007..6a4451b2e787ebf0287b058ecf23adaea0e26894 100644 --- a/lib/kokkos/CMakeLists.txt +++ b/lib/kokkos/CMakeLists.txt @@ -1,128 +1,218 @@ -# Is this a build as part of Trilinos? +# We want to determine if options are given with the wrong case +# In order to detect which arguments are given to compare against +# the list of valid arguments, at the beginning here we need to +# form a list of all the given variables. If it begins with any +# case of KoKkOS, we add it to the list. + + +GET_CMAKE_PROPERTY(_variableNames VARIABLES) +SET(KOKKOS_GIVEN_VARIABLES) +FOREACH (var ${_variableNames}) + STRING(TOUPPER ${var} UC_VAR) + STRING(FIND ${UC_VAR} KOKKOS IDX) + IF (${IDX} EQUAL 0) + LIST(APPEND KOKKOS_GIVEN_VARIABLES ${var}) + ENDIF() +ENDFOREACH() + +# Basic initialization (Used in KOKKOS_SETTINGS) +SET(Kokkos_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +SET(KOKKOS_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +SET(KOKKOS_SRC_PATH ${Kokkos_SOURCE_DIR}) +SET(KOKKOS_PATH ${Kokkos_SOURCE_DIR}) +SET(KOKKOS_TOP_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}) + +# Needed to simplify syntax of if statements +CMAKE_POLICY(SET CMP0054 NEW) + +# Is this a build as part of Trilinos? IF(COMMAND TRIBITS_PACKAGE_DECL) - SET(KOKKOS_HAS_TRILINOS ON CACHE BOOL "") + SET(KOKKOS_HAS_TRILINOS ON) ELSE() - SET(KOKKOS_HAS_TRILINOS OFF CACHE BOOL "") + SET(KOKKOS_HAS_TRILINOS OFF) ENDIF() -IF(NOT KOKKOS_HAS_TRILINOS) - cmake_minimum_required(VERSION 3.3 FATAL_ERROR) +INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_functions.cmake) +INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_pick_cxx_std.cmake) + +SET(KOKKOS_ENABLED_OPTIONS) #exported in config file +SET(KOKKOS_ENABLED_DEVICES) #exported in config file +SET(KOKKOS_ENABLED_TPLS) #exported in config file +SET(KOKKOS_ENABLED_ARCH_LIST) #exported in config file + +#These are helper flags used for sanity checks during config +#Certain features should depend on other features being configured first +SET(KOKKOS_CFG_DAG_NONE On) #sentinel to indicate no dependencies +SET(KOKKOS_CFG_DAG_DEVICES_DONE Off) +SET(KOKKOS_CFG_DAG_OPTIONS_DONE Off) +SET(KOKKOS_CFG_DAG_ARCH_DONE Off) +SET(KOKKOS_CFG_DAG_CXX_STD_DONE Off) +SET(KOKKOS_CFG_DAG_COMPILER_ID_DONE Off) +FUNCTION(KOKKOS_CFG_DEPENDS SUCCESSOR PRECURSOR) + SET(PRE_FLAG KOKKOS_CFG_DAG_${PRECURSOR}) + SET(POST_FLAG KOKKOS_CFG_DAG_${SUCCESSOR}) + IF (NOT ${PRE_FLAG}) + MESSAGE(FATAL_ERROR "Bad CMake refactor: feature ${SUCCESSOR} cannot be configured until ${PRECURSOR} is configured") + ENDIF() + GLOBAL_SET(${POST_FLAG} On) +ENDFUNCTION() - # Define Project Name if this is a standalone build + +LIST(APPEND CMAKE_MODULE_PATH cmake/Modules) + +IF(NOT KOKKOS_HAS_TRILINOS) + cmake_minimum_required(VERSION 3.10 FATAL_ERROR) + set(CMAKE_DISABLE_SOURCE_CHANGES ON) + set(CMAKE_DISABLE_IN_SOURCE_BUILD ON) + IF (Spack_WORKAROUND) + #if we are explicitly using Spack for development, + #nuke the Spack compiler + SET(SPACK_CXX $ENV{SPACK_CXX}) + IF(SPACK_CXX) + SET(CMAKE_CXX_COMPILER ${SPACK_CXX} CACHE STRING "the C++ compiler" FORCE) + SET(ENV{CXX} ${SPACK_CXX}) + ENDIF() + ENDif() IF(NOT DEFINED ${PROJECT_NAME}) - project(Kokkos CXX) + PROJECT(Kokkos CXX) ENDIF() +ENDIF() - # Basic initialization (Used in KOKKOS_SETTINGS) - set(KOKKOS_SRC_PATH ${Kokkos_SOURCE_DIR}) - set(KOKKOS_PATH ${KOKKOS_SRC_PATH}) - - #------------ COMPILER AND FEATURE CHECKS ------------------------------------ - include(${KOKKOS_SRC_PATH}/cmake/kokkos_functions.cmake) - set_kokkos_cxx_compiler() - set_kokkos_cxx_standard() - - #------------ GET OPTIONS AND KOKKOS_SETTINGS -------------------------------- - # Add Kokkos' modules to CMake's module path. - set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${Kokkos_SOURCE_DIR}/cmake/Modules/") - - set(KOKKOS_CMAKE_VERBOSE True) - include(${KOKKOS_SRC_PATH}/cmake/kokkos_options.cmake) - - include(${KOKKOS_SRC_PATH}/cmake/kokkos_settings.cmake) - - #------------ GENERATE HEADER AND SOURCE FILES ------------------------------- - execute_process( - COMMAND ${KOKKOS_SETTINGS} make -f ${KOKKOS_SRC_PATH}/cmake/Makefile.generate_cmake_settings CXX=${CMAKE_CXX_COMPILER} PREFIX=${CMAKE_INSTALL_PREFIX} generate_build_settings - WORKING_DIRECTORY "${Kokkos_BINARY_DIR}" - OUTPUT_FILE ${Kokkos_BINARY_DIR}/core_src_make.out - RESULT_VARIABLE GEN_SETTINGS_RESULT - ) - if (GEN_SETTINGS_RESULT) - message(FATAL_ERROR "Kokkos settings generation failed:\n" - "${KOKKOS_SETTINGS} make -f ${KOKKOS_SRC_PATH}/cmake/Makefile.generate_cmake_settings CXX=${CMAKE_CXX_COMPILER} generate_build_settings") - endif() - include(${Kokkos_BINARY_DIR}/kokkos_generated_settings.cmake) - install(FILES ${Kokkos_BINARY_DIR}/kokkos_generated_settings.cmake DESTINATION lib/cmake/Kokkos) - install(FILES ${Kokkos_BINARY_DIR}/kokkos_generated_settings.cmake DESTINATION ${CMAKE_INSTALL_PREFIX}) - string(REPLACE " " ";" KOKKOS_TPL_INCLUDE_DIRS "${KOKKOS_GMAKE_TPL_INCLUDE_DIRS}") - string(REPLACE " " ";" KOKKOS_TPL_LIBRARY_DIRS "${KOKKOS_GMAKE_TPL_LIBRARY_DIRS}") - string(REPLACE " " ";" KOKKOS_TPL_LIBRARY_NAMES "${KOKKOS_GMAKE_TPL_LIBRARY_NAMES}") - list(REMOVE_ITEM KOKKOS_TPL_INCLUDE_DIRS "") - list(REMOVE_ITEM KOKKOS_TPL_LIBRARY_DIRS "") - list(REMOVE_ITEM KOKKOS_TPL_LIBRARY_NAMES "") - set_kokkos_srcs(KOKKOS_SRC ${KOKKOS_SRC}) - - #------------ NOW BUILD ------------------------------------------------------ - include(${KOKKOS_SRC_PATH}/cmake/kokkos_build.cmake) - - #------------ Add in Fake Tribits Handling to allow unit test builds- -------- - - include(${KOKKOS_SRC_PATH}/cmake/tribits.cmake) - - TRIBITS_PACKAGE_DECL(Kokkos) - - ADD_SUBDIRECTORY(core) - ADD_SUBDIRECTORY(containers) - ADD_SUBDIRECTORY(algorithms) +IF (NOT CMAKE_SIZEOF_VOID_P) + STRING(FIND ${CMAKE_CXX_COMPILER} nvcc_wrapper FIND_IDX) + IF (NOT FIND_IDX STREQUAL -1) + MESSAGE(FATAL_ERROR "Kokkos did not configure correctly and failed to validate compiler. The most likely cause is CUDA linkage using nvcc_wrapper. Please ensure your CUDA environment is correctly configured.") + ELSE() + MESSAGE(FATAL_ERROR "Kokkos did not configure correctly and failed to validate compiler. The most likely cause is linkage errors during CMake compiler validation. Please consult the CMake error log shown below for the exact error during compiler validation") + ENDIF() +ELSEIF (NOT CMAKE_SIZEOF_VOID_P EQUAL 8) + MESSAGE(FATAL_ERROR "Kokkos assumes a 64-bit build; i.e., 8-byte pointers, but found ${CMAKE_SIZEOF_VOID_P}-byte pointers instead") +ENDIF() -ELSE() -#------------------------------------------------------------------------------ -# -# A) Forward declare the package so that certain options are also defined for -# subpackages -# -TRIBITS_PACKAGE_DECL(Kokkos) # ENABLE_SHADOWING_WARNINGS) +set(Kokkos_VERSION_MAJOR 3) +set(Kokkos_VERSION_MINOR 0) +set(Kokkos_VERSION_PATCH 0) +set(Kokkos_VERSION "${Kokkos_VERSION_MAJOR}.${Kokkos_VERSION_MINOR}.${Kokkos_VERSION_PATCH}") +IF(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.12.0") + MESSAGE(STATUS "Setting policy CMP0074 to use _ROOT variables") + CMAKE_POLICY(SET CMP0074 NEW) +ENDIF() + +# Load either the real TriBITS or a TriBITS wrapper +# for certain utility functions that are universal (like GLOBAL_SET) +INCLUDE(${KOKKOS_SRC_PATH}/cmake/fake_tribits.cmake) + +IF (Kokkos_ENABLE_CUDA AND ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.14.0") + #If we are building CUDA, we have tricked CMake because we declare a CXX project + #If the default C++ standard for a given compiler matches the requested + #standard, then CMake just omits the -std flag in later versions of CMake + #This breaks CUDA compilation (CUDA compiler can have a different default + #-std then the underlying host compiler by itself). Setting this variable + #forces CMake to always add the -std flag even if it thinks it doesn't need it + GLOBAL_SET(CMAKE_CXX_STANDARD_DEFAULT 98) +ENDIF() + +# These are the variables we will append to as we go +# I really wish these were regular variables +# but scoping issues can make it difficult +GLOBAL_RESET(KOKKOS_COMPILE_OPTIONS) +GLOBAL_RESET(KOKKOS_LINK_OPTIONS) +GLOBAL_RESET(KOKKOS_CUDA_OPTIONS) +GLOBAL_RESET(KOKKOS_CUDAFE_OPTIONS) +GLOBAL_RESET(KOKKOS_XCOMPILER_OPTIONS) +# We need to append text here for making sure TPLs +# we import are available for an installed Kokkos +GLOBAL_RESET(KOKKOS_TPL_EXPORTS) +# We need these for controlling the exact -std flag +GLOBAL_RESET(KOKKOS_DONT_ALLOW_EXTENSIONS) +GLOBAL_RESET(KOKKOS_USE_CXX_EXTENSIONS) +GLOBAL_RESET(KOKKOS_CXX_STANDARD_FEATURE) + +# Include a set of Kokkos-specific wrapper functions that +# will either call raw CMake or TriBITS +# These are functions like KOKKOS_INCLUDE_DIRECTORIES +INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_tribits.cmake) + + +# The build environment setup goes in the following steps +# 1) Check all the enable options. This includes checking Kokkos_DEVICES +# 2) Check the compiler ID (type and version) +# 3) Check the CXX standard and select important CXX flags +# 4) Check for any third-party libraries (TPLs) like hwloc +# 5) Check if optimizing for a particular architecture and add arch-specific flags +KOKKOS_SETUP_BUILD_ENVIRONMENT() + +# Finish off the build +# 6) Recurse into subdirectories and configure individual libraries +# 7) Export and install targets + +OPTION(BUILD_SHARED_LIBS "Build shared libraries" OFF) +# Workaround for building position independent code. +IF(BUILD_SHARED_LIBS) + SET(CMAKE_POSITION_INDEPENDENT_CODE ON) +ENDIF() + +SET(KOKKOS_EXT_LIBRARIES Kokkos::kokkos Kokkos::kokkoscore Kokkos::kokkoscontainers Kokkos::kokkosalgorithms) +SET(KOKKOS_INT_LIBRARIES kokkos kokkoscore kokkoscontainers kokkosalgorithms) +SET_PROPERTY(GLOBAL PROPERTY KOKKOS_INT_LIBRARIES ${KOKKOS_INT_LIBRARIES}) + +GET_DIRECTORY_PROPERTY(HAS_PARENT PARENT_DIRECTORY) +IF (KOKKOS_HAS_TRILINOS) + SET(TRILINOS_INCDIR ${CMAKE_INSTALL_PREFIX}/${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}) + SET(KOKKOS_HEADER_DIR ${TRILINOS_INCDIR}) + SET(KOKKOS_IS_SUBDIRECTORY TRUE) +ELSEIF(HAS_PARENT) + SET(KOKKOS_HEADER_DIR "include/kokkos") + SET(KOKKOS_IS_SUBDIRECTORY TRUE) +ELSE() + SET(KOKKOS_HEADER_DIR "${CMAKE_INSTALL_INCLUDEDIR}") + SET(KOKKOS_IS_SUBDIRECTORY FALSE) +ENDIF() -#------------------------------------------------------------------------------ -# -# B) Install Kokkos' build files -# -# If using the Makefile-generated files, then need to set things up. -# Here, assume that TriBITS has been run from ProjectCompilerPostConfig.cmake -# and already generated KokkosCore_config.h and kokkos_generated_settings.cmake -# in the previously define Kokkos_GEN_DIR -# We need to copy them over to the correct place and source the cmake file - -if(NOT KOKKOS_LEGACY_TRIBITS) - set(Kokkos_GEN_DIR ${CMAKE_BINARY_DIR}) - file(COPY "${Kokkos_GEN_DIR}/KokkosCore_config.h" - DESTINATION "${CMAKE_CURRENT_BINARY_DIR}" USE_SOURCE_PERMISSIONS) - install(FILES "${Kokkos_GEN_DIR}/KokkosCore_config.h" - DESTINATION include) - file(COPY "${Kokkos_GEN_DIR}/kokkos_generated_settings.cmake" - DESTINATION "${CMAKE_CURRENT_BINARY_DIR}" USE_SOURCE_PERMISSIONS) - - include(${CMAKE_CURRENT_BINARY_DIR}/kokkos_generated_settings.cmake) - # Sources come from makefile-generated kokkos_generated_settings.cmake file - # Enable using the individual sources if needed - set_kokkos_srcs(KOKKOS_SRC ${KOKKOS_SRC}) -endif () #------------------------------------------------------------------------------ # -# C) Install Kokkos' executable scripts -# +# A) Forward declare the package so that certain options are also defined for +# subpackages -# nvcc_wrapper is Kokkos' wrapper for NVIDIA's NVCC CUDA compiler. -# Kokkos needs nvcc_wrapper in order to build. Other libraries and -# executables also need nvcc_wrapper. Thus, we need to install it. -# If the argument of DESTINATION is a relative path, CMake computes it -# as relative to ${CMAKE_INSTALL_PATH}. +## This restores the old behavior of ProjectCompilerPostConfig.cmake +# It sets the CMAKE_CXX_FLAGS globally to those used by Kokkos +# We must do this before KOKKOS_PACKAGE_DECL +IF (KOKKOS_HAS_TRILINOS) + # Overwrite the old flags at the top-level + # Because Tribits doesn't use lists, it uses spaces for the list of CXX flags + # we have to match the annoying behavior + STRING(REPLACE ";" " " KOKKOSCORE_COMPILE_OPTIONS "${KOKKOS_COMPILE_OPTIONS}") + STRING(REPLACE ";" " " KOKKOSCORE_CUDA_OPTIONS "${KOKKOS_CUDA_OPTIONS}") + FOREACH(CUDAFE_FLAG ${KOKKOS_CUDAFE_OPTIONS}) + SET(KOKKOSCORE_CUDAFE_OPTIONS "${KOKKOSCORE_CUDAFE_OPTIONS} -Xcudafe ${CUDAFE_FLAG}") + ENDFOREACH() + FOREACH(XCOMP_FLAG ${KOKKOS_XCOMPILER_OPTIONS}) + SET(KOKKOSCORE_XCOMPILER_OPTIONS "${KOKKOSCORE_XCOMPILER_OPTIONS} -Xcompiler ${XCOMP_FLAG}") + ENDFOREACH() + SET(KOKKOSCORE_CXX_FLAGS "${KOKKOSCORE_COMPILE_OPTIONS} ${CMAKE_CXX${KOKKOS_CXX_STANDARD}_STANDARD_COMPILE_OPTION} ${KOKKOSCORE_CUDA_OPTIONS} ${KOKKOSCORE_CUDAFE_OPTIONS} ${KOKKOSCORE_XCOMPILER_OPTIONS}") + # Both parent scope and this package + # In ProjectCompilerPostConfig.cmake, we capture the "global" flags Trilinos wants in + # TRILINOS_TOPLEVEL_CXX_FLAGS + SET(CMAKE_CXX_FLAGS "${TRILINOS_TOPLEVEL_CXX_FLAGS} ${KOKKOSCORE_CXX_FLAGS}" PARENT_SCOPE) + SET(CMAKE_CXX_FLAGS "${TRILINOS_TOPLEVEL_CXX_FLAGS} ${KOKKOSCORE_CXX_FLAGS}") + #CMAKE_CXX_FLAGS will get added to Kokkos and Kokkos dependencies automatically here + #These flags get set up in KOKKOS_PACKAGE_DECL, which means they + #must be configured before KOKKOS_PACKAGE_DECL +ENDIF() -INSTALL(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/bin/nvcc_wrapper DESTINATION bin) +KOKKOS_PACKAGE_DECL() #------------------------------------------------------------------------------ # -# D) Process the subpackages for Kokkos +# D) Process the subpackages (subdirectories) for Kokkos # - -TRIBITS_PROCESS_SUBPACKAGES() +KOKKOS_PROCESS_SUBPACKAGES() #------------------------------------------------------------------------------ @@ -130,10 +220,39 @@ TRIBITS_PROCESS_SUBPACKAGES() # E) If Kokkos itself is enabled, process the Kokkos package # -TRIBITS_PACKAGE_DEF() +KOKKOS_PACKAGE_DEF() +KOKKOS_EXCLUDE_AUTOTOOLS_FILES() +KOKKOS_PACKAGE_POSTPROCESS() -TRIBITS_EXCLUDE_AUTOTOOLS_FILES() +#We are ready to configure the header +CONFIGURE_FILE(cmake/KokkosCore_config.h.in KokkosCore_config.h @ONLY) -TRIBITS_PACKAGE_POSTPROCESS() +IF (NOT KOKKOS_HAS_TRILINOS) + ADD_LIBRARY(kokkos INTERFACE) + #Make sure in-tree projects can reference this as Kokkos:: + #to match the installed target names + ADD_LIBRARY(Kokkos::kokkos ALIAS kokkos) + TARGET_LINK_LIBRARIES(kokkos INTERFACE kokkoscore kokkoscontainers kokkosalgorithms) + KOKKOS_INTERNAL_ADD_LIBRARY_INSTALL(kokkos) +ENDIF() +INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_install.cmake) +# nvcc_wrapper is Kokkos' wrapper for NVIDIA's NVCC CUDA compiler. +# Kokkos needs nvcc_wrapper in order to build. Other libraries and +# executables also need nvcc_wrapper. Thus, we need to install it. +# If the argument of DESTINATION is a relative path, CMake computes it +# as relative to ${CMAKE_INSTALL_PATH}. +INSTALL(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/bin/nvcc_wrapper DESTINATION ${CMAKE_INSTALL_BINDIR}) +INSTALL(FILES "${CMAKE_CURRENT_BINARY_DIR}/KokkosCore_config.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + + +# Finally - if we are a subproject - make sure the enabled devices are visible +IF (HAS_PARENT) + FOREACH(DEV Kokkos_ENABLED_DEVICES) + #I would much rather not make these cache variables or global properties, but I can't + #make any guarantees on whether PARENT_SCOPE is good enough to make + #these variables visible where I need them + SET(Kokkos_ENABLE_${DEV} ON PARENT_SCOPE) + SET_PROPERTY(GLOBAL PROPERTY Kokkos_ENABLE_${DEV} ON) + ENDFOREACH() ENDIF() diff --git a/lib/kokkos/CONTRIBUTING.md b/lib/kokkos/CONTRIBUTING.md new file mode 100644 index 0000000000000000000000000000000000000000..b4f3057cef2cb707b55b3ae3331000ff4e55c0e1 --- /dev/null +++ b/lib/kokkos/CONTRIBUTING.md @@ -0,0 +1,14 @@ +# Contributing to Kokkos + +## Pull Requests +We actively welcome pull requests. +1. Fork the repo and create your branch from `develop`. +2. If you've added code that should be tested, add tests. +3. If you've changed APIs, update the documentation. +4. Ensure the test suite passes. + +## Issues +We use GitHub issues to track public bugs. Please ensure your description is clear and has sufficient instructions to be able to reproduce the issue. + +## License +By contributing to Kokkos, you agree that your contributions will be licensed under the LICENSE file in the root directory of this source tree. diff --git a/lib/kokkos/Copyright.txt b/lib/kokkos/Copyright.txt index 50b76995af47381395ba4b9d0ad72ac7f57e4655..06184796b268ee8b6edb8255aebcd0d28f8dfaa5 100644 --- a/lib/kokkos/Copyright.txt +++ b/lib/kokkos/Copyright.txt @@ -1,10 +1,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -22,10 +23,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR diff --git a/lib/kokkos/LICENSE b/lib/kokkos/LICENSE index c68a8a2a9f98983e6bb7e02aaae0ce705a241ffc..c6f17087d5a1b160a5fddeaab39ed9380328b485 100644 --- a/lib/kokkos/LICENSE +++ b/lib/kokkos/LICENSE @@ -1,10 +1,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Kokkos is licensed under 3-clause BSD terms of use: @@ -24,10 +25,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR diff --git a/lib/kokkos/Makefile.kokkos b/lib/kokkos/Makefile.kokkos index e9ad57f0aebc71bea0f17a73b7b8c675e725fe6b..e2b34fb4538669d82477b802219c3264c1f16074 100644 --- a/lib/kokkos/Makefile.kokkos +++ b/lib/kokkos/Makefile.kokkos @@ -23,14 +23,16 @@ KOKKOS_DEBUG ?= "no" KOKKOS_USE_TPLS ?= "" # Options: c++11,c++14,c++1y,c++17,c++1z,c++2a KOKKOS_CXX_STANDARD ?= "c++11" -# Options: aggressive_vectorization,disable_profiling,enable_deprecated_code,disable_deprecated_code,enable_large_mem_tests +# Options: aggressive_vectorization,disable_profiling,enable_deprecated_code,disable_deprecated_code,enable_large_mem_tests,disable_complex_align KOKKOS_OPTIONS ?= "" # Option for setting ETI path KOKKOS_ETI_PATH ?= ${KOKKOS_PATH}/core/src/eti KOKKOS_CMAKE ?= "no" +KOKKOS_TRIBITS ?= "no" +KOKKOS_STANDALONE_CMAKE ?= "no" # Default settings specific options. -# Options: force_uvm,use_ldg,rdc,enable_lambda +# Options: force_uvm,use_ldg,rdc,enable_lambda,enable_constexpr KOKKOS_CUDA_OPTIONS ?= "enable_lambda" # Default settings specific options. @@ -47,7 +49,8 @@ kokkos_has_string=$(if $(findstring $2,$1),1,0) # Will return a 1 if /path/to/file exists kokkos_path_exists=$(if $(wildcard $1),1,0) -# Check for general settings. +# Check for general settings + KOKKOS_INTERNAL_ENABLE_DEBUG := $(call kokkos_has_string,$(KOKKOS_DEBUG),yes) KOKKOS_INTERNAL_ENABLE_CXX11 := $(call kokkos_has_string,$(KOKKOS_CXX_STANDARD),c++11) KOKKOS_INTERNAL_ENABLE_CXX14 := $(call kokkos_has_string,$(KOKKOS_CXX_STANDARD),c++14) @@ -67,6 +70,7 @@ KOKKOS_INTERNAL_OPT_RANGE_AGGRESSIVE_VECTORIZATION := $(call kokkos_has_string,$ KOKKOS_INTERNAL_DISABLE_PROFILING := $(call kokkos_has_string,$(KOKKOS_OPTIONS),disable_profiling) KOKKOS_INTERNAL_DISABLE_DEPRECATED_CODE := $(call kokkos_has_string,$(KOKKOS_OPTIONS),disable_deprecated_code) KOKKOS_INTERNAL_ENABLE_DEPRECATED_CODE := $(call kokkos_has_string,$(KOKKOS_OPTIONS),enable_deprecated_code) +KOKKOS_INTERNAL_DISABLE_COMPLEX_ALIGN := $(call kokkos_has_string,$(KOKKOS_OPTIONS),disable_complex_align) KOKKOS_INTERNAL_DISABLE_DUALVIEW_MODIFY_CHECK := $(call kokkos_has_string,$(KOKKOS_OPTIONS),disable_dualview_modify_check) KOKKOS_INTERNAL_ENABLE_PROFILING_LOAD_PRINT := $(call kokkos_has_string,$(KOKKOS_OPTIONS),enable_profile_load_print) KOKKOS_INTERNAL_ENABLE_LARGE_MEM_TESTS := $(call kokkos_has_string,$(KOKKOS_OPTIONS),enable_large_mem_tests) @@ -74,6 +78,7 @@ KOKKOS_INTERNAL_CUDA_USE_LDG := $(call kokkos_has_string,$(KOKKOS_CUDA_OPTIONS), KOKKOS_INTERNAL_CUDA_USE_UVM := $(call kokkos_has_string,$(KOKKOS_CUDA_OPTIONS),force_uvm) KOKKOS_INTERNAL_CUDA_USE_RELOC := $(call kokkos_has_string,$(KOKKOS_CUDA_OPTIONS),rdc) KOKKOS_INTERNAL_CUDA_USE_LAMBDA := $(call kokkos_has_string,$(KOKKOS_CUDA_OPTIONS),enable_lambda) +KOKKOS_INTERNAL_CUDA_USE_CONSTEXPR := $(call kokkos_has_string,$(KOKKOS_CUDA_OPTIONS),enable_constexpr) KOKKOS_INTERNAL_HPX_ENABLE_ASYNC_DISPATCH := $(call kokkos_has_string,$(KOKKOS_HPX_OPTIONS),enable_async_dispatch) KOKKOS_INTERNAL_ENABLE_ETI := $(call kokkos_has_string,$(KOKKOS_OPTIONS),enable_eti) @@ -123,7 +128,7 @@ KOKKOS_INTERNAL_COMPILER_INTEL := $(call kokkos_has_string,$(KOKKOS_CXX_VE KOKKOS_INTERNAL_COMPILER_PGI := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),PGI) KOKKOS_INTERNAL_COMPILER_XL := $(strip $(shell $(CXX) -qversion 2>&1 | grep XL | wc -l)) KOKKOS_INTERNAL_COMPILER_CRAY := $(strip $(shell $(CXX) -craype-verbose 2>&1 | grep "CC-" | wc -l)) -KOKKOS_INTERNAL_COMPILER_NVCC := $(strip $(shell export OMPI_CXX=$(OMPI_CXX); export MPICH_CXX=$(MPICH_CXX); $(CXX) --version 2>&1 | grep nvcc | wc -l)) +KOKKOS_INTERNAL_COMPILER_NVCC := $(strip $(shell echo "$(shell export OMPI_CXX=$(OMPI_CXX); export MPICH_CXX=$(MPICH_CXX); $(CXX) --version 2>&1 | grep nvcc | wc -l)>0" | bc)) KOKKOS_INTERNAL_COMPILER_CLANG := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),clang) KOKKOS_INTERNAL_COMPILER_APPLE_CLANG := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),Apple LLVM) KOKKOS_INTERNAL_COMPILER_HCC := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),HCC) @@ -383,10 +388,10 @@ endif # Generating the list of Flags. -#CPPFLAGS is now unused KOKKOS_CPPFLAGS = +KOKKOS_LIBDIRS = ifneq ($(KOKKOS_CMAKE), yes) - KOKKOS_CXXFLAGS = -I./ -I$(KOKKOS_PATH)/core/src -I$(KOKKOS_PATH)/containers/src -I$(KOKKOS_PATH)/algorithms/src -I$(KOKKOS_ETI_PATH) + KOKKOS_CPPFLAGS = -I./ -I$(KOKKOS_PATH)/core/src -I$(KOKKOS_PATH)/containers/src -I$(KOKKOS_PATH)/algorithms/src -I$(KOKKOS_ETI_PATH) endif KOKKOS_TPL_INCLUDE_DIRS = KOKKOS_TPL_LIBRARY_DIRS = @@ -399,7 +404,7 @@ endif KOKKOS_LIBS = -ldl KOKKOS_TPL_LIBRARY_NAMES += dl ifneq ($(KOKKOS_CMAKE), yes) - KOKKOS_LDFLAGS = -L$(shell pwd) + KOKKOS_LIBDIRS = -L$(shell pwd) # CXXLDFLAGS is used together with CXXFLAGS in a combined compile/link command KOKKOS_CXXLDFLAGS = -L$(shell pwd) endif @@ -492,28 +497,38 @@ ifeq ($(KOKKOS_INTERNAL_USE_ISA_POWERPCBE), 1) tmp := $(call kokkos_append_header,"\#endif") endif +#only add the c++ standard flags if this is not CMake tmp := $(call kokkos_append_header,"/* General Settings */") ifeq ($(KOKKOS_INTERNAL_ENABLE_CXX11), 1) +ifneq ($(KOKKOS_STANDALONE_CMAKE), yes) KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_CXX11_FLAG) +endif tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_CXX11") endif ifeq ($(KOKKOS_INTERNAL_ENABLE_CXX14), 1) +ifneq ($(KOKKOS_STANDALONE_CMAKE), yes) KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_CXX14_FLAG) +endif tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_CXX14") endif ifeq ($(KOKKOS_INTERNAL_ENABLE_CXX1Y), 1) + #I cannot make CMake add this in a good way - so add it here KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_CXX1Y_FLAG) tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_CXX14") endif ifeq ($(KOKKOS_INTERNAL_ENABLE_CXX17), 1) +ifneq ($(KOKKOS_STANDALONE_CMAKE), yes) KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_CXX17_FLAG) +endif tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_CXX17") endif ifeq ($(KOKKOS_INTERNAL_ENABLE_CXX1Z), 1) + #I cannot make CMake add this in a good way - so add it here KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_CXX1Z_FLAG) tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_CXX17") endif ifeq ($(KOKKOS_INTERNAL_ENABLE_CXX2A), 1) + #I cannot make CMake add this in a good way - so add it here KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_CXX2A_FLAG) tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_CXX20") endif @@ -531,23 +546,26 @@ ifeq ($(KOKKOS_INTERNAL_ENABLE_DEBUG), 1) tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK") endif endif +ifeq ($(KOKKOS_INTERNAL_DISABLE_COMPLEX_ALIGN), 0) + tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_COMPLEX_ALIGN") +endif ifeq ($(KOKKOS_INTERNAL_ENABLE_PROFILING_LOAD_PRINT), 1) tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_PROFILING_LOAD_PRINT") endif ifeq ($(KOKKOS_INTERNAL_USE_HWLOC), 1) - ifneq ($(HWLOC_PATH),) - ifneq ($(KOKKOS_CMAKE), yes) - KOKKOS_CXXFLAGS += -I$(HWLOC_PATH)/include + ifneq ($(KOKKOS_CMAKE), yes) + ifneq ($(HWLOC_PATH),) + KOKKOS_CPPFLAGS += -I$(HWLOC_PATH)/include + KOKKOS_LIBDIRS += -L$(HWLOC_PATH)/lib + KOKKOS_CXXLDFLAGS += -L$(HWLOC_PATH)/lib + KOKKOS_TPL_INCLUDE_DIRS += $(HWLOC_PATH)/include + KOKKOS_TPL_LIBRARY_DIRS += $(HWLOC_PATH)/lib endif - KOKKOS_LDFLAGS += -L$(HWLOC_PATH)/lib - KOKKOS_CXXLDFLAGS += -L$(HWLOC_PATH)/lib - KOKKOS_TPL_INCLUDE_DIRS += $(HWLOC_PATH)/include - KOKKOS_TPL_LIBRARY_DIRS += $(HWLOC_PATH)/lib + KOKKOS_LIBS += -lhwloc + KOKKOS_TPL_LIBRARY_NAMES += hwloc endif - KOKKOS_LIBS += -lhwloc - KOKKOS_TPL_LIBRARY_NAMES += hwloc tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_HWLOC") endif @@ -558,17 +576,17 @@ ifeq ($(KOKKOS_INTERNAL_USE_LIBRT), 1) endif ifeq ($(KOKKOS_INTERNAL_USE_MEMKIND), 1) - ifneq ($(MEMKIND_PATH),) - ifneq ($(KOKKOS_CMAKE), yes) - KOKKOS_CXXFLAGS += -I$(MEMKIND_PATH)/include + ifneq ($(KOKKOS_CMAKE), yes) + ifneq ($(MEMKIND_PATH),) + KOKKOS_CPPFLAGS += -I$(MEMKIND_PATH)/include + KOKKOS_LIBDIRS += -L$(MEMKIND_PATH)/lib + KOKKOS_CXXLDFLAGS += -L$(MEMKIND_PATH)/lib + KOKKOS_TPL_INCLUDE_DIRS += $(MEMKIND_PATH)/include + KOKKOS_TPL_LIBRARY_DIRS += $(MEMKIND_PATH)/lib endif - KOKKOS_LDFLAGS += -L$(MEMKIND_PATH)/lib - KOKKOS_CXXLDFLAGS += -L$(MEMKIND_PATH)/lib - KOKKOS_TPL_INCLUDE_DIRS += $(MEMKIND_PATH)/include - KOKKOS_TPL_LIBRARY_DIRS += $(MEMKIND_PATH)/lib + KOKKOS_LIBS += -lmemkind -lnuma + KOKKOS_TPL_LIBRARY_NAMES += memkind numa endif - KOKKOS_LIBS += -lmemkind -lnuma - KOKKOS_TPL_LIBRARY_NAMES += memkind numa tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_HBWSPACE") endif @@ -580,9 +598,6 @@ ifeq ($(KOKKOS_INTERNAL_USE_HPX), 0) ifeq ($(KOKKOS_INTERNAL_ENABLE_DEPRECATED_CODE), 1) tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_DEPRECATED_CODE") endif - ifeq ($(KOKKOS_INTERNAL_DISABLE_DEPRECATED_CODE), 0) - tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_DEPRECATED_CODE") - endif endif ifeq ($(KOKKOS_INTERNAL_ENABLE_ETI), 1) @@ -648,6 +663,21 @@ ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1) endif endif + ifeq ($(KOKKOS_INTERNAL_CUDA_USE_CONSTEXPR), 1) + ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC), 1) + ifeq ($(shell test $(KOKKOS_INTERNAL_COMPILER_NVCC_VERSION) -ge 80; echo $$?),0) + tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_CUDA_CONSTEXPR") + KOKKOS_CXXFLAGS += -expt-relaxed-constexpr + else + $(warning Warning: Cuda relaxed constexpr support was requested but NVCC version is too low. This requires NVCC for Cuda version 8.0 or higher. Disabling relaxed constexpr support now.) + endif + endif + + ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1) + tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_CUDA_CONSTEXPR") + endif + endif + ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1) tmp := $(call kokkos_append_header,"\#define KOKKOS_IMPL_CUDA_CLANG_WORKAROUND") endif @@ -1089,15 +1119,13 @@ ifeq ($(KOKKOS_INTERNAL_ENABLE_ETI), 1) endif KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/Cuda/*.hpp) ifneq ($(CUDA_PATH),) - ifneq ($(KOKKOS_CMAKE), yes) - KOKKOS_CXXFLAGS += -I$(CUDA_PATH)/include - endif + KOKKOS_CPPLAGS += -I$(CUDA_PATH)/include ifeq ($(call kokkos_path_exists,$(CUDA_PATH)/lib64), 1) - KOKKOS_LDFLAGS += -L$(CUDA_PATH)/lib64 + KOKKOS_LIBDIRS += -L$(CUDA_PATH)/lib64 KOKKOS_CXXLDFLAGS += -L$(CUDA_PATH)/lib64 KOKKOS_TPL_LIBRARY_DIRS += $(CUDA_PATH)/lib64 else ifeq ($(call kokkos_path_exists,$(CUDA_PATH)/lib), 1) - KOKKOS_LDFLAGS += -L$(CUDA_PATH)/lib + KOKKOS_LIBDIRS += -L$(CUDA_PATH)/lib KOKKOS_CXXLDFLAGS += -L$(CUDA_PATH)/lib KOKKOS_TPL_LIBRARY_DIRS += $(CUDA_PATH)/lib else @@ -1153,17 +1181,17 @@ endif ifeq ($(KOKKOS_INTERNAL_USE_QTHREADS), 1) KOKKOS_SRC += $(wildcard $(KOKKOS_PATH)/core/src/Qthreads/*.cpp) KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/Qthreads/*.hpp) - ifneq ($(QTHREADS_PATH),) - ifneq ($(KOKKOS_CMAKE), yes) - KOKKOS_CXXFLAGS += -I$(QTHREADS_PATH)/include + ifneq ($(KOKKOS_CMAKE), yes) + ifneq ($(QTHREADS_PATH),) + KOKKOS_CPPFLAGS += -I$(QTHREADS_PATH)/include + KOKKOS_LIBDIRS += -L$(QTHREADS_PATH)/lib + KOKKOS_CXXLDFLAGS += -L$(QTHREADS_PATH)/lib + KOKKOS_TPL_INCLUDE_DIRS += $(QTHREADS_PATH)/include + KOKKOS_TPL_LIBRARY_DIRS += $(QTHREADS_PATH)/lib64 endif - KOKKOS_LDFLAGS += -L$(QTHREADS_PATH)/lib - KOKKOS_CXXLDFLAGS += -L$(QTHREADS_PATH)/lib - KOKKOS_TPL_INCLUDE_DIRS += $(QTHREADS_PATH)/include - KOKKOS_TPL_LIBRARY_DIRS += $(QTHREADS_PATH)/lib64 + KOKKOS_LIBS += -lqthread + KOKKOS_TPL_LIBRARY_NAMES += qthread endif - KOKKOS_LIBS += -lqthread - KOKKOS_TPL_LIBRARY_NAMES += qthread endif ifeq ($(KOKKOS_INTERNAL_USE_HPX), 1) @@ -1173,21 +1201,21 @@ ifeq ($(KOKKOS_INTERNAL_USE_HPX), 1) ifeq ($(KOKKOS_INTERNAL_ENABLE_DEBUG), 1) KOKKOS_CXXFLAGS += $(shell PKG_CONFIG_PATH=$(HPX_PATH)/lib64/pkgconfig pkg-config --cflags hpx_application_debug) KOKKOS_CXXLDFLAGS += $(shell PKG_CONFIG_PATH=$(HPX_PATH)/lib64/pkgconfig pkg-config --libs hpx_application_debug) - KOKKOS_LDFLAGS += $(shell PKG_CONFIG_PATH=$(HPX_PATH)/lib64/pkgconfig pkg-config --libs hpx_application_debug) + KOKKOS_LIBS += $(shell PKG_CONFIG_PATH=$(HPX_PATH)/lib64/pkgconfig pkg-config --libs hpx_application_debug) else KOKKOS_CXXFLAGS += $(shell PKG_CONFIG_PATH=$(HPX_PATH)/lib64/pkgconfig pkg-config --cflags hpx_application) KOKKOS_CXXLDFLAGS += $(shell PKG_CONFIG_PATH=$(HPX_PATH)/lib64/pkgconfig pkg-config --libs hpx_application) - KOKKOS_LDFLAGS += $(shell PKG_CONFIG_PATH=$(HPX_PATH)/lib64/pkgconfig pkg-config --libs hpx_application) + KOKKOS_LIBS += $(shell PKG_CONFIG_PATH=$(HPX_PATH)/lib64/pkgconfig pkg-config --libs hpx_application) endif else ifeq ($(KOKKOS_INTERNAL_ENABLE_DEBUG), 1) KOKKOS_CXXFLAGS += $(shell pkg-config --cflags hpx_application_debug) KOKKOS_CXXLDFLAGS += $(shell pkg-config --libs hpx_application_debug) - KOKKOS_LDFLAGS += $(shell pkg-config --libs hpx_application_debug) + KOKKOS_LIBS += $(shell pkg-config --libs hpx_application_debug) else KOKKOS_CXXFLAGS += $(shell pkg-config --cflags hpx_application) KOKKOS_CXXLDFLAGS += $(shell pkg-config --libs hpx_application) - KOKKOS_LDFLAGS += $(shell pkg-config --libs hpx_application) + KOKKOS_LIBS += $(shell pkg-config --libs hpx_application) endif endif KOKKOS_TPL_LIBRARY_NAMES += hpx @@ -1248,4 +1276,16 @@ libkokkos.a: $(KOKKOS_OBJ_LINK) $(KOKKOS_SRC) $(KOKKOS_HEADERS) ar cr libkokkos.a $(KOKKOS_OBJ_LINK) ranlib libkokkos.a +print-cxx-flags: + echo "$(KOKKOS_CXXFLAGS)" + KOKKOS_LINK_DEPENDS=libkokkos.a + +#we have carefully separated LDFLAGS from LIBS and LIBDIRS +#we have also separated CPPFLAGS from CXXFLAGS +#if this is not cmake, for backwards compatibility +#we just jam everything together into the CXXFLAGS and LDFLAGS +ifneq ($(KOKKOS_CMAKE), yes) + KOKKOS_CXXFLAGS += $(KOKKOS_CPPFLAGS) + KOKKOS_LDFLAGS += $(KOKKOS_LIBDIRS) +endif diff --git a/lib/kokkos/Makefile.targets b/lib/kokkos/Makefile.targets index e7d5a3c90769230f31672ef9f098b42ef4bb6f7d..0a1f5220161521832e0f7736789de6e88f6f11b8 100644 --- a/lib/kokkos/Makefile.targets +++ b/lib/kokkos/Makefile.targets @@ -6,6 +6,8 @@ Kokkos_CPUDiscovery.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/impl/Kokkos $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/impl/Kokkos_CPUDiscovery.cpp Kokkos_Error.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/impl/Kokkos_Error.cpp $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/impl/Kokkos_Error.cpp +Kokkos_Stacktrace.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/impl/Kokkos_Stacktrace.cpp + $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/impl/Kokkos_Stacktrace.cpp Kokkos_ExecPolicy.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/impl/Kokkos_ExecPolicy.cpp $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/impl/Kokkos_ExecPolicy.cpp Kokkos_HostSpace.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/impl/Kokkos_HostSpace.cpp diff --git a/lib/kokkos/README b/lib/kokkos/README deleted file mode 100644 index cb6ceb5581d58fd3841bb43c4797708c52f41a1e..0000000000000000000000000000000000000000 --- a/lib/kokkos/README +++ /dev/null @@ -1,193 +0,0 @@ -Kokkos Core implements a programming model in C++ for writing performance portable -applications targeting all major HPC platforms. For that purpose it provides -abstractions for both parallel execution of code and data management. -Kokkos is designed to target complex node architectures with N-level memory -hierarchies and multiple types of execution resources. It currently can use -OpenMP, Pthreads and CUDA as backend programming models. - -Kokkos Core is part of the Kokkos C++ Performance Portability Programming EcoSystem, -which also provides math kernels (https://github.com/kokkos/kokkos-kernels), as well as -profiling and debugging tools (https://github.com/kokkos/kokkos-tools). - -# Learning about Kokkos - -A programming guide can be found on the Wiki, the API reference is under development. - -For questions find us on Slack: https://kokkosteam.slack.com or open a github issue. - -For non-public questions send an email to -crtrott(at)sandia.gov - -A separate repository with extensive tutorial material can be found under -https://github.com/kokkos/kokkos-tutorials. - -Furthermore, the 'example/tutorial' directory provides step by step tutorial -examples which explain many of the features of Kokkos. They work with -simple Makefiles. To build with g++ and OpenMP simply type 'make' -in the 'example/tutorial' directory. This will build all examples in the -subfolders. To change the build options refer to the Programming Guide -in the compilation section. - -To learn more about Kokkos consider watching one of our presentations: -* GTC 2015: - - http://on-demand.gputechconf.com/gtc/2015/video/S5166.html - - http://on-demand.gputechconf.com/gtc/2015/presentation/S5166-H-Carter-Edwards.pdf - - -# Contributing to Kokkos - -We are open and try to encourage contributions from external developers. -To do so please first open an issue describing the contribution and then issue -a pull request against the develop branch. For larger features it may be good -to get guidance from the core development team first through the github issue. - -Note that Kokkos Core is licensed under standard 3-clause BSD terms of use. -Which means contributing to Kokkos allows anyone else to use your contributions -not just for public purposes but also for closed source commercial projects. -For specifics see the LICENSE file contained in the repository or distribution. - -# Requirements - -### Primary tested compilers on X86 are: - * GCC 4.8.4 - * GCC 4.9.3 - * GCC 5.1.0 - * GCC 5.5.0 - * GCC 6.1.0 - * GCC 7.2.0 - * GCC 7.3.0 - * GCC 8.1.0 - * Intel 15.0.2 - * Intel 16.0.1 - * Intel 17.0.1 - * Intel 17.4.196 - * Intel 18.2.128 - * Clang 3.6.1 - * Clang 3.7.1 - * Clang 3.8.1 - * Clang 3.9.0 - * Clang 4.0.0 - * Clang 6.0.0 for CUDA (CUDA Toolkit 9.0) - * Clang 7.0.0 for CUDA (CUDA Toolkit 9.1) - * PGI 18.7 - * NVCC 7.5 for CUDA (with gcc 4.8.4) - * NVCC 8.0.44 for CUDA (with gcc 5.3.0) - * NVCC 9.1 for CUDA (with gcc 6.1.0) - * NVCC 9.2 for CUDA (with gcc 7.2.0) - * NVCC 10.0 for CUDA (with gcc 7.4.0) - -### Primary tested compilers on Power 8 are: - * GCC 6.4.0 (OpenMP,Serial) - * GCC 7.2.0 (OpenMP,Serial) - * IBM XL 16.1.0 (OpenMP, Serial) - * NVCC 9.2.88 for CUDA (with gcc 7.2.0 and XL 16.1.0) - -### Primary tested compilers on Intel KNL are: - * Intel 16.4.258 (with gcc 4.7.2) - * Intel 17.2.174 (with gcc 4.9.3) - * Intel 18.2.199 (with gcc 4.9.3) - -### Primary tested compilers on ARM (Cavium ThunderX2) - * GCC 7.2.0 - * ARM/Clang 18.4.0 - -### Other compilers working: - * X86: - - Cygwin 2.1.0 64bit with gcc 4.9.3 - - GCC 8.1.0 (not warning free) - -### Known non-working combinations: - * Power8: - - Pthreads backend - * ARM - - Pthreads backend - - -Primary tested compiler are passing in release mode -with warnings as errors. They also are tested with a comprehensive set of -backend combinations (i.e. OpenMP, Pthreads, Serial, OpenMP+Serial, ...). -We are using the following set of flags: -GCC: -Wall -Wshadow -pedantic -Werror -Wsign-compare -Wtype-limits - -Wignored-qualifiers -Wempty-body -Wclobbered -Wuninitialized -Intel: -Wall -Wshadow -pedantic -Werror -Wsign-compare -Wtype-limits -Wuninitialized -Clang: -Wall -Wshadow -pedantic -Werror -Wsign-compare -Wtype-limits -Wuninitialized -NVCC: -Wall -Wshadow -pedantic -Werror -Wsign-compare -Wtype-limits -Wuninitialized - -Other compilers are tested occasionally, in particular when pushing from develop to -master branch, without -Werror and only for a select set of backends. - -# Running Unit Tests - -To run the unit tests create a build directory and run the following commands - -KOKKOS_PATH/generate_makefile.bash -make build-test -make test - -Run KOKKOS_PATH/generate_makefile.bash --help for more detailed options such as -changing the device type for which to build. - -# Installing the library - -To install Kokkos as a library create a build directory and run the following - -KOKKOS_PATH/generate_makefile.bash --prefix=INSTALL_PATH -make kokkoslib -make install - -KOKKOS_PATH/generate_makefile.bash --help for more detailed options such as -changing the device type for which to build. - -Note that in many cases it is preferable to build Kokkos inline with an -application. The main reason is that you may otherwise need many different -configurations of Kokkos installed depending on the required compile time -features an application needs. For example there is only one default -execution space, which means you need different installations to have OpenMP -or Pthreads as the default space. Also for the CUDA backend there are certain -choices, such as allowing relocatable device code, which must be made at -installation time. Building Kokkos inline uses largely the same process -as compiling an application against an installed Kokkos library. See for -example benchmarks/bytes_and_flops/Makefile which can be used with an installed -library and for an inline build. - -### CMake - -Kokkos supports being build as part of a CMake applications. An example can -be found in example/cmake_build. - -# Kokkos and CUDA UVM - -Kokkos does support UVM as a specific memory space called CudaUVMSpace. -Allocations made with that space are accessible from host and device. -You can tell Kokkos to use that as the default space for Cuda allocations. -In either case UVM comes with a number of restrictions: -(i) You can't access allocations on the host while a kernel is potentially -running. This will lead to segfaults. To avoid that you either need to -call Kokkos::Cuda::fence() (or just Kokkos::fence()), after kernels, or -you can set the environment variable CUDA_LAUNCH_BLOCKING=1. -Furthermore in multi socket multi GPU machines without NVLINK, UVM defaults -to using zero copy allocations for technical reasons related to using multiple -GPUs from the same process. If an executable doesn't do that (e.g. each -MPI rank of an application uses a single GPU [can be the same GPU for -multiple MPI ranks]) you can set CUDA_MANAGED_FORCE_DEVICE_ALLOC=1. -This will enforce proper UVM allocations, but can lead to errors if -more than a single GPU is used by a single process. - - -# Citing Kokkos - -If you publish work which mentions Kokkos, please cite the following paper: - -@article{CarterEdwards20143202, -title = "Kokkos: Enabling manycore performance portability through polymorphic memory access patterns ", -journal = "Journal of Parallel and Distributed Computing ", -volume = "74", -number = "12", -pages = "3202 - 3216", -year = "2014", -note = "Domain-Specific Languages and High-Level Frameworks for High-Performance Computing ", -issn = "0743-7315", -doi = "https://doi.org/10.1016/j.jpdc.2014.07.003", -url = "http://www.sciencedirect.com/science/article/pii/S0743731514001257", -author = "H. Carter Edwards and Christian R. Trott and Daniel Sunderland" -} diff --git a/lib/kokkos/README.md b/lib/kokkos/README.md new file mode 100644 index 0000000000000000000000000000000000000000..322dabfdab7745b600696b92ecb4ec7e148bc7bd --- /dev/null +++ b/lib/kokkos/README.md @@ -0,0 +1,299 @@ +![Kokkos](https://avatars2.githubusercontent.com/u/10199860?s=200&v=4) + +# Kokkos: Core Libraries + +Kokkos Core implements a programming model in C++ for writing performance portable +applications targeting all major HPC platforms. For that purpose it provides +abstractions for both parallel execution of code and data management. +Kokkos is designed to target complex node architectures with N-level memory +hierarchies and multiple types of execution resources. It currently can use +CUDA, HPX, OpenMP and Pthreads as backend programming models with several other +backends in development. + +Kokkos Core is part of the Kokkos C++ Performance Portability Programming EcoSystem, +which also provides math kernels (https://github.com/kokkos/kokkos-kernels), as well as +profiling and debugging tools (https://github.com/kokkos/kokkos-tools). + +# Learning about Kokkos + +A programming guide can be found on the Wiki, the API reference is under development. + +For questions find us on Slack: https://kokkosteam.slack.com or open a github issue. + +For non-public questions send an email to +crtrott(at)sandia.gov + +A separate repository with extensive tutorial material can be found under +https://github.com/kokkos/kokkos-tutorials. + +Furthermore, the 'example/tutorial' directory provides step by step tutorial +examples which explain many of the features of Kokkos. They work with +simple Makefiles. To build with g++ and OpenMP simply type 'make' +in the 'example/tutorial' directory. This will build all examples in the +subfolders. To change the build options refer to the Programming Guide +in the compilation section. + +To learn more about Kokkos consider watching one of our presentations: +* GTC 2015: + - http://on-demand.gputechconf.com/gtc/2015/video/S5166.html + - http://on-demand.gputechconf.com/gtc/2015/presentation/S5166-H-Carter-Edwards.pdf + + +# Contributing to Kokkos + +We are open and try to encourage contributions from external developers. +To do so please first open an issue describing the contribution and then issue +a pull request against the develop branch. For larger features it may be good +to get guidance from the core development team first through the github issue. + +Note that Kokkos Core is licensed under standard 3-clause BSD terms of use. +Which means contributing to Kokkos allows anyone else to use your contributions +not just for public purposes but also for closed source commercial projects. +For specifics see the LICENSE file contained in the repository or distribution. + +# Requirements + +### Primary tested compilers on X86 are: +* GCC 4.8.4 +* GCC 4.9.3 +* GCC 5.1.0 +* GCC 5.4.0 +* GCC 5.5.0 +* GCC 6.1.0 +* GCC 7.2.0 +* GCC 7.3.0 +* GCC 8.1.0 +* Intel 15.0.2 +* Intel 16.0.1 +* Intel 17.0.1 +* Intel 17.4.196 +* Intel 18.2.128 +* Clang 3.6.1 +* Clang 3.7.1 +* Clang 3.8.1 +* Clang 3.9.0 +* Clang 4.0.0 +* Clang 6.0.0 for CUDA (CUDA Toolkit 9.0) +* Clang 7.0.0 for CUDA (CUDA Toolkit 9.1) +* Clang 8.0.0 for CUDA (CUDA Toolkit 9.2) +* PGI 18.7 +* NVCC 9.1 for CUDA (with gcc 6.1.0) +* NVCC 9.2 for CUDA (with gcc 7.2.0) +* NVCC 10.0 for CUDA (with gcc 7.4.0) +* NVCC 10.1 for CUDA (with gcc 7.4.0) + +### Primary tested compilers on Power 8 are: +* GCC 6.4.0 (OpenMP,Serial) +* GCC 7.2.0 (OpenMP,Serial) +* IBM XL 16.1.0 (OpenMP, Serial) +* NVCC 9.2.88 for CUDA (with gcc 7.2.0 and XL 16.1.0) + +### Primary tested compilers on Intel KNL are: +* Intel 16.4.258 (with gcc 4.7.2) +* Intel 17.2.174 (with gcc 4.9.3) +* Intel 18.2.199 (with gcc 4.9.3) + +### Primary tested compilers on ARM (Cavium ThunderX2) +* GCC 7.2.0 +* ARM/Clang 18.4.0 + +### Other compilers working: +* X86: + * Cygwin 2.1.0 64bit with gcc 4.9.3 + * GCC 8.1.0 (not warning free) + +### Known non-working combinations: +* Power8: + * Pthreads backend +* ARM + * Pthreads backend + + +Primary tested compiler are passing in release mode +with warnings as errors. They also are tested with a comprehensive set of +backend combinations (i.e. OpenMP, Pthreads, Serial, OpenMP+Serial, ...). +We are using the following set of flags: +* GCC: + ```` + -Wall -Wshadow -pedantic + -Werror -Wsign-compare -Wtype-limits + -Wignored-qualifiers -Wempty-body + -Wclobbered -Wuninitialized + ```` +* Intel: + ```` + -Wall -Wshadow -pedantic + -Werror -Wsign-compare -Wtype-limits + -Wuninitialized + ```` +* Clang: + ```` + -Wall -Wshadow -pedantic + -Werror -Wsign-compare -Wtype-limits + -Wuninitialized + ```` + +* NVCC: + ```` + -Wall -Wshadow -pedantic + -Werror -Wsign-compare -Wtype-limits + -Wuninitialized + ```` + +Other compilers are tested occasionally, in particular when pushing from develop to +master branch. These are tested less rigorously without `-Werror` and only for a select set of backends. + +# Building and Installing Kokkos +Kokkos provide a CMake build system and a raw Makefile build system. +The CMake build system is strongly encouraged and will be the most rigorously supported in future releases. +Full details are given in the [build instructions](BUILD.md). Basic setups are shown here: + +## CMake + +The best way to install Kokkos is using the CMake build system. Assuming Kokkos lives in `$srcdir`: +```` +cmake $srcdir \ + -DCMAKE_CXX_COMPILER=$path_to_compiler \ + -DCMAKE_INSTALL_PREFIX=$path_to_install \ + -DKokkos_ENABLE_OPENMP=On \ + -DKokkos_ARCH_HSW=On \ + -DKokkos_ENABLE_HWLOC=On \ + -DKokkos_HWLOC_DIR=$path_to_hwloc +```` +then simply type `make install`. The Kokkos CMake package will then be installed in `$path_to_install` to be used by downstream packages. + +To validate the Kokkos build, configure with +```` + -DKokkos_ENABLE_TESTS=On +```` +and run `make test` after completing the build. + +For your CMake project using Kokkos, code such as the following: + +```` +find_package(Kokkos) +... +target_link_libraries(myTarget Kokkos::kokkos) +```` +should be added to your CMakeLists.txt. Your configure should additionally include +```` +-DKokkos_DIR=$path_to_install/cmake/lib/Kokkos +```` +or +```` +-DKokkos_ROOT=$path_to_install +```` +for the install location given above. + +## Spack +An alternative to manually building with the CMake is to use the Spack package manager. +To do so, download the `kokkos-spack` git repo and add to the package list: +```` +spack repo add $path-to-kokkos-spack +```` +A basic installation would be done as: +```` +spack install kokkos +```` +Spack allows options and and compilers to be tuned in the install command. +```` +spack install kokkos@3.0 %gcc@7.3.0 +openmp +```` +This example illustrates the three most common parameters to Spack: +* Variants: specified with, e.g. `+openmp`, this activates (or deactivates with, e.g. `~openmp`) certain options. +* Version: immediately following `kokkos` the `@version` can specify a particular Kokkos to build +* Compiler: a default compiler will be chosen if not specified, but an exact compiler version can be given with the `%`option. + +For a complete list of Kokkos options, run: +```` +spack info kokkos +```` +Spack currently installs packages to a location determined by a unique hash. This hash name is not really "human readable". +Generally, Spack usage should never really require you to reference the computer-generated unique install folder. +More details are given in the [build instructions](BUILD.md). If you must know, you can locate Spack Kokkos installations with: +```` +spack find -p kokkos ... +```` +where `...` is the unique spec identifying the particular Kokkos configuration and version. + + +## Raw Makefile +A bash script is provided to generate raw makefiles. +To install Kokkos as a library create a build directory and run the following +```` +$KOKKOS_PATH/generate_makefile.bash --prefix=$path_to_install +```` +Once the Makefile is generated, run: +```` +make kokkoslib +make install +```` +To additionally run the unit tests: +```` +make build-test +make test +```` +Run `generate_makefile.bash --help` for more detailed options such as +changing the device type for which to build. + +## Inline Builds vs. Installed Package +For individual projects, it may be preferable to build Kokkos inline rather than link to an installed package. +The main reason is that you may otherwise need many different +configurations of Kokkos installed depending on the required compile time +features an application needs. For example there is only one default +execution space, which means you need different installations to have OpenMP +or Pthreads as the default space. Also for the CUDA backend there are certain +choices, such as allowing relocatable device code, which must be made at +installation time. Building Kokkos inline uses largely the same process +as compiling an application against an installed Kokkos library. + +For CMake, this means copying over the Kokkos source code into your project and adding `add_subdirectory(kokkos)` to your CMakeLists.txt. + +For raw Makefiles, see the example benchmarks/bytes_and_flops/Makefile which can be used with an installed library and or an inline build. + +# Kokkos and CUDA UVM + +Kokkos does support UVM as a specific memory space called CudaUVMSpace. +Allocations made with that space are accessible from host and device. +You can tell Kokkos to use that as the default space for Cuda allocations. +In either case UVM comes with a number of restrictions: +* You can't access allocations on the host while a kernel is potentially +running. This will lead to segfaults. To avoid that you either need to +call Kokkos::Cuda::fence() (or just Kokkos::fence()), after kernels, or +you can set the environment variable CUDA_LAUNCH_BLOCKING=1. +* In multi socket multi GPU machines without NVLINK, UVM defaults +to using zero copy allocations for technical reasons related to using multiple +GPUs from the same process. If an executable doesn't do that (e.g. each +MPI rank of an application uses a single GPU [can be the same GPU for +multiple MPI ranks]) you can set CUDA_MANAGED_FORCE_DEVICE_ALLOC=1. +This will enforce proper UVM allocations, but can lead to errors if +more than a single GPU is used by a single process. + + +# Citing Kokkos + +If you publish work which mentions Kokkos, please cite the following paper: + +```` +@article{CarterEdwards20143202, + title = "Kokkos: Enabling manycore performance portability through polymorphic memory access patterns ", + journal = "Journal of Parallel and Distributed Computing ", + volume = "74", + number = "12", + pages = "3202 - 3216", + year = "2014", + note = "Domain-Specific Languages and High-Level Frameworks for High-Performance Computing ", + issn = "0743-7315", + doi = "https://doi.org/10.1016/j.jpdc.2014.07.003", + url = "http://www.sciencedirect.com/science/article/pii/S0743731514001257", + author = "H. Carter Edwards and Christian R. Trott and Daniel Sunderland" +} +```` + +##### [LICENSE](https://github.com/kokkos/kokkos/blob/master/LICENSE) + +[![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) + +Under the terms of Contract DE-NA0003525 with NTESS, +the U.S. Government retains certain rights in this software. + diff --git a/lib/kokkos/algorithms/CMakeLists.txt b/lib/kokkos/algorithms/CMakeLists.txt index 507c9f2fdb70662c80f591d04737d45752be81a1..38747c152c73d450fe12431707fc438f9622e293 100644 --- a/lib/kokkos/algorithms/CMakeLists.txt +++ b/lib/kokkos/algorithms/CMakeLists.txt @@ -1,12 +1,12 @@ - - -TRIBITS_SUBPACKAGE(Algorithms) - -IF(KOKKOS_HAS_TRILINOS) - ADD_SUBDIRECTORY(src) -ENDIF() - -TRIBITS_ADD_TEST_DIRECTORIES(unit_tests) -#TRIBITS_ADD_TEST_DIRECTORIES(performance_tests) - -TRIBITS_SUBPACKAGE_POSTPROCESS() + + +KOKKOS_SUBPACKAGE(Algorithms) + +ADD_SUBDIRECTORY(src) + +KOKKOS_ADD_TEST_DIRECTORIES(unit_tests) + +KOKKOS_SUBPACKAGE_POSTPROCESS() + + + diff --git a/lib/kokkos/algorithms/src/CMakeLists.txt b/lib/kokkos/algorithms/src/CMakeLists.txt index dfbf3323c2d51953a12d8e82371d9f971aaa1e13..5afd319fcc7b0348b3401193f129897bcad11ebd 100644 --- a/lib/kokkos/algorithms/src/CMakeLists.txt +++ b/lib/kokkos/algorithms/src/CMakeLists.txt @@ -1,8 +1,9 @@ -TRIBITS_CONFIGURE_FILE(${PACKAGE_NAME}_config.h) +KOKKOS_CONFIGURE_FILE(${PACKAGE_NAME}_config.h) -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) +#I have to leave these here for tribits +KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) +KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) #----------------------------------------------------------------------------- @@ -12,10 +13,18 @@ LIST(APPEND HEADERS ${CMAKE_CURRENT_BINARY_DIR}/${PACKAGE_NAME}_config.h) #----------------------------------------------------------------------------- -TRIBITS_ADD_LIBRARY( - kokkosalgorithms - HEADERS ${HEADERS} - SOURCES ${SOURCES} - DEPLIBS - ) +# We have to pass the sources in here for Tribits +# These will get ignored for standalone CMake and a true interface library made +KOKKOS_ADD_INTERFACE_LIBRARY( + kokkosalgorithms + HEADERS ${HEADERS} + SOURCES ${SOURCES} +) +KOKKOS_LIB_INCLUDE_DIRECTORIES(kokkosalgorithms + ${KOKKOS_TOP_BUILD_DIR} + ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_SOURCE_DIR} +) + + diff --git a/lib/kokkos/algorithms/src/Kokkos_Random.hpp b/lib/kokkos/algorithms/src/Kokkos_Random.hpp index da781de4fe749d6af745a9c35e29cc20e64486cd..078db18eddf699a983503763a2de98fbc618215c 100644 --- a/lib/kokkos/algorithms/src/Kokkos_Random.hpp +++ b/lib/kokkos/algorithms/src/Kokkos_Random.hpp @@ -1,13 +1,14 @@ /* //@HEADER // ************************************************************************ -// -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -36,7 +37,7 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// +// // ************************************************************************ //@HEADER */ @@ -59,6 +60,7 @@ namespace Kokkos { +// clang-format off /*Template functions to get equidistributed random numbers from a generator for a specific Scalar type template @@ -229,1019 +231,979 @@ namespace Kokkos { ViewType::value_type start, ViewType::value_type end); */ +// clang-format on - template - struct rand; - - - template - struct rand { - - KOKKOS_INLINE_FUNCTION - static short max(){return 127;} - KOKKOS_INLINE_FUNCTION - static short draw(Generator& gen) - {return short((gen.rand()&0xff+256)%256);} - KOKKOS_INLINE_FUNCTION - static short draw(Generator& gen, const char& range) - {return char(gen.rand(range));} - KOKKOS_INLINE_FUNCTION - static short draw(Generator& gen, const char& start, const char& end) - {return char(gen.rand(start,end));} - - }; - - template - struct rand { - KOKKOS_INLINE_FUNCTION - static short max(){return 32767;} - KOKKOS_INLINE_FUNCTION - static short draw(Generator& gen) - {return short((gen.rand()&0xffff+65536)%32768);} - KOKKOS_INLINE_FUNCTION - static short draw(Generator& gen, const short& range) - {return short(gen.rand(range));} - KOKKOS_INLINE_FUNCTION - static short draw(Generator& gen, const short& start, const short& end) - {return short(gen.rand(start,end));} - - }; - - template - struct rand { - KOKKOS_INLINE_FUNCTION - static int max(){return Generator::MAX_RAND;} - KOKKOS_INLINE_FUNCTION - static int draw(Generator& gen) - {return gen.rand();} - KOKKOS_INLINE_FUNCTION - static int draw(Generator& gen, const int& range) - {return gen.rand(range);} - KOKKOS_INLINE_FUNCTION - static int draw(Generator& gen, const int& start, const int& end) - {return gen.rand(start,end);} +template +struct rand; - }; +template +struct rand { + KOKKOS_INLINE_FUNCTION + static short max() { return 127; } + KOKKOS_INLINE_FUNCTION + static short draw(Generator& gen) { + return short((gen.rand() & 0xff + 256) % 256); + } + KOKKOS_INLINE_FUNCTION + static short draw(Generator& gen, const char& range) { + return char(gen.rand(range)); + } + KOKKOS_INLINE_FUNCTION + static short draw(Generator& gen, const char& start, const char& end) { + return char(gen.rand(start, end)); + } +}; - template - struct rand { - KOKKOS_INLINE_FUNCTION - static unsigned int max () { - return Generator::MAX_URAND; - } - KOKKOS_INLINE_FUNCTION - static unsigned int draw (Generator& gen) { - return gen.urand (); - } - KOKKOS_INLINE_FUNCTION - static unsigned int draw(Generator& gen, const unsigned int& range) { - return gen.urand (range); - } - KOKKOS_INLINE_FUNCTION - static unsigned int - draw (Generator& gen, const unsigned int& start, const unsigned int& end) { - return gen.urand (start, end); - } - }; +template +struct rand { + KOKKOS_INLINE_FUNCTION + static short max() { return 32767; } + KOKKOS_INLINE_FUNCTION + static short draw(Generator& gen) { + return short((gen.rand() & 0xffff + 65536) % 32768); + } + KOKKOS_INLINE_FUNCTION + static short draw(Generator& gen, const short& range) { + return short(gen.rand(range)); + } + KOKKOS_INLINE_FUNCTION + static short draw(Generator& gen, const short& start, const short& end) { + return short(gen.rand(start, end)); + } +}; - template - struct rand { - KOKKOS_INLINE_FUNCTION - static long max () { - // FIXME (mfh 26 Oct 2014) It would be better to select the - // return value at compile time, using something like enable_if. - return sizeof (long) == 4 ? - static_cast (Generator::MAX_RAND) : - static_cast (Generator::MAX_RAND64); - } - KOKKOS_INLINE_FUNCTION - static long draw (Generator& gen) { - // FIXME (mfh 26 Oct 2014) It would be better to select the - // return value at compile time, using something like enable_if. - return sizeof (long) == 4 ? - static_cast (gen.rand ()) : - static_cast (gen.rand64 ()); - } - KOKKOS_INLINE_FUNCTION - static long draw (Generator& gen, const long& range) { - // FIXME (mfh 26 Oct 2014) It would be better to select the - // return value at compile time, using something like enable_if. - return sizeof (long) == 4 ? - static_cast (gen.rand (static_cast (range))) : - static_cast (gen.rand64 (range)); - } - KOKKOS_INLINE_FUNCTION - static long draw (Generator& gen, const long& start, const long& end) { - // FIXME (mfh 26 Oct 2014) It would be better to select the - // return value at compile time, using something like enable_if. - return sizeof (long) == 4 ? - static_cast (gen.rand (static_cast (start), - static_cast (end))) : - static_cast (gen.rand64 (start, end)); - } - }; +template +struct rand { + KOKKOS_INLINE_FUNCTION + static int max() { return Generator::MAX_RAND; } + KOKKOS_INLINE_FUNCTION + static int draw(Generator& gen) { return gen.rand(); } + KOKKOS_INLINE_FUNCTION + static int draw(Generator& gen, const int& range) { return gen.rand(range); } + KOKKOS_INLINE_FUNCTION + static int draw(Generator& gen, const int& start, const int& end) { + return gen.rand(start, end); + } +}; - template - struct rand { - KOKKOS_INLINE_FUNCTION - static unsigned long max () { - // FIXME (mfh 26 Oct 2014) It would be better to select the - // return value at compile time, using something like enable_if. - return sizeof (unsigned long) == 4 ? - static_cast (Generator::MAX_URAND) : - static_cast (Generator::MAX_URAND64); - } - KOKKOS_INLINE_FUNCTION - static unsigned long draw (Generator& gen) { - // FIXME (mfh 26 Oct 2014) It would be better to select the - // return value at compile time, using something like enable_if. - return sizeof (unsigned long) == 4 ? - static_cast (gen.urand ()) : - static_cast (gen.urand64 ()); - } - KOKKOS_INLINE_FUNCTION - static unsigned long draw(Generator& gen, const unsigned long& range) { - // FIXME (mfh 26 Oct 2014) It would be better to select the - // return value at compile time, using something like enable_if. - return sizeof (unsigned long) == 4 ? - static_cast (gen.urand (static_cast (range))) : - static_cast (gen.urand64 (range)); - } - KOKKOS_INLINE_FUNCTION - static unsigned long - draw (Generator& gen, const unsigned long& start, const unsigned long& end) { - // FIXME (mfh 26 Oct 2014) It would be better to select the - // return value at compile time, using something like enable_if. - return sizeof (unsigned long) == 4 ? - static_cast (gen.urand (static_cast (start), - static_cast (end))) : - static_cast (gen.urand64 (start, end)); - } - }; +template +struct rand { + KOKKOS_INLINE_FUNCTION + static unsigned int max() { return Generator::MAX_URAND; } + KOKKOS_INLINE_FUNCTION + static unsigned int draw(Generator& gen) { return gen.urand(); } + KOKKOS_INLINE_FUNCTION + static unsigned int draw(Generator& gen, const unsigned int& range) { + return gen.urand(range); + } + KOKKOS_INLINE_FUNCTION + static unsigned int draw(Generator& gen, const unsigned int& start, + const unsigned int& end) { + return gen.urand(start, end); + } +}; - // NOTE (mfh 26 oct 2014) This is a partial specialization for long - // long, a C99 / C++11 signed type which is guaranteed to be at - // least 64 bits. Do NOT write a partial specialization for - // int64_t!!! This is just a typedef! It could be either long or - // long long. We don't know which a priori, and I've seen both. - // The types long and long long are guaranteed to differ, so it's - // always safe to specialize for both. - template - struct rand { - KOKKOS_INLINE_FUNCTION - static long long max () { - // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. - return Generator::MAX_RAND64; - } - KOKKOS_INLINE_FUNCTION - static long long draw (Generator& gen) { - // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. - return gen.rand64 (); - } - KOKKOS_INLINE_FUNCTION - static long long draw (Generator& gen, const long long& range) { - // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. - return gen.rand64 (range); - } - KOKKOS_INLINE_FUNCTION - static long long draw (Generator& gen, const long long& start, const long long& end) { - // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. - return gen.rand64 (start, end); - } - }; +template +struct rand { + KOKKOS_INLINE_FUNCTION + static long max() { + // FIXME (mfh 26 Oct 2014) It would be better to select the + // return value at compile time, using something like enable_if. + return sizeof(long) == 4 ? static_cast(Generator::MAX_RAND) + : static_cast(Generator::MAX_RAND64); + } + KOKKOS_INLINE_FUNCTION + static long draw(Generator& gen) { + // FIXME (mfh 26 Oct 2014) It would be better to select the + // return value at compile time, using something like enable_if. + return sizeof(long) == 4 ? static_cast(gen.rand()) + : static_cast(gen.rand64()); + } + KOKKOS_INLINE_FUNCTION + static long draw(Generator& gen, const long& range) { + // FIXME (mfh 26 Oct 2014) It would be better to select the + // return value at compile time, using something like enable_if. + return sizeof(long) == 4 + ? static_cast(gen.rand(static_cast(range))) + : static_cast(gen.rand64(range)); + } + KOKKOS_INLINE_FUNCTION + static long draw(Generator& gen, const long& start, const long& end) { + // FIXME (mfh 26 Oct 2014) It would be better to select the + // return value at compile time, using something like enable_if. + return sizeof(long) == 4 + ? static_cast( + gen.rand(static_cast(start), static_cast(end))) + : static_cast(gen.rand64(start, end)); + } +}; - // NOTE (mfh 26 oct 2014) This is a partial specialization for - // unsigned long long, a C99 / C++11 unsigned type which is - // guaranteed to be at least 64 bits. Do NOT write a partial - // specialization for uint64_t!!! This is just a typedef! It could - // be either unsigned long or unsigned long long. We don't know - // which a priori, and I've seen both. The types unsigned long and - // unsigned long long are guaranteed to differ, so it's always safe - // to specialize for both. - template - struct rand { - KOKKOS_INLINE_FUNCTION - static unsigned long long max () { - // FIXME (mfh 26 Oct 2014) It's legal for unsigned long long to be > 64 bits. - return Generator::MAX_URAND64; - } - KOKKOS_INLINE_FUNCTION - static unsigned long long draw (Generator& gen) { - // FIXME (mfh 26 Oct 2014) It's legal for unsigned long long to be > 64 bits. - return gen.urand64 (); - } - KOKKOS_INLINE_FUNCTION - static unsigned long long draw (Generator& gen, const unsigned long long& range) { - // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. - return gen.urand64 (range); - } - KOKKOS_INLINE_FUNCTION - static unsigned long long - draw (Generator& gen, const unsigned long long& start, const unsigned long long& end) { - // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. - return gen.urand64 (start, end); - } - }; +template +struct rand { + KOKKOS_INLINE_FUNCTION + static unsigned long max() { + // FIXME (mfh 26 Oct 2014) It would be better to select the + // return value at compile time, using something like enable_if. + return sizeof(unsigned long) == 4 + ? static_cast(Generator::MAX_URAND) + : static_cast(Generator::MAX_URAND64); + } + KOKKOS_INLINE_FUNCTION + static unsigned long draw(Generator& gen) { + // FIXME (mfh 26 Oct 2014) It would be better to select the + // return value at compile time, using something like enable_if. + return sizeof(unsigned long) == 4 + ? static_cast(gen.urand()) + : static_cast(gen.urand64()); + } + KOKKOS_INLINE_FUNCTION + static unsigned long draw(Generator& gen, const unsigned long& range) { + // FIXME (mfh 26 Oct 2014) It would be better to select the + // return value at compile time, using something like enable_if. + return sizeof(unsigned long) == 4 + ? static_cast( + gen.urand(static_cast(range))) + : static_cast(gen.urand64(range)); + } + KOKKOS_INLINE_FUNCTION + static unsigned long draw(Generator& gen, const unsigned long& start, + const unsigned long& end) { + // FIXME (mfh 26 Oct 2014) It would be better to select the + // return value at compile time, using something like enable_if. + return sizeof(unsigned long) == 4 + ? static_cast( + gen.urand(static_cast(start), + static_cast(end))) + : static_cast(gen.urand64(start, end)); + } +}; - template - struct rand { - KOKKOS_INLINE_FUNCTION - static float max(){return 1.0f;} - KOKKOS_INLINE_FUNCTION - static float draw(Generator& gen) - {return gen.frand();} - KOKKOS_INLINE_FUNCTION - static float draw(Generator& gen, const float& range) - {return gen.frand(range);} - KOKKOS_INLINE_FUNCTION - static float draw(Generator& gen, const float& start, const float& end) - {return gen.frand(start,end);} +// NOTE (mfh 26 oct 2014) This is a partial specialization for long +// long, a C99 / C++11 signed type which is guaranteed to be at +// least 64 bits. Do NOT write a partial specialization for +// int64_t!!! This is just a typedef! It could be either long or +// long long. We don't know which a priori, and I've seen both. +// The types long and long long are guaranteed to differ, so it's +// always safe to specialize for both. +template +struct rand { + KOKKOS_INLINE_FUNCTION + static long long max() { + // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. + return Generator::MAX_RAND64; + } + KOKKOS_INLINE_FUNCTION + static long long draw(Generator& gen) { + // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. + return gen.rand64(); + } + KOKKOS_INLINE_FUNCTION + static long long draw(Generator& gen, const long long& range) { + // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. + return gen.rand64(range); + } + KOKKOS_INLINE_FUNCTION + static long long draw(Generator& gen, const long long& start, + const long long& end) { + // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. + return gen.rand64(start, end); + } +}; - }; +// NOTE (mfh 26 oct 2014) This is a partial specialization for +// unsigned long long, a C99 / C++11 unsigned type which is +// guaranteed to be at least 64 bits. Do NOT write a partial +// specialization for uint64_t!!! This is just a typedef! It could +// be either unsigned long or unsigned long long. We don't know +// which a priori, and I've seen both. The types unsigned long and +// unsigned long long are guaranteed to differ, so it's always safe +// to specialize for both. +template +struct rand { + KOKKOS_INLINE_FUNCTION + static unsigned long long max() { + // FIXME (mfh 26 Oct 2014) It's legal for unsigned long long to be > 64 + // bits. + return Generator::MAX_URAND64; + } + KOKKOS_INLINE_FUNCTION + static unsigned long long draw(Generator& gen) { + // FIXME (mfh 26 Oct 2014) It's legal for unsigned long long to be > 64 + // bits. + return gen.urand64(); + } + KOKKOS_INLINE_FUNCTION + static unsigned long long draw(Generator& gen, + const unsigned long long& range) { + // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. + return gen.urand64(range); + } + KOKKOS_INLINE_FUNCTION + static unsigned long long draw(Generator& gen, + const unsigned long long& start, + const unsigned long long& end) { + // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. + return gen.urand64(start, end); + } +}; - template - struct rand { - KOKKOS_INLINE_FUNCTION - static double max(){return 1.0;} - KOKKOS_INLINE_FUNCTION - static double draw(Generator& gen) - {return gen.drand();} - KOKKOS_INLINE_FUNCTION - static double draw(Generator& gen, const double& range) - {return gen.drand(range);} - KOKKOS_INLINE_FUNCTION - static double draw(Generator& gen, const double& start, const double& end) - {return gen.drand(start,end);} +template +struct rand { + KOKKOS_INLINE_FUNCTION + static float max() { return 1.0f; } + KOKKOS_INLINE_FUNCTION + static float draw(Generator& gen) { return gen.frand(); } + KOKKOS_INLINE_FUNCTION + static float draw(Generator& gen, const float& range) { + return gen.frand(range); + } + KOKKOS_INLINE_FUNCTION + static float draw(Generator& gen, const float& start, const float& end) { + return gen.frand(start, end); + } +}; - }; +template +struct rand { + KOKKOS_INLINE_FUNCTION + static double max() { return 1.0; } + KOKKOS_INLINE_FUNCTION + static double draw(Generator& gen) { return gen.drand(); } + KOKKOS_INLINE_FUNCTION + static double draw(Generator& gen, const double& range) { + return gen.drand(range); + } + KOKKOS_INLINE_FUNCTION + static double draw(Generator& gen, const double& start, const double& end) { + return gen.drand(start, end); + } +}; - template - struct rand > { - KOKKOS_INLINE_FUNCTION - static Kokkos::complex max () { - return Kokkos::complex (1.0, 1.0); - } - KOKKOS_INLINE_FUNCTION - static Kokkos::complex draw (Generator& gen) { - const float re = gen.frand (); - const float im = gen.frand (); - return Kokkos::complex (re, im); - } - KOKKOS_INLINE_FUNCTION - static Kokkos::complex draw (Generator& gen, const Kokkos::complex& range) { - const float re = gen.frand (real (range)); - const float im = gen.frand (imag (range)); - return Kokkos::complex (re, im); - } - KOKKOS_INLINE_FUNCTION - static Kokkos::complex draw (Generator& gen, const Kokkos::complex& start, const Kokkos::complex& end) { - const float re = gen.frand (real (start), real (end)); - const float im = gen.frand (imag (start), imag (end)); - return Kokkos::complex (re, im); - } - }; +template +struct rand > { + KOKKOS_INLINE_FUNCTION + static Kokkos::complex max() { + return Kokkos::complex(1.0, 1.0); + } + KOKKOS_INLINE_FUNCTION + static Kokkos::complex draw(Generator& gen) { + const float re = gen.frand(); + const float im = gen.frand(); + return Kokkos::complex(re, im); + } + KOKKOS_INLINE_FUNCTION + static Kokkos::complex draw(Generator& gen, + const Kokkos::complex& range) { + const float re = gen.frand(real(range)); + const float im = gen.frand(imag(range)); + return Kokkos::complex(re, im); + } + KOKKOS_INLINE_FUNCTION + static Kokkos::complex draw(Generator& gen, + const Kokkos::complex& start, + const Kokkos::complex& end) { + const float re = gen.frand(real(start), real(end)); + const float im = gen.frand(imag(start), imag(end)); + return Kokkos::complex(re, im); + } +}; - template - struct rand > { - KOKKOS_INLINE_FUNCTION - static Kokkos::complex max () { - return Kokkos::complex (1.0, 1.0); - } - KOKKOS_INLINE_FUNCTION - static Kokkos::complex draw (Generator& gen) { - const double re = gen.drand (); - const double im = gen.drand (); - return Kokkos::complex (re, im); - } - KOKKOS_INLINE_FUNCTION - static Kokkos::complex draw (Generator& gen, const Kokkos::complex& range) { - const double re = gen.drand (real (range)); - const double im = gen.drand (imag (range)); - return Kokkos::complex (re, im); - } - KOKKOS_INLINE_FUNCTION - static Kokkos::complex draw (Generator& gen, const Kokkos::complex& start, const Kokkos::complex& end) { - const double re = gen.drand (real (start), real (end)); - const double im = gen.drand (imag (start), imag (end)); - return Kokkos::complex (re, im); - } - }; +template +struct rand > { + KOKKOS_INLINE_FUNCTION + static Kokkos::complex max() { + return Kokkos::complex(1.0, 1.0); + } + KOKKOS_INLINE_FUNCTION + static Kokkos::complex draw(Generator& gen) { + const double re = gen.drand(); + const double im = gen.drand(); + return Kokkos::complex(re, im); + } + KOKKOS_INLINE_FUNCTION + static Kokkos::complex draw(Generator& gen, + const Kokkos::complex& range) { + const double re = gen.drand(real(range)); + const double im = gen.drand(imag(range)); + return Kokkos::complex(re, im); + } + KOKKOS_INLINE_FUNCTION + static Kokkos::complex draw(Generator& gen, + const Kokkos::complex& start, + const Kokkos::complex& end) { + const double re = gen.drand(real(start), real(end)); + const double im = gen.drand(imag(start), imag(end)); + return Kokkos::complex(re, im); + } +}; - template - class Random_XorShift64_Pool; +template +class Random_XorShift64_Pool; - template - class Random_XorShift64 { - private: - uint64_t state_; - const int state_idx_; - friend class Random_XorShift64_Pool; - public: +template +class Random_XorShift64 { + private: + uint64_t state_; + const int state_idx_; + friend class Random_XorShift64_Pool; - typedef DeviceType device_type; + public: + typedef DeviceType device_type; - enum {MAX_URAND = 0xffffffffU}; - enum {MAX_URAND64 = 0xffffffffffffffffULL-1}; - enum {MAX_RAND = static_cast(0xffffffff/2)}; - enum {MAX_RAND64 = static_cast(0xffffffffffffffffLL/2-1)}; + enum { MAX_URAND = 0xffffffffU }; + enum { MAX_URAND64 = 0xffffffffffffffffULL - 1 }; + enum { MAX_RAND = static_cast(0xffffffff / 2) }; + enum { MAX_RAND64 = static_cast(0xffffffffffffffffLL / 2 - 1) }; - KOKKOS_INLINE_FUNCTION - Random_XorShift64 (uint64_t state, int state_idx = 0) - : state_(state==0?uint64_t(1318319):state),state_idx_(state_idx){} + KOKKOS_INLINE_FUNCTION + Random_XorShift64(uint64_t state, int state_idx = 0) + : state_(state == 0 ? uint64_t(1318319) : state), state_idx_(state_idx) {} - KOKKOS_INLINE_FUNCTION - uint32_t urand() { - state_ ^= state_ >> 12; - state_ ^= state_ << 25; - state_ ^= state_ >> 27; - - uint64_t tmp = state_ * 2685821657736338717ULL; - tmp = tmp>>16; - return static_cast(tmp&MAX_URAND); - } + KOKKOS_INLINE_FUNCTION + uint32_t urand() { + state_ ^= state_ >> 12; + state_ ^= state_ << 25; + state_ ^= state_ >> 27; + + uint64_t tmp = state_ * 2685821657736338717ULL; + tmp = tmp >> 16; + return static_cast(tmp & MAX_URAND); + } - KOKKOS_INLINE_FUNCTION - uint64_t urand64() { - state_ ^= state_ >> 12; - state_ ^= state_ << 25; - state_ ^= state_ >> 27; - return (state_ * 2685821657736338717ULL) - 1; - } + KOKKOS_INLINE_FUNCTION + uint64_t urand64() { + state_ ^= state_ >> 12; + state_ ^= state_ << 25; + state_ ^= state_ >> 27; + return (state_ * 2685821657736338717ULL) - 1; + } - KOKKOS_INLINE_FUNCTION - uint32_t urand(const uint32_t& range) { - const uint32_t max_val = (MAX_URAND/range)*range; - uint32_t tmp = urand(); - while(tmp>=max_val) - tmp = urand(); - return tmp%range; - } + KOKKOS_INLINE_FUNCTION + uint32_t urand(const uint32_t& range) { + const uint32_t max_val = (MAX_URAND / range) * range; + uint32_t tmp = urand(); + while (tmp >= max_val) tmp = urand(); + return tmp % range; + } - KOKKOS_INLINE_FUNCTION - uint32_t urand(const uint32_t& start, const uint32_t& end ) { - return urand(end-start)+start; - } + KOKKOS_INLINE_FUNCTION + uint32_t urand(const uint32_t& start, const uint32_t& end) { + return urand(end - start) + start; + } - KOKKOS_INLINE_FUNCTION - uint64_t urand64(const uint64_t& range) { - const uint64_t max_val = (MAX_URAND64/range)*range; - uint64_t tmp = urand64(); - while(tmp>=max_val) - tmp = urand64(); - return tmp%range; - } + KOKKOS_INLINE_FUNCTION + uint64_t urand64(const uint64_t& range) { + const uint64_t max_val = (MAX_URAND64 / range) * range; + uint64_t tmp = urand64(); + while (tmp >= max_val) tmp = urand64(); + return tmp % range; + } - KOKKOS_INLINE_FUNCTION - uint64_t urand64(const uint64_t& start, const uint64_t& end ) { - return urand64(end-start)+start; - } + KOKKOS_INLINE_FUNCTION + uint64_t urand64(const uint64_t& start, const uint64_t& end) { + return urand64(end - start) + start; + } - KOKKOS_INLINE_FUNCTION - int rand() { - return static_cast(urand()/2); - } + KOKKOS_INLINE_FUNCTION + int rand() { return static_cast(urand() / 2); } - KOKKOS_INLINE_FUNCTION - int rand(const int& range) { - const int max_val = (MAX_RAND/range)*range; - int tmp = rand(); - while(tmp>=max_val) - tmp = rand(); - return tmp%range; - } + KOKKOS_INLINE_FUNCTION + int rand(const int& range) { + const int max_val = (MAX_RAND / range) * range; + int tmp = rand(); + while (tmp >= max_val) tmp = rand(); + return tmp % range; + } - KOKKOS_INLINE_FUNCTION - int rand(const int& start, const int& end ) { - return rand(end-start)+start; - } + KOKKOS_INLINE_FUNCTION + int rand(const int& start, const int& end) { + return rand(end - start) + start; + } - KOKKOS_INLINE_FUNCTION - int64_t rand64() { - return static_cast(urand64()/2); - } + KOKKOS_INLINE_FUNCTION + int64_t rand64() { return static_cast(urand64() / 2); } - KOKKOS_INLINE_FUNCTION - int64_t rand64(const int64_t& range) { - const int64_t max_val = (MAX_RAND64/range)*range; - int64_t tmp = rand64(); - while(tmp>=max_val) - tmp = rand64(); - return tmp%range; - } + KOKKOS_INLINE_FUNCTION + int64_t rand64(const int64_t& range) { + const int64_t max_val = (MAX_RAND64 / range) * range; + int64_t tmp = rand64(); + while (tmp >= max_val) tmp = rand64(); + return tmp % range; + } - KOKKOS_INLINE_FUNCTION - int64_t rand64(const int64_t& start, const int64_t& end ) { - return rand64(end-start)+start; - } + KOKKOS_INLINE_FUNCTION + int64_t rand64(const int64_t& start, const int64_t& end) { + return rand64(end - start) + start; + } - KOKKOS_INLINE_FUNCTION - float frand() { - return 1.0f * urand64()/MAX_URAND64; - } + KOKKOS_INLINE_FUNCTION + float frand() { return 1.0f * urand64() / MAX_URAND64; } - KOKKOS_INLINE_FUNCTION - float frand(const float& range) { - return range * urand64()/MAX_URAND64; - } + KOKKOS_INLINE_FUNCTION + float frand(const float& range) { return range * urand64() / MAX_URAND64; } - KOKKOS_INLINE_FUNCTION - float frand(const float& start, const float& end ) { - return frand(end-start)+start; - } + KOKKOS_INLINE_FUNCTION + float frand(const float& start, const float& end) { + return frand(end - start) + start; + } - KOKKOS_INLINE_FUNCTION - double drand() { - return 1.0 * urand64()/MAX_URAND64; - } + KOKKOS_INLINE_FUNCTION + double drand() { return 1.0 * urand64() / MAX_URAND64; } - KOKKOS_INLINE_FUNCTION - double drand(const double& range) { - return range * urand64()/MAX_URAND64; - } + KOKKOS_INLINE_FUNCTION + double drand(const double& range) { return range * urand64() / MAX_URAND64; } - KOKKOS_INLINE_FUNCTION - double drand(const double& start, const double& end ) { - return drand(end-start)+start; - } + KOKKOS_INLINE_FUNCTION + double drand(const double& start, const double& end) { + return drand(end - start) + start; + } - //Marsaglia polar method for drawing a standard normal distributed random number - KOKKOS_INLINE_FUNCTION - double normal() { - double S = 2.0; - double U; - while(S>=1.0) { - U = 2.0*drand() - 1.0; - const double V = 2.0*drand() - 1.0; - S = U*U+V*V; - } - return U*std::sqrt(-2.0*log(S)/S); - } + // Marsaglia polar method for drawing a standard normal distributed random + // number + KOKKOS_INLINE_FUNCTION + double normal() { + double S = 2.0; + double U; + while (S >= 1.0) { + U = 2.0 * drand() - 1.0; + const double V = 2.0 * drand() - 1.0; + S = U * U + V * V; + } + return U * std::sqrt(-2.0 * log(S) / S); + } - KOKKOS_INLINE_FUNCTION - double normal(const double& mean, const double& std_dev=1.0) { - return mean + normal()*std_dev; - } + KOKKOS_INLINE_FUNCTION + double normal(const double& mean, const double& std_dev = 1.0) { + return mean + normal() * std_dev; + } +}; - }; +template +class Random_XorShift64_Pool { + private: + typedef View lock_type; + typedef View state_data_type; + lock_type locks_; + state_data_type state_; + int num_states_; - template - class Random_XorShift64_Pool { - private: - typedef View lock_type; - typedef View state_data_type; - lock_type locks_; - state_data_type state_; - int num_states_; - - public: - typedef Random_XorShift64 generator_type; - typedef DeviceType device_type; + public: + typedef Random_XorShift64 generator_type; + typedef DeviceType device_type; - KOKKOS_INLINE_FUNCTION - Random_XorShift64_Pool() { - num_states_ = 0; - } - Random_XorShift64_Pool(uint64_t seed) { - num_states_ = 0; + KOKKOS_INLINE_FUNCTION + Random_XorShift64_Pool() { num_states_ = 0; } + Random_XorShift64_Pool(uint64_t seed) { + num_states_ = 0; #ifdef KOKKOS_ENABLE_DEPRECATED_CODE - init(seed,DeviceType::max_hardware_threads()); + init(seed, DeviceType::max_hardware_threads()); #else - init(seed,DeviceType::impl_max_hardware_threads()); + init(seed, DeviceType::impl_max_hardware_threads()); #endif - } + } - KOKKOS_INLINE_FUNCTION - Random_XorShift64_Pool(const Random_XorShift64_Pool& src): - locks_(src.locks_), - state_(src.state_), - num_states_(src.num_states_) - {} + KOKKOS_INLINE_FUNCTION + Random_XorShift64_Pool(const Random_XorShift64_Pool& src) + : locks_(src.locks_), state_(src.state_), num_states_(src.num_states_) {} - KOKKOS_INLINE_FUNCTION - Random_XorShift64_Pool operator = (const Random_XorShift64_Pool& src) { - locks_ = src.locks_; - state_ = src.state_; - num_states_ = src.num_states_; - return *this; - } + KOKKOS_INLINE_FUNCTION + Random_XorShift64_Pool operator=(const Random_XorShift64_Pool& src) { + locks_ = src.locks_; + state_ = src.state_; + num_states_ = src.num_states_; + return *this; + } - void init(uint64_t seed, int num_states) { - if(seed==0) - seed = uint64_t(1318319); - - num_states_ = num_states; - - locks_ = lock_type("Kokkos::Random_XorShift64::locks",num_states_); - state_ = state_data_type("Kokkos::Random_XorShift64::state",num_states_); - - typename state_data_type::HostMirror h_state = create_mirror_view(state_); - typename lock_type::HostMirror h_lock = create_mirror_view(locks_); - - // Execute on the HostMirror's default execution space. - Random_XorShift64 gen(seed,0); - for(int i = 0; i < 17; i++) - gen.rand(); - for(int i = 0; i < num_states_; i++) { - int n1 = gen.rand(); - int n2 = gen.rand(); - int n3 = gen.rand(); - int n4 = gen.rand(); - h_state(i) = (((static_cast(n1)) & 0xffff)<<00) | - (((static_cast(n2)) & 0xffff)<<16) | - (((static_cast(n3)) & 0xffff)<<32) | - (((static_cast(n4)) & 0xffff)<<48); - h_lock(i) = 0; - } - deep_copy(state_,h_state); - deep_copy(locks_,h_lock); - } + void init(uint64_t seed, int num_states) { + if (seed == 0) seed = uint64_t(1318319); + + num_states_ = num_states; + + locks_ = lock_type("Kokkos::Random_XorShift64::locks", num_states_); + state_ = state_data_type("Kokkos::Random_XorShift64::state", num_states_); + + typename state_data_type::HostMirror h_state = create_mirror_view(state_); + typename lock_type::HostMirror h_lock = create_mirror_view(locks_); + + // Execute on the HostMirror's default execution space. + Random_XorShift64 + gen(seed, 0); + for (int i = 0; i < 17; i++) gen.rand(); + for (int i = 0; i < num_states_; i++) { + int n1 = gen.rand(); + int n2 = gen.rand(); + int n3 = gen.rand(); + int n4 = gen.rand(); + h_state(i) = (((static_cast(n1)) & 0xffff) << 00) | + (((static_cast(n2)) & 0xffff) << 16) | + (((static_cast(n3)) & 0xffff) << 32) | + (((static_cast(n4)) & 0xffff) << 48); + h_lock(i) = 0; + } + deep_copy(state_, h_state); + deep_copy(locks_, h_lock); + } - KOKKOS_INLINE_FUNCTION - Random_XorShift64 get_state() const { + KOKKOS_INLINE_FUNCTION + Random_XorShift64 get_state() const { #ifdef KOKKOS_ENABLE_DEPRECATED_CODE - const int i = DeviceType::hardware_thread_id();; + const int i = DeviceType::hardware_thread_id(); + ; #else - const int i = DeviceType::impl_hardware_thread_id();; + const int i = DeviceType::impl_hardware_thread_id(); + ; #endif - return Random_XorShift64(state_(i),i); - } - - // NOTE: state_idx MUST be unique and less than num_states - KOKKOS_INLINE_FUNCTION - Random_XorShift64 get_state(const int state_idx) const { - return Random_XorShift64(state_(state_idx),state_idx); - } - - KOKKOS_INLINE_FUNCTION - void free_state(const Random_XorShift64& state) const { - state_(state.state_idx_) = state.state_; - } - }; + return Random_XorShift64(state_(i), i); + } + // NOTE: state_idx MUST be unique and less than num_states + KOKKOS_INLINE_FUNCTION + Random_XorShift64 get_state(const int state_idx) const { + return Random_XorShift64(state_(state_idx), state_idx); + } - template - class Random_XorShift1024_Pool; + KOKKOS_INLINE_FUNCTION + void free_state(const Random_XorShift64& state) const { + state_(state.state_idx_) = state.state_; + } +}; - template - class Random_XorShift1024 { - private: - int p_; - const int state_idx_; - uint64_t state_[16]; - friend class Random_XorShift1024_Pool; - public: +template +class Random_XorShift1024_Pool; - typedef Random_XorShift1024_Pool pool_type; - typedef DeviceType device_type; +template +class Random_XorShift1024 { + private: + int p_; + const int state_idx_; + uint64_t state_[16]; + friend class Random_XorShift1024_Pool; - enum {MAX_URAND = 0xffffffffU}; - enum {MAX_URAND64 = 0xffffffffffffffffULL-1}; - enum {MAX_RAND = static_cast(0xffffffffU/2)}; - enum {MAX_RAND64 = static_cast(0xffffffffffffffffULL/2-1)}; + public: + typedef Random_XorShift1024_Pool pool_type; + typedef DeviceType device_type; - KOKKOS_INLINE_FUNCTION - Random_XorShift1024 (const typename pool_type::state_data_type& state, int p, int state_idx = 0): - p_(p),state_idx_(state_idx){ - for(int i=0 ; i<16; i++) - state_[i] = state(state_idx,i); - } + enum { MAX_URAND = 0xffffffffU }; + enum { MAX_URAND64 = 0xffffffffffffffffULL - 1 }; + enum { MAX_RAND = static_cast(0xffffffffU / 2) }; + enum { MAX_RAND64 = static_cast(0xffffffffffffffffULL / 2 - 1) }; - KOKKOS_INLINE_FUNCTION - uint32_t urand() { - uint64_t state_0 = state_[ p_ ]; - uint64_t state_1 = state_[ p_ = ( p_ + 1 ) & 15 ]; - state_1 ^= state_1 << 31; - state_1 ^= state_1 >> 11; - state_0 ^= state_0 >> 30; - uint64_t tmp = ( state_[ p_ ] = state_0 ^ state_1 ) * 1181783497276652981ULL; - tmp = tmp>>16; - return static_cast(tmp&MAX_URAND); - } + KOKKOS_INLINE_FUNCTION + Random_XorShift1024(const typename pool_type::state_data_type& state, int p, + int state_idx = 0) + : p_(p), state_idx_(state_idx) { + for (int i = 0; i < 16; i++) state_[i] = state(state_idx, i); + } - KOKKOS_INLINE_FUNCTION - uint64_t urand64() { - uint64_t state_0 = state_[ p_ ]; - uint64_t state_1 = state_[ p_ = ( p_ + 1 ) & 15 ]; - state_1 ^= state_1 << 31; - state_1 ^= state_1 >> 11; - state_0 ^= state_0 >> 30; - return (( state_[ p_ ] = state_0 ^ state_1 ) * 1181783497276652981LL) - 1; - } + KOKKOS_INLINE_FUNCTION + uint32_t urand() { + uint64_t state_0 = state_[p_]; + uint64_t state_1 = state_[p_ = (p_ + 1) & 15]; + state_1 ^= state_1 << 31; + state_1 ^= state_1 >> 11; + state_0 ^= state_0 >> 30; + uint64_t tmp = (state_[p_] = state_0 ^ state_1) * 1181783497276652981ULL; + tmp = tmp >> 16; + return static_cast(tmp & MAX_URAND); + } - KOKKOS_INLINE_FUNCTION - uint32_t urand(const uint32_t& range) { - const uint32_t max_val = (MAX_URAND/range)*range; - uint32_t tmp = urand(); - while(tmp>=max_val) - tmp = urand(); - return tmp%range; - } + KOKKOS_INLINE_FUNCTION + uint64_t urand64() { + uint64_t state_0 = state_[p_]; + uint64_t state_1 = state_[p_ = (p_ + 1) & 15]; + state_1 ^= state_1 << 31; + state_1 ^= state_1 >> 11; + state_0 ^= state_0 >> 30; + return ((state_[p_] = state_0 ^ state_1) * 1181783497276652981LL) - 1; + } - KOKKOS_INLINE_FUNCTION - uint32_t urand(const uint32_t& start, const uint32_t& end ) { - return urand(end-start)+start; - } + KOKKOS_INLINE_FUNCTION + uint32_t urand(const uint32_t& range) { + const uint32_t max_val = (MAX_URAND / range) * range; + uint32_t tmp = urand(); + while (tmp >= max_val) tmp = urand(); + return tmp % range; + } - KOKKOS_INLINE_FUNCTION - uint64_t urand64(const uint64_t& range) { - const uint64_t max_val = (MAX_URAND64/range)*range; - uint64_t tmp = urand64(); - while(tmp>=max_val) - tmp = urand64(); - return tmp%range; - } + KOKKOS_INLINE_FUNCTION + uint32_t urand(const uint32_t& start, const uint32_t& end) { + return urand(end - start) + start; + } - KOKKOS_INLINE_FUNCTION - uint64_t urand64(const uint64_t& start, const uint64_t& end ) { - return urand64(end-start)+start; - } + KOKKOS_INLINE_FUNCTION + uint64_t urand64(const uint64_t& range) { + const uint64_t max_val = (MAX_URAND64 / range) * range; + uint64_t tmp = urand64(); + while (tmp >= max_val) tmp = urand64(); + return tmp % range; + } - KOKKOS_INLINE_FUNCTION - int rand() { - return static_cast(urand()/2); - } + KOKKOS_INLINE_FUNCTION + uint64_t urand64(const uint64_t& start, const uint64_t& end) { + return urand64(end - start) + start; + } - KOKKOS_INLINE_FUNCTION - int rand(const int& range) { - const int max_val = (MAX_RAND/range)*range; - int tmp = rand(); - while(tmp>=max_val) - tmp = rand(); - return tmp%range; - } + KOKKOS_INLINE_FUNCTION + int rand() { return static_cast(urand() / 2); } - KOKKOS_INLINE_FUNCTION - int rand(const int& start, const int& end ) { - return rand(end-start)+start; - } + KOKKOS_INLINE_FUNCTION + int rand(const int& range) { + const int max_val = (MAX_RAND / range) * range; + int tmp = rand(); + while (tmp >= max_val) tmp = rand(); + return tmp % range; + } - KOKKOS_INLINE_FUNCTION - int64_t rand64() { - return static_cast(urand64()/2); - } + KOKKOS_INLINE_FUNCTION + int rand(const int& start, const int& end) { + return rand(end - start) + start; + } - KOKKOS_INLINE_FUNCTION - int64_t rand64(const int64_t& range) { - const int64_t max_val = (MAX_RAND64/range)*range; - int64_t tmp = rand64(); - while(tmp>=max_val) - tmp = rand64(); - return tmp%range; - } + KOKKOS_INLINE_FUNCTION + int64_t rand64() { return static_cast(urand64() / 2); } - KOKKOS_INLINE_FUNCTION - int64_t rand64(const int64_t& start, const int64_t& end ) { - return rand64(end-start)+start; - } + KOKKOS_INLINE_FUNCTION + int64_t rand64(const int64_t& range) { + const int64_t max_val = (MAX_RAND64 / range) * range; + int64_t tmp = rand64(); + while (tmp >= max_val) tmp = rand64(); + return tmp % range; + } - KOKKOS_INLINE_FUNCTION - float frand() { - return 1.0f * urand64()/MAX_URAND64; - } + KOKKOS_INLINE_FUNCTION + int64_t rand64(const int64_t& start, const int64_t& end) { + return rand64(end - start) + start; + } - KOKKOS_INLINE_FUNCTION - float frand(const float& range) { - return range * urand64()/MAX_URAND64; - } + KOKKOS_INLINE_FUNCTION + float frand() { return 1.0f * urand64() / MAX_URAND64; } - KOKKOS_INLINE_FUNCTION - float frand(const float& start, const float& end ) { - return frand(end-start)+start; - } + KOKKOS_INLINE_FUNCTION + float frand(const float& range) { return range * urand64() / MAX_URAND64; } - KOKKOS_INLINE_FUNCTION - double drand() { - return 1.0 * urand64()/MAX_URAND64; - } + KOKKOS_INLINE_FUNCTION + float frand(const float& start, const float& end) { + return frand(end - start) + start; + } - KOKKOS_INLINE_FUNCTION - double drand(const double& range) { - return range * urand64()/MAX_URAND64; - } + KOKKOS_INLINE_FUNCTION + double drand() { return 1.0 * urand64() / MAX_URAND64; } - KOKKOS_INLINE_FUNCTION - double drand(const double& start, const double& end ) { - return frand(end-start)+start; - } + KOKKOS_INLINE_FUNCTION + double drand(const double& range) { return range * urand64() / MAX_URAND64; } - //Marsaglia polar method for drawing a standard normal distributed random number - KOKKOS_INLINE_FUNCTION - double normal() { - double S = 2.0; - double U; - while(S>=1.0) { - U = 2.0*drand() - 1.0; - const double V = 2.0*drand() - 1.0; - S = U*U+V*V; - } - return U*std::sqrt(-2.0*log(S)/S); - } + KOKKOS_INLINE_FUNCTION + double drand(const double& start, const double& end) { + return frand(end - start) + start; + } - KOKKOS_INLINE_FUNCTION - double normal(const double& mean, const double& std_dev=1.0) { - return mean + normal()*std_dev; - } - }; + // Marsaglia polar method for drawing a standard normal distributed random + // number + KOKKOS_INLINE_FUNCTION + double normal() { + double S = 2.0; + double U; + while (S >= 1.0) { + U = 2.0 * drand() - 1.0; + const double V = 2.0 * drand() - 1.0; + S = U * U + V * V; + } + return U * std::sqrt(-2.0 * log(S) / S); + } + KOKKOS_INLINE_FUNCTION + double normal(const double& mean, const double& std_dev = 1.0) { + return mean + normal() * std_dev; + } +}; - template - class Random_XorShift1024_Pool { - private: - typedef View int_view_type; - typedef View state_data_type; +template +class Random_XorShift1024_Pool { + private: + typedef View int_view_type; + typedef View state_data_type; - int_view_type locks_; - state_data_type state_; - int_view_type p_; - int num_states_; - friend class Random_XorShift1024; + int_view_type locks_; + state_data_type state_; + int_view_type p_; + int num_states_; + friend class Random_XorShift1024; - public: - typedef Random_XorShift1024 generator_type; + public: + typedef Random_XorShift1024 generator_type; - typedef DeviceType device_type; + typedef DeviceType device_type; - KOKKOS_INLINE_FUNCTION - Random_XorShift1024_Pool() { - num_states_ = 0; - } + KOKKOS_INLINE_FUNCTION + Random_XorShift1024_Pool() { num_states_ = 0; } - inline - Random_XorShift1024_Pool(uint64_t seed){ - num_states_ = 0; + inline Random_XorShift1024_Pool(uint64_t seed) { + num_states_ = 0; #ifdef KOKKOS_ENABLE_DEPRECATED_CODE - init(seed,DeviceType::max_hardware_threads()); + init(seed, DeviceType::max_hardware_threads()); #else - init(seed,DeviceType::impl_max_hardware_threads()); + init(seed, DeviceType::impl_max_hardware_threads()); #endif - } + } - KOKKOS_INLINE_FUNCTION - Random_XorShift1024_Pool(const Random_XorShift1024_Pool& src): - locks_(src.locks_), - state_(src.state_), - p_(src.p_), - num_states_(src.num_states_) - {} + KOKKOS_INLINE_FUNCTION + Random_XorShift1024_Pool(const Random_XorShift1024_Pool& src) + : locks_(src.locks_), + state_(src.state_), + p_(src.p_), + num_states_(src.num_states_) {} - KOKKOS_INLINE_FUNCTION - Random_XorShift1024_Pool operator = (const Random_XorShift1024_Pool& src) { - locks_ = src.locks_; - state_ = src.state_; - p_ = src.p_; - num_states_ = src.num_states_; - return *this; - } + KOKKOS_INLINE_FUNCTION + Random_XorShift1024_Pool operator=(const Random_XorShift1024_Pool& src) { + locks_ = src.locks_; + state_ = src.state_; + p_ = src.p_; + num_states_ = src.num_states_; + return *this; + } - inline - void init(uint64_t seed, int num_states) { - if(seed==0) - seed = uint64_t(1318319); - num_states_ = num_states; - locks_ = int_view_type("Kokkos::Random_XorShift1024::locks",num_states_); - state_ = state_data_type("Kokkos::Random_XorShift1024::state",num_states_); - p_ = int_view_type("Kokkos::Random_XorShift1024::p",num_states_); - - typename state_data_type::HostMirror h_state = create_mirror_view(state_); - typename int_view_type::HostMirror h_lock = create_mirror_view(locks_); - typename int_view_type::HostMirror h_p = create_mirror_view(p_); - - // Execute on the HostMirror's default execution space. - Random_XorShift64 gen(seed,0); - for(int i = 0; i < 17; i++) - gen.rand(); - for(int i = 0; i < num_states_; i++) { - for(int j = 0; j < 16 ; j++) { - int n1 = gen.rand(); - int n2 = gen.rand(); - int n3 = gen.rand(); - int n4 = gen.rand(); - h_state(i,j) = (((static_cast(n1)) & 0xffff)<<00) | - (((static_cast(n2)) & 0xffff)<<16) | - (((static_cast(n3)) & 0xffff)<<32) | - (((static_cast(n4)) & 0xffff)<<48); - } - h_p(i) = 0; - h_lock(i) = 0; + inline void init(uint64_t seed, int num_states) { + if (seed == 0) seed = uint64_t(1318319); + num_states_ = num_states; + locks_ = int_view_type("Kokkos::Random_XorShift1024::locks", num_states_); + state_ = state_data_type("Kokkos::Random_XorShift1024::state", num_states_); + p_ = int_view_type("Kokkos::Random_XorShift1024::p", num_states_); + + typename state_data_type::HostMirror h_state = create_mirror_view(state_); + typename int_view_type::HostMirror h_lock = create_mirror_view(locks_); + typename int_view_type::HostMirror h_p = create_mirror_view(p_); + + // Execute on the HostMirror's default execution space. + Random_XorShift64 + gen(seed, 0); + for (int i = 0; i < 17; i++) gen.rand(); + for (int i = 0; i < num_states_; i++) { + for (int j = 0; j < 16; j++) { + int n1 = gen.rand(); + int n2 = gen.rand(); + int n3 = gen.rand(); + int n4 = gen.rand(); + h_state(i, j) = (((static_cast(n1)) & 0xffff) << 00) | + (((static_cast(n2)) & 0xffff) << 16) | + (((static_cast(n3)) & 0xffff) << 32) | + (((static_cast(n4)) & 0xffff) << 48); } - deep_copy(state_,h_state); - deep_copy(locks_,h_lock); + h_p(i) = 0; + h_lock(i) = 0; } + deep_copy(state_, h_state); + deep_copy(locks_, h_lock); + } - KOKKOS_INLINE_FUNCTION - Random_XorShift1024 get_state() const { + KOKKOS_INLINE_FUNCTION + Random_XorShift1024 get_state() const { #ifdef KOKKOS_ENABLE_DEPRECATED_CODE - const int i = DeviceType::hardware_thread_id(); + const int i = DeviceType::hardware_thread_id(); #else - const int i = DeviceType::impl_hardware_thread_id(); + const int i = DeviceType::impl_hardware_thread_id(); #endif - return Random_XorShift1024(state_,p_(i),i); - }; + return Random_XorShift1024(state_, p_(i), i); + }; - // NOTE: state_idx MUST be unique and less than num_states - KOKKOS_INLINE_FUNCTION - Random_XorShift1024 get_state(const int state_idx) const { - return Random_XorShift1024(state_,p_(state_idx),state_idx); - } + // NOTE: state_idx MUST be unique and less than num_states + KOKKOS_INLINE_FUNCTION + Random_XorShift1024 get_state(const int state_idx) const { + return Random_XorShift1024(state_, p_(state_idx), state_idx); + } - KOKKOS_INLINE_FUNCTION - void free_state(const Random_XorShift1024& state) const { - for(int i = 0; i<16; i++) - state_(state.state_idx_,i) = state.state_[i]; - p_(state.state_idx_) = state.p_; - } - }; + KOKKOS_INLINE_FUNCTION + void free_state(const Random_XorShift1024& state) const { + for (int i = 0; i < 16; i++) state_(state.state_idx_, i) = state.state_[i]; + p_(state.state_idx_) = state.p_; + } +}; #if defined(KOKKOS_ENABLE_CUDA) && defined(__CUDACC__) - template<> - class Random_XorShift1024 { - private: - int p_; - const int state_idx_; - uint64_t* state_; - const int stride_; - friend class Random_XorShift1024_Pool; - public: +template <> +class Random_XorShift1024 { + private: + int p_; + const int state_idx_; + uint64_t* state_; + const int stride_; + friend class Random_XorShift1024_Pool; - typedef Kokkos::Cuda device_type; - typedef Random_XorShift1024_Pool pool_type; + public: + typedef Kokkos::Cuda device_type; + typedef Random_XorShift1024_Pool pool_type; - enum {MAX_URAND = 0xffffffffU}; - enum {MAX_URAND64 = 0xffffffffffffffffULL-1}; - enum {MAX_RAND = static_cast(0xffffffffU/2)}; - enum {MAX_RAND64 = static_cast(0xffffffffffffffffULL/2-1)}; + enum { MAX_URAND = 0xffffffffU }; + enum { MAX_URAND64 = 0xffffffffffffffffULL - 1 }; + enum { MAX_RAND = static_cast(0xffffffffU / 2) }; + enum { MAX_RAND64 = static_cast(0xffffffffffffffffULL / 2 - 1) }; - KOKKOS_INLINE_FUNCTION - Random_XorShift1024 (const typename pool_type::state_data_type& state, int p, int state_idx = 0): - p_(p),state_idx_(state_idx),state_(&state(state_idx,0)),stride_(state.stride_1()){ - } + KOKKOS_INLINE_FUNCTION + Random_XorShift1024(const typename pool_type::state_data_type& state, int p, + int state_idx = 0) + : p_(p), + state_idx_(state_idx), + state_(&state(state_idx, 0)), + stride_(state.stride_1()) {} - KOKKOS_INLINE_FUNCTION - uint32_t urand() { - uint64_t state_0 = state_[ p_ * stride_ ]; - uint64_t state_1 = state_[ (p_ = ( p_ + 1 ) & 15) * stride_ ]; - state_1 ^= state_1 << 31; - state_1 ^= state_1 >> 11; - state_0 ^= state_0 >> 30; - uint64_t tmp = ( state_[ p_ * stride_ ] = state_0 ^ state_1 ) * 1181783497276652981ULL; - tmp = tmp>>16; - return static_cast(tmp&MAX_URAND); - } + KOKKOS_INLINE_FUNCTION + uint32_t urand() { + uint64_t state_0 = state_[p_ * stride_]; + uint64_t state_1 = state_[(p_ = (p_ + 1) & 15) * stride_]; + state_1 ^= state_1 << 31; + state_1 ^= state_1 >> 11; + state_0 ^= state_0 >> 30; + uint64_t tmp = + (state_[p_ * stride_] = state_0 ^ state_1) * 1181783497276652981ULL; + tmp = tmp >> 16; + return static_cast(tmp & MAX_URAND); + } - KOKKOS_INLINE_FUNCTION - uint64_t urand64() { - uint64_t state_0 = state_[ p_ * stride_ ]; - uint64_t state_1 = state_[ (p_ = ( p_ + 1 ) & 15) * stride_ ]; - state_1 ^= state_1 << 31; - state_1 ^= state_1 >> 11; - state_0 ^= state_0 >> 30; - return (( state_[ p_ * stride_ ] = state_0 ^ state_1 ) * 1181783497276652981LL) - 1; - } + KOKKOS_INLINE_FUNCTION + uint64_t urand64() { + uint64_t state_0 = state_[p_ * stride_]; + uint64_t state_1 = state_[(p_ = (p_ + 1) & 15) * stride_]; + state_1 ^= state_1 << 31; + state_1 ^= state_1 >> 11; + state_0 ^= state_0 >> 30; + return ((state_[p_ * stride_] = state_0 ^ state_1) * + 1181783497276652981LL) - + 1; + } - KOKKOS_INLINE_FUNCTION - uint32_t urand(const uint32_t& range) { - const uint32_t max_val = (MAX_URAND/range)*range; - uint32_t tmp = urand(); - while(tmp>=max_val) - urand(); - return tmp%range; - } + KOKKOS_INLINE_FUNCTION + uint32_t urand(const uint32_t& range) { + const uint32_t max_val = (MAX_URAND / range) * range; + uint32_t tmp = urand(); + while (tmp >= max_val) urand(); + return tmp % range; + } - KOKKOS_INLINE_FUNCTION - uint32_t urand(const uint32_t& start, const uint32_t& end ) { - return urand(end-start)+start; - } + KOKKOS_INLINE_FUNCTION + uint32_t urand(const uint32_t& start, const uint32_t& end) { + return urand(end - start) + start; + } - KOKKOS_INLINE_FUNCTION - uint64_t urand64(const uint64_t& range) { - const uint64_t max_val = (MAX_URAND64/range)*range; - uint64_t tmp = urand64(); - while(tmp>=max_val) - urand64(); - return tmp%range; - } + KOKKOS_INLINE_FUNCTION + uint64_t urand64(const uint64_t& range) { + const uint64_t max_val = (MAX_URAND64 / range) * range; + uint64_t tmp = urand64(); + while (tmp >= max_val) urand64(); + return tmp % range; + } - KOKKOS_INLINE_FUNCTION - uint64_t urand64(const uint64_t& start, const uint64_t& end ) { - return urand64(end-start)+start; - } + KOKKOS_INLINE_FUNCTION + uint64_t urand64(const uint64_t& start, const uint64_t& end) { + return urand64(end - start) + start; + } - KOKKOS_INLINE_FUNCTION - int rand() { - return static_cast(urand()/2); - } + KOKKOS_INLINE_FUNCTION + int rand() { return static_cast(urand() / 2); } - KOKKOS_INLINE_FUNCTION - int rand(const int& range) { - const int max_val = (MAX_RAND/range)*range; - int tmp = rand(); - while(tmp>=max_val) - rand(); - return tmp%range; - } + KOKKOS_INLINE_FUNCTION + int rand(const int& range) { + const int max_val = (MAX_RAND / range) * range; + int tmp = rand(); + while (tmp >= max_val) rand(); + return tmp % range; + } - KOKKOS_INLINE_FUNCTION - int rand(const int& start, const int& end ) { - return rand(end-start)+start; - } + KOKKOS_INLINE_FUNCTION + int rand(const int& start, const int& end) { + return rand(end - start) + start; + } - KOKKOS_INLINE_FUNCTION - int64_t rand64() { - return static_cast(urand64()/2); - } + KOKKOS_INLINE_FUNCTION + int64_t rand64() { return static_cast(urand64() / 2); } - KOKKOS_INLINE_FUNCTION - int64_t rand64(const int64_t& range) { - const int64_t max_val = (MAX_RAND64/range)*range; - int64_t tmp = rand64(); - while(tmp>=max_val) - rand64(); - return tmp%range; - } + KOKKOS_INLINE_FUNCTION + int64_t rand64(const int64_t& range) { + const int64_t max_val = (MAX_RAND64 / range) * range; + int64_t tmp = rand64(); + while (tmp >= max_val) rand64(); + return tmp % range; + } - KOKKOS_INLINE_FUNCTION - int64_t rand64(const int64_t& start, const int64_t& end ) { - return rand64(end-start)+start; - } + KOKKOS_INLINE_FUNCTION + int64_t rand64(const int64_t& start, const int64_t& end) { + return rand64(end - start) + start; + } - KOKKOS_INLINE_FUNCTION - float frand() { - return 1.0f * urand64()/MAX_URAND64; - } + KOKKOS_INLINE_FUNCTION + float frand() { return 1.0f * urand64() / MAX_URAND64; } - KOKKOS_INLINE_FUNCTION - float frand(const float& range) { - return range * urand64()/MAX_URAND64; - } + KOKKOS_INLINE_FUNCTION + float frand(const float& range) { return range * urand64() / MAX_URAND64; } - KOKKOS_INLINE_FUNCTION - float frand(const float& start, const float& end ) { - return frand(end-start)+start; - } + KOKKOS_INLINE_FUNCTION + float frand(const float& start, const float& end) { + return frand(end - start) + start; + } - KOKKOS_INLINE_FUNCTION - double drand() { - return 1.0 * urand64()/MAX_URAND64; - } + KOKKOS_INLINE_FUNCTION + double drand() { return 1.0 * urand64() / MAX_URAND64; } - KOKKOS_INLINE_FUNCTION - double drand(const double& range) { - return range * urand64()/MAX_URAND64; - } + KOKKOS_INLINE_FUNCTION + double drand(const double& range) { return range * urand64() / MAX_URAND64; } - KOKKOS_INLINE_FUNCTION - double drand(const double& start, const double& end ) { - return frand(end-start)+start; - } + KOKKOS_INLINE_FUNCTION + double drand(const double& start, const double& end) { + return frand(end - start) + start; + } - //Marsaglia polar method for drawing a standard normal distributed random number - KOKKOS_INLINE_FUNCTION - double normal() { - double S = 2.0; - double U; - while(S>=1.0) { - U = 2.0*drand() - 1.0; - const double V = 2.0*drand() - 1.0; - S = U*U+V*V; - } - return U*std::sqrt(-2.0*log(S)/S); - } + // Marsaglia polar method for drawing a standard normal distributed random + // number + KOKKOS_INLINE_FUNCTION + double normal() { + double S = 2.0; + double U; + while (S >= 1.0) { + U = 2.0 * drand() - 1.0; + const double V = 2.0 * drand() - 1.0; + S = U * U + V * V; + } + return U * std::sqrt(-2.0 * log(S) / S); + } - KOKKOS_INLINE_FUNCTION - double normal(const double& mean, const double& std_dev=1.0) { - return mean + normal()*std_dev; - } - }; + KOKKOS_INLINE_FUNCTION + double normal(const double& mean, const double& std_dev = 1.0) { + return mean + normal() * std_dev; + } +}; -template<> -inline -Random_XorShift64_Pool::Random_XorShift64_Pool(uint64_t seed) { +template <> +inline Random_XorShift64_Pool::Random_XorShift64_Pool( + uint64_t seed) { num_states_ = 0; - init(seed,4*32768); + init(seed, 4 * 32768); } -template<> -KOKKOS_INLINE_FUNCTION -Random_XorShift64 Random_XorShift64_Pool::get_state() const { +template <> +KOKKOS_INLINE_FUNCTION Random_XorShift64 +Random_XorShift64_Pool::get_state() const { #ifdef __CUDA_ARCH__ - const int i_offset = (threadIdx.x*blockDim.y + threadIdx.y)*blockDim.z+threadIdx.z; - int i = (((blockIdx.x*gridDim.y+blockIdx.y)*gridDim.z + blockIdx.z) * - blockDim.x*blockDim.y*blockDim.z + i_offset)%num_states_; - while(Kokkos::atomic_compare_exchange(&locks_(i),0,1)) { - i+=blockDim.x*blockDim.y*blockDim.z; - if(i>=num_states_) {i = i_offset;} + const int i_offset = + (threadIdx.x * blockDim.y + threadIdx.y) * blockDim.z + threadIdx.z; + int i = (((blockIdx.x * gridDim.y + blockIdx.y) * gridDim.z + blockIdx.z) * + blockDim.x * blockDim.y * blockDim.z + + i_offset) % + num_states_; + while (Kokkos::atomic_compare_exchange(&locks_(i), 0, 1)) { + i += blockDim.x * blockDim.y * blockDim.z; + if (i >= num_states_) { + i = i_offset; + } } - return Random_XorShift64(state_(i),i); + return Random_XorShift64(state_(i), i); #else - return Random_XorShift64(state_(0),0); + return Random_XorShift64(state_(0), 0); #endif } -template<> -KOKKOS_INLINE_FUNCTION -void Random_XorShift64_Pool::free_state(const Random_XorShift64 &state) const { +template <> +KOKKOS_INLINE_FUNCTION void Random_XorShift64_Pool::free_state( + const Random_XorShift64& state) const { state_(state.state_idx_) = state.state_; #ifdef __CUDA_ARCH__ locks_(state.state_idx_) = 0; @@ -1249,24 +1211,28 @@ void Random_XorShift64_Pool::free_state(const Random_XorShift64 -inline -Random_XorShift1024_Pool::Random_XorShift1024_Pool(uint64_t seed) { +template <> +inline Random_XorShift1024_Pool::Random_XorShift1024_Pool( + uint64_t seed) { num_states_ = 0; - init(seed,4*32768); + init(seed, 4 * 32768); } -template<> -KOKKOS_INLINE_FUNCTION -Random_XorShift1024 Random_XorShift1024_Pool::get_state() const { +template <> +KOKKOS_INLINE_FUNCTION Random_XorShift1024 +Random_XorShift1024_Pool::get_state() const { #ifdef __CUDA_ARCH__ - const int i_offset = (threadIdx.x*blockDim.y + threadIdx.y)*blockDim.z+threadIdx.z; - int i = (((blockIdx.x*gridDim.y+blockIdx.y)*gridDim.z + blockIdx.z) * - blockDim.x*blockDim.y*blockDim.z + i_offset)%num_states_; - while(Kokkos::atomic_compare_exchange(&locks_(i),0,1)) { - i+=blockDim.x*blockDim.y*blockDim.z; - if(i>=num_states_) {i = i_offset;} + const int i_offset = + (threadIdx.x * blockDim.y + threadIdx.y) * blockDim.z + threadIdx.z; + int i = (((blockIdx.x * gridDim.y + blockIdx.y) * gridDim.z + blockIdx.z) * + blockDim.x * blockDim.y * blockDim.z + + i_offset) % + num_states_; + while (Kokkos::atomic_compare_exchange(&locks_(i), 0, 1)) { + i += blockDim.x * blockDim.y * blockDim.z; + if (i >= num_states_) { + i = i_offset; + } } return Random_XorShift1024(state_, p_(i), i); @@ -1275,210 +1241,205 @@ Random_XorShift1024 Random_XorShift1024_Pool::get_st #endif } -template<> -KOKKOS_INLINE_FUNCTION -void Random_XorShift1024_Pool::free_state(const Random_XorShift1024 &state) const { - for(int i=0; i<16; i++) - state_(state.state_idx_,i) = state.state_[i]; +template <> +KOKKOS_INLINE_FUNCTION void Random_XorShift1024_Pool::free_state( + const Random_XorShift1024& state) const { + for (int i = 0; i < 16; i++) state_(state.state_idx_, i) = state.state_[i]; #ifdef __CUDA_ARCH__ locks_(state.state_idx_) = 0; return; #endif } - #endif -#if defined(KOKKOS_ENABLE_ROCM) +#if defined(KOKKOS_ENABLE_ROCM) - template<> - class Random_XorShift1024 { - private: - int p_; - const int state_idx_; - uint64_t* state_; - const int stride_; - friend class Random_XorShift1024_Pool; - public: +template <> +class Random_XorShift1024 { + private: + int p_; + const int state_idx_; + uint64_t* state_; + const int stride_; + friend class Random_XorShift1024_Pool; - typedef Kokkos::Experimental::ROCm device_type; - typedef Random_XorShift1024_Pool pool_type; + public: + typedef Kokkos::Experimental::ROCm device_type; + typedef Random_XorShift1024_Pool pool_type; - enum {MAX_URAND = 0xffffffffU}; - enum {MAX_URAND64 = 0xffffffffffffffffULL-1}; - enum {MAX_RAND = static_cast(0xffffffffU/2)}; - enum {MAX_RAND64 = static_cast(0xffffffffffffffffULL/2-1)}; + enum { MAX_URAND = 0xffffffffU }; + enum { MAX_URAND64 = 0xffffffffffffffffULL - 1 }; + enum { MAX_RAND = static_cast(0xffffffffU / 2) }; + enum { MAX_RAND64 = static_cast(0xffffffffffffffffULL / 2 - 1) }; - KOKKOS_INLINE_FUNCTION - Random_XorShift1024 (const typename pool_type::state_data_type& state, int p, int state_idx = 0): - p_(p),state_idx_(state_idx),state_(&state(state_idx,0)),stride_(state.stride_1()){ - } + KOKKOS_INLINE_FUNCTION + Random_XorShift1024(const typename pool_type::state_data_type& state, int p, + int state_idx = 0) + : p_(p), + state_idx_(state_idx), + state_(&state(state_idx, 0)), + stride_(state.stride_1()) {} - KOKKOS_INLINE_FUNCTION - uint32_t urand() { - uint64_t state_0 = state_[ p_ * stride_ ]; - uint64_t state_1 = state_[ (p_ = ( p_ + 1 ) & 15) * stride_ ]; - state_1 ^= state_1 << 31; - state_1 ^= state_1 >> 11; - state_0 ^= state_0 >> 30; - uint64_t tmp = ( state_[ p_ * stride_ ] = state_0 ^ state_1 ) * 1181783497276652981ULL; - tmp = tmp>>16; - return static_cast(tmp&MAX_URAND); - } + KOKKOS_INLINE_FUNCTION + uint32_t urand() { + uint64_t state_0 = state_[p_ * stride_]; + uint64_t state_1 = state_[(p_ = (p_ + 1) & 15) * stride_]; + state_1 ^= state_1 << 31; + state_1 ^= state_1 >> 11; + state_0 ^= state_0 >> 30; + uint64_t tmp = + (state_[p_ * stride_] = state_0 ^ state_1) * 1181783497276652981ULL; + tmp = tmp >> 16; + return static_cast(tmp & MAX_URAND); + } - KOKKOS_INLINE_FUNCTION - uint64_t urand64() { - uint64_t state_0 = state_[ p_ * stride_ ]; - uint64_t state_1 = state_[ (p_ = ( p_ + 1 ) & 15) * stride_ ]; - state_1 ^= state_1 << 31; - state_1 ^= state_1 >> 11; - state_0 ^= state_0 >> 30; - return (( state_[ p_ * stride_ ] = state_0 ^ state_1 ) * 1181783497276652981LL) - 1; - } + KOKKOS_INLINE_FUNCTION + uint64_t urand64() { + uint64_t state_0 = state_[p_ * stride_]; + uint64_t state_1 = state_[(p_ = (p_ + 1) & 15) * stride_]; + state_1 ^= state_1 << 31; + state_1 ^= state_1 >> 11; + state_0 ^= state_0 >> 30; + return ((state_[p_ * stride_] = state_0 ^ state_1) * + 1181783497276652981LL) - + 1; + } - KOKKOS_INLINE_FUNCTION - uint32_t urand(const uint32_t& range) { - const uint32_t max_val = (MAX_URAND/range)*range; - uint32_t tmp = urand(); - while(tmp>=max_val) - urand(); - return tmp%range; - } + KOKKOS_INLINE_FUNCTION + uint32_t urand(const uint32_t& range) { + const uint32_t max_val = (MAX_URAND / range) * range; + uint32_t tmp = urand(); + while (tmp >= max_val) urand(); + return tmp % range; + } - KOKKOS_INLINE_FUNCTION - uint32_t urand(const uint32_t& start, const uint32_t& end ) { - return urand(end-start)+start; - } + KOKKOS_INLINE_FUNCTION + uint32_t urand(const uint32_t& start, const uint32_t& end) { + return urand(end - start) + start; + } - KOKKOS_INLINE_FUNCTION - uint64_t urand64(const uint64_t& range) { - const uint64_t max_val = (MAX_URAND64/range)*range; - uint64_t tmp = urand64(); - while(tmp>=max_val) - urand64(); - return tmp%range; - } + KOKKOS_INLINE_FUNCTION + uint64_t urand64(const uint64_t& range) { + const uint64_t max_val = (MAX_URAND64 / range) * range; + uint64_t tmp = urand64(); + while (tmp >= max_val) urand64(); + return tmp % range; + } - KOKKOS_INLINE_FUNCTION - uint64_t urand64(const uint64_t& start, const uint64_t& end ) { - return urand64(end-start)+start; - } + KOKKOS_INLINE_FUNCTION + uint64_t urand64(const uint64_t& start, const uint64_t& end) { + return urand64(end - start) + start; + } - KOKKOS_INLINE_FUNCTION - int rand() { - return static_cast(urand()/2); - } + KOKKOS_INLINE_FUNCTION + int rand() { return static_cast(urand() / 2); } - KOKKOS_INLINE_FUNCTION - int rand(const int& range) { - const int max_val = (MAX_RAND/range)*range; - int tmp = rand(); - while(tmp>=max_val) - rand(); - return tmp%range; - } + KOKKOS_INLINE_FUNCTION + int rand(const int& range) { + const int max_val = (MAX_RAND / range) * range; + int tmp = rand(); + while (tmp >= max_val) rand(); + return tmp % range; + } - KOKKOS_INLINE_FUNCTION - int rand(const int& start, const int& end ) { - return rand(end-start)+start; - } + KOKKOS_INLINE_FUNCTION + int rand(const int& start, const int& end) { + return rand(end - start) + start; + } - KOKKOS_INLINE_FUNCTION - int64_t rand64() { - return static_cast(urand64()/2); - } + KOKKOS_INLINE_FUNCTION + int64_t rand64() { return static_cast(urand64() / 2); } - KOKKOS_INLINE_FUNCTION - int64_t rand64(const int64_t& range) { - const int64_t max_val = (MAX_RAND64/range)*range; - int64_t tmp = rand64(); - while(tmp>=max_val) - rand64(); - return tmp%range; - } + KOKKOS_INLINE_FUNCTION + int64_t rand64(const int64_t& range) { + const int64_t max_val = (MAX_RAND64 / range) * range; + int64_t tmp = rand64(); + while (tmp >= max_val) rand64(); + return tmp % range; + } - KOKKOS_INLINE_FUNCTION - int64_t rand64(const int64_t& start, const int64_t& end ) { - return rand64(end-start)+start; - } + KOKKOS_INLINE_FUNCTION + int64_t rand64(const int64_t& start, const int64_t& end) { + return rand64(end - start) + start; + } - KOKKOS_INLINE_FUNCTION - float frand() { - return 1.0f * urand64()/MAX_URAND64; - } + KOKKOS_INLINE_FUNCTION + float frand() { return 1.0f * urand64() / MAX_URAND64; } - KOKKOS_INLINE_FUNCTION - float frand(const float& range) { - return range * urand64()/MAX_URAND64; - } + KOKKOS_INLINE_FUNCTION + float frand(const float& range) { return range * urand64() / MAX_URAND64; } - KOKKOS_INLINE_FUNCTION - float frand(const float& start, const float& end ) { - return frand(end-start)+start; - } + KOKKOS_INLINE_FUNCTION + float frand(const float& start, const float& end) { + return frand(end - start) + start; + } - KOKKOS_INLINE_FUNCTION - double drand() { - return 1.0 * urand64()/MAX_URAND64; - } + KOKKOS_INLINE_FUNCTION + double drand() { return 1.0 * urand64() / MAX_URAND64; } - KOKKOS_INLINE_FUNCTION - double drand(const double& range) { - return range * urand64()/MAX_URAND64; - } + KOKKOS_INLINE_FUNCTION + double drand(const double& range) { return range * urand64() / MAX_URAND64; } - KOKKOS_INLINE_FUNCTION - double drand(const double& start, const double& end ) { - return frand(end-start)+start; - } + KOKKOS_INLINE_FUNCTION + double drand(const double& start, const double& end) { + return frand(end - start) + start; + } - //Marsaglia polar method for drawing a standard normal distributed random number - KOKKOS_INLINE_FUNCTION - double normal() { - double S = 2.0; - double U; - while(S>=1.0) { - U = 2.0*drand() - 1.0; - const double V = 2.0*drand() - 1.0; - S = U*U+V*V; - } - return U*std::sqrt(-2.0*log(S)/S); - } + // Marsaglia polar method for drawing a standard normal distributed random + // number + KOKKOS_INLINE_FUNCTION + double normal() { + double S = 2.0; + double U; + while (S >= 1.0) { + U = 2.0 * drand() - 1.0; + const double V = 2.0 * drand() - 1.0; + S = U * U + V * V; + } + return U * std::sqrt(-2.0 * log(S) / S); + } - KOKKOS_INLINE_FUNCTION - double normal(const double& mean, const double& std_dev=1.0) { - return mean + normal()*std_dev; - } - }; + KOKKOS_INLINE_FUNCTION + double normal(const double& mean, const double& std_dev = 1.0) { + return mean + normal() * std_dev; + } +}; -template<> -inline -Random_XorShift64_Pool::Random_XorShift64_Pool(uint64_t seed) { +template <> +inline Random_XorShift64_Pool< + Kokkos::Experimental::ROCm>::Random_XorShift64_Pool(uint64_t seed) { num_states_ = 0; - init(seed,4*32768); + init(seed, 4 * 32768); } -template<> -KOKKOS_INLINE_FUNCTION -Random_XorShift64 Random_XorShift64_Pool::get_state() const { +template <> +KOKKOS_INLINE_FUNCTION Random_XorShift64 +Random_XorShift64_Pool::get_state() const { #ifdef __HCC_ACCELERATOR__ - const int i_offset = (threadIdx_x*blockDim_y + threadIdx_y)*blockDim_z+threadIdx_z; - int i = (((blockIdx_x*gridDim_y+blockIdx_y)*gridDim_z + blockIdx_z) * - blockDim_x*blockDim_y*blockDim_z + i_offset)%num_states_; - while(Kokkos::atomic_compare_exchange(&locks_(i),0,1)) { - i+=blockDim_x*blockDim_y*blockDim_z; - if(i>=num_states_) {i = i_offset;} + const int i_offset = + (threadIdx_x * blockDim_y + threadIdx_y) * blockDim_z + threadIdx_z; + int i = (((blockIdx_x * gridDim_y + blockIdx_y) * gridDim_z + blockIdx_z) * + blockDim_x * blockDim_y * blockDim_z + + i_offset) % + num_states_; + while (Kokkos::atomic_compare_exchange(&locks_(i), 0, 1)) { + i += blockDim_x * blockDim_y * blockDim_z; + if (i >= num_states_) { + i = i_offset; + } } - return Random_XorShift64(state_(i),i); + return Random_XorShift64(state_(i), i); #else - return Random_XorShift64(state_(0),0); + return Random_XorShift64(state_(0), 0); #endif } -template<> -KOKKOS_INLINE_FUNCTION -void Random_XorShift64_Pool::free_state(const Random_XorShift64 &state) const { +template <> +KOKKOS_INLINE_FUNCTION void +Random_XorShift64_Pool::free_state( + const Random_XorShift64& state) const { #ifdef __HCC_ACCELERATOR__ state_(state.state_idx_) = state.state_; locks_(state.state_idx_) = 0; @@ -1486,24 +1447,28 @@ void Random_XorShift64_Pool::free_state(const Random #endif } - -template<> -inline -Random_XorShift1024_Pool::Random_XorShift1024_Pool(uint64_t seed) { +template <> +inline Random_XorShift1024_Pool< + Kokkos::Experimental::ROCm>::Random_XorShift1024_Pool(uint64_t seed) { num_states_ = 0; - init(seed,4*32768); + init(seed, 4 * 32768); } -template<> -KOKKOS_INLINE_FUNCTION -Random_XorShift1024 Random_XorShift1024_Pool::get_state() const { +template <> +KOKKOS_INLINE_FUNCTION Random_XorShift1024 +Random_XorShift1024_Pool::get_state() const { #ifdef __HCC_ACCELERATOR__ - const int i_offset = (threadIdx_x*blockDim_y + threadIdx_y)*blockDim_z+threadIdx_z; - int i = (((blockIdx_x*gridDim_y+blockIdx_y)*gridDim_z + blockIdx_z) * - blockDim_x*blockDim_y*blockDim_z + i_offset)%num_states_; - while(Kokkos::atomic_compare_exchange(&locks_(i),0,1)) { - i+=blockDim_x*blockDim_y*blockDim_z; - if(i>=num_states_) {i = i_offset;} + const int i_offset = + (threadIdx_x * blockDim_y + threadIdx_y) * blockDim_z + threadIdx_z; + int i = (((blockIdx_x * gridDim_y + blockIdx_y) * gridDim_z + blockIdx_z) * + blockDim_x * blockDim_y * blockDim_z + + i_offset) % + num_states_; + while (Kokkos::atomic_compare_exchange(&locks_(i), 0, 1)) { + i += blockDim_x * blockDim_y * blockDim_z; + if (i >= num_states_) { + i = i_offset; + } } return Random_XorShift1024(state_, p_(i), i); @@ -1512,515 +1477,589 @@ Random_XorShift1024 Random_XorShift1024_Pool -KOKKOS_INLINE_FUNCTION -void Random_XorShift1024_Pool::free_state(const Random_XorShift1024 &state) const { +template <> +KOKKOS_INLINE_FUNCTION void +Random_XorShift1024_Pool::free_state( + const Random_XorShift1024& state) const { #ifdef __HCC_ACCELERATOR__ - for(int i=0; i<16; i++) - state_(state.state_idx_,i) = state.state_[i]; + for (int i = 0; i < 16; i++) state_(state.state_idx_, i) = state.state_[i]; locks_(state.state_idx_) = 0; return; #endif } - #endif - namespace Impl { -template +template struct fill_random_functor_range; -template +template struct fill_random_functor_begin_end; -template -struct fill_random_functor_range{ +template +struct fill_random_functor_range { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; typename ViewType::const_value_type range; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_range(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type range_): - a(a_),rand_pool(rand_pool_),range(range_) {} + typename ViewType::const_value_type range_) + : a(a_), rand_pool(rand_pool_), range(range_) {} KOKKOS_INLINE_FUNCTION - void operator() (const IndexType& i) const { + void operator()(const IndexType& i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) - a(idx) = Rand::draw(gen,range); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) + a(idx) = Rand::draw(gen, range); } rand_pool.free_state(gen); } }; -template -struct fill_random_functor_range{ +template +struct fill_random_functor_range { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; typename ViewType::const_value_type range; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_range(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type range_): - a(a_),rand_pool(rand_pool_),range(range_) {} + typename ViewType::const_value_type range_) + : a(a_), rand_pool(rand_pool_), range(range_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) { - for(IndexType k=0;k(a.extent(1));k++) - a(idx,k) = Rand::draw(gen,range); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType k = 0; k < static_cast(a.extent(1)); k++) + a(idx, k) = Rand::draw(gen, range); } } rand_pool.free_state(gen); } }; - -template -struct fill_random_functor_range{ +template +struct fill_random_functor_range { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; typename ViewType::const_value_type range; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_range(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type range_): - a(a_),rand_pool(rand_pool_),range(range_) {} + typename ViewType::const_value_type range_) + : a(a_), rand_pool(rand_pool_), range(range_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) { - for(IndexType k=0;k(a.extent(1));k++) - for(IndexType l=0;l(a.extent(2));l++) - a(idx,k,l) = Rand::draw(gen,range); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType k = 0; k < static_cast(a.extent(1)); k++) + for (IndexType l = 0; l < static_cast(a.extent(2)); l++) + a(idx, k, l) = Rand::draw(gen, range); } } rand_pool.free_state(gen); } }; -template -struct fill_random_functor_range{ +template +struct fill_random_functor_range { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; typename ViewType::const_value_type range; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_range(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type range_): - a(a_),rand_pool(rand_pool_),range(range_) {} + typename ViewType::const_value_type range_) + : a(a_), rand_pool(rand_pool_), range(range_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) { - for(IndexType k=0;k(a.extent(1));k++) - for(IndexType l=0;l(a.extent(2));l++) - for(IndexType m=0;m(a.extent(3));m++) - a(idx,k,l,m) = Rand::draw(gen,range); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType k = 0; k < static_cast(a.extent(1)); k++) + for (IndexType l = 0; l < static_cast(a.extent(2)); l++) + for (IndexType m = 0; m < static_cast(a.extent(3)); m++) + a(idx, k, l, m) = Rand::draw(gen, range); } } rand_pool.free_state(gen); } }; -template -struct fill_random_functor_range{ +template +struct fill_random_functor_range { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; typename ViewType::const_value_type range; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_range(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type range_): - a(a_),rand_pool(rand_pool_),range(range_) {} + typename ViewType::const_value_type range_) + : a(a_), rand_pool(rand_pool_), range(range_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) { - for(IndexType k=0;k(a.extent(1));k++) - for(IndexType l=0;l(a.extent(2));l++) - for(IndexType m=0;m(a.extent(3));m++) - for(IndexType n=0;n(a.extent(4));n++) - a(idx,k,l,m,n) = Rand::draw(gen,range); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType k = 0; k < static_cast(a.extent(1)); k++) + for (IndexType l = 0; l < static_cast(a.extent(2)); l++) + for (IndexType m = 0; m < static_cast(a.extent(3)); m++) + for (IndexType n = 0; n < static_cast(a.extent(4)); + n++) + a(idx, k, l, m, n) = Rand::draw(gen, range); } } rand_pool.free_state(gen); } }; -template -struct fill_random_functor_range{ +template +struct fill_random_functor_range { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; typename ViewType::const_value_type range; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_range(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type range_): - a(a_),rand_pool(rand_pool_),range(range_) {} + typename ViewType::const_value_type range_) + : a(a_), rand_pool(rand_pool_), range(range_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) { - for(IndexType k=0;k(a.extent(1));k++) - for(IndexType l=0;l(a.extent(2));l++) - for(IndexType m=0;m(a.extent(3));m++) - for(IndexType n=0;n(a.extent(4));n++) - for(IndexType o=0;o(a.extent(5));o++) - a(idx,k,l,m,n,o) = Rand::draw(gen,range); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType k = 0; k < static_cast(a.extent(1)); k++) + for (IndexType l = 0; l < static_cast(a.extent(2)); l++) + for (IndexType m = 0; m < static_cast(a.extent(3)); m++) + for (IndexType n = 0; n < static_cast(a.extent(4)); + n++) + for (IndexType o = 0; o < static_cast(a.extent(5)); + o++) + a(idx, k, l, m, n, o) = Rand::draw(gen, range); } } rand_pool.free_state(gen); } }; -template -struct fill_random_functor_range{ +template +struct fill_random_functor_range { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; typename ViewType::const_value_type range; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_range(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type range_): - a(a_),rand_pool(rand_pool_),range(range_) {} + typename ViewType::const_value_type range_) + : a(a_), rand_pool(rand_pool_), range(range_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) { - for(IndexType k=0;k(a.extent(1));k++) - for(IndexType l=0;l(a.extent(2));l++) - for(IndexType m=0;m(a.extent(3));m++) - for(IndexType n=0;n(a.extent(4));n++) - for(IndexType o=0;o(a.extent(5));o++) - for(IndexType p=0;p(a.extent(6));p++) - a(idx,k,l,m,n,o,p) = Rand::draw(gen,range); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType k = 0; k < static_cast(a.extent(1)); k++) + for (IndexType l = 0; l < static_cast(a.extent(2)); l++) + for (IndexType m = 0; m < static_cast(a.extent(3)); m++) + for (IndexType n = 0; n < static_cast(a.extent(4)); + n++) + for (IndexType o = 0; o < static_cast(a.extent(5)); + o++) + for (IndexType p = 0; p < static_cast(a.extent(6)); + p++) + a(idx, k, l, m, n, o, p) = Rand::draw(gen, range); } } rand_pool.free_state(gen); } }; -template -struct fill_random_functor_range{ +template +struct fill_random_functor_range { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; typename ViewType::const_value_type range; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_range(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type range_): - a(a_),rand_pool(rand_pool_),range(range_) {} + typename ViewType::const_value_type range_) + : a(a_), rand_pool(rand_pool_), range(range_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) { - for(IndexType k=0;k(a.extent(1));k++) - for(IndexType l=0;l(a.extent(2));l++) - for(IndexType m=0;m(a.extent(3));m++) - for(IndexType n=0;n(a.extent(4));n++) - for(IndexType o=0;o(a.extent(5));o++) - for(IndexType p=0;p(a.extent(6));p++) - for(IndexType q=0;q(a.extent(7));q++) - a(idx,k,l,m,n,o,p,q) = Rand::draw(gen,range); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType k = 0; k < static_cast(a.extent(1)); k++) + for (IndexType l = 0; l < static_cast(a.extent(2)); l++) + for (IndexType m = 0; m < static_cast(a.extent(3)); m++) + for (IndexType n = 0; n < static_cast(a.extent(4)); + n++) + for (IndexType o = 0; o < static_cast(a.extent(5)); + o++) + for (IndexType p = 0; p < static_cast(a.extent(6)); + p++) + for (IndexType q = 0; + q < static_cast(a.extent(7)); q++) + a(idx, k, l, m, n, o, p, q) = Rand::draw(gen, range); } } rand_pool.free_state(gen); } }; -template -struct fill_random_functor_begin_end{ +template +struct fill_random_functor_begin_end { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; - typename ViewType::const_value_type begin,end; + typename ViewType::const_value_type begin, end; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_begin_end(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type begin_, typename ViewType::const_value_type end_): - a(a_),rand_pool(rand_pool_),begin(begin_),end(end_) {} + typename ViewType::const_value_type begin_, + typename ViewType::const_value_type end_) + : a(a_), rand_pool(rand_pool_), begin(begin_), end(end_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) - a(idx) = Rand::draw(gen,begin,end); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) + a(idx) = Rand::draw(gen, begin, end); } rand_pool.free_state(gen); } }; -template -struct fill_random_functor_begin_end{ +template +struct fill_random_functor_begin_end { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; - typename ViewType::const_value_type begin,end; + typename ViewType::const_value_type begin, end; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_begin_end(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type begin_, typename ViewType::const_value_type end_): - a(a_),rand_pool(rand_pool_),begin(begin_),end(end_) {} + typename ViewType::const_value_type begin_, + typename ViewType::const_value_type end_) + : a(a_), rand_pool(rand_pool_), begin(begin_), end(end_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) { - for(IndexType k=0;k(a.extent(1));k++) - a(idx,k) = Rand::draw(gen,begin,end); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType k = 0; k < static_cast(a.extent(1)); k++) + a(idx, k) = Rand::draw(gen, begin, end); } } rand_pool.free_state(gen); } }; - -template -struct fill_random_functor_begin_end{ +template +struct fill_random_functor_begin_end { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; - typename ViewType::const_value_type begin,end; + typename ViewType::const_value_type begin, end; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_begin_end(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type begin_, typename ViewType::const_value_type end_): - a(a_),rand_pool(rand_pool_),begin(begin_),end(end_) {} + typename ViewType::const_value_type begin_, + typename ViewType::const_value_type end_) + : a(a_), rand_pool(rand_pool_), begin(begin_), end(end_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) { - for(IndexType k=0;k(a.extent(1));k++) - for(IndexType l=0;l(a.extent(2));l++) - a(idx,k,l) = Rand::draw(gen,begin,end); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType k = 0; k < static_cast(a.extent(1)); k++) + for (IndexType l = 0; l < static_cast(a.extent(2)); l++) + a(idx, k, l) = Rand::draw(gen, begin, end); } } rand_pool.free_state(gen); } }; -template -struct fill_random_functor_begin_end{ +template +struct fill_random_functor_begin_end { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; - typename ViewType::const_value_type begin,end; + typename ViewType::const_value_type begin, end; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_begin_end(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type begin_, typename ViewType::const_value_type end_): - a(a_),rand_pool(rand_pool_),begin(begin_),end(end_) {} + typename ViewType::const_value_type begin_, + typename ViewType::const_value_type end_) + : a(a_), rand_pool(rand_pool_), begin(begin_), end(end_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) { - for(IndexType k=0;k(a.extent(1));k++) - for(IndexType l=0;l(a.extent(2));l++) - for(IndexType m=0;m(a.extent(3));m++) - a(idx,k,l,m) = Rand::draw(gen,begin,end); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType k = 0; k < static_cast(a.extent(1)); k++) + for (IndexType l = 0; l < static_cast(a.extent(2)); l++) + for (IndexType m = 0; m < static_cast(a.extent(3)); m++) + a(idx, k, l, m) = Rand::draw(gen, begin, end); } } rand_pool.free_state(gen); } }; -template -struct fill_random_functor_begin_end{ +template +struct fill_random_functor_begin_end { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; - typename ViewType::const_value_type begin,end; + typename ViewType::const_value_type begin, end; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_begin_end(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type begin_, typename ViewType::const_value_type end_): - a(a_),rand_pool(rand_pool_),begin(begin_),end(end_) {} + typename ViewType::const_value_type begin_, + typename ViewType::const_value_type end_) + : a(a_), rand_pool(rand_pool_), begin(begin_), end(end_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))){ - for(IndexType l=0;l(a.extent(1));l++) - for(IndexType m=0;m(a.extent(2));m++) - for(IndexType n=0;n(a.extent(3));n++) - for(IndexType o=0;o(a.extent(4));o++) - a(idx,l,m,n,o) = Rand::draw(gen,begin,end); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType l = 0; l < static_cast(a.extent(1)); l++) + for (IndexType m = 0; m < static_cast(a.extent(2)); m++) + for (IndexType n = 0; n < static_cast(a.extent(3)); n++) + for (IndexType o = 0; o < static_cast(a.extent(4)); + o++) + a(idx, l, m, n, o) = Rand::draw(gen, begin, end); } } rand_pool.free_state(gen); } }; -template -struct fill_random_functor_begin_end{ +template +struct fill_random_functor_begin_end { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; - typename ViewType::const_value_type begin,end; + typename ViewType::const_value_type begin, end; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_begin_end(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type begin_, typename ViewType::const_value_type end_): - a(a_),rand_pool(rand_pool_),begin(begin_),end(end_) {} + typename ViewType::const_value_type begin_, + typename ViewType::const_value_type end_) + : a(a_), rand_pool(rand_pool_), begin(begin_), end(end_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) { - for(IndexType k=0;k(a.extent(1));k++) - for(IndexType l=0;l(a.extent(2));l++) - for(IndexType m=0;m(a.extent(3));m++) - for(IndexType n=0;n(a.extent(4));n++) - for(IndexType o=0;o(a.extent(5));o++) - a(idx,k,l,m,n,o) = Rand::draw(gen,begin,end); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType k = 0; k < static_cast(a.extent(1)); k++) + for (IndexType l = 0; l < static_cast(a.extent(2)); l++) + for (IndexType m = 0; m < static_cast(a.extent(3)); m++) + for (IndexType n = 0; n < static_cast(a.extent(4)); + n++) + for (IndexType o = 0; o < static_cast(a.extent(5)); + o++) + a(idx, k, l, m, n, o) = Rand::draw(gen, begin, end); } } rand_pool.free_state(gen); } }; - -template -struct fill_random_functor_begin_end{ +template +struct fill_random_functor_begin_end { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; - typename ViewType::const_value_type begin,end; + typename ViewType::const_value_type begin, end; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_begin_end(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type begin_, typename ViewType::const_value_type end_): - a(a_),rand_pool(rand_pool_),begin(begin_),end(end_) {} + typename ViewType::const_value_type begin_, + typename ViewType::const_value_type end_) + : a(a_), rand_pool(rand_pool_), begin(begin_), end(end_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) { - for(IndexType k=0;k(a.extent(1));k++) - for(IndexType l=0;l(a.extent(2));l++) - for(IndexType m=0;m(a.extent(3));m++) - for(IndexType n=0;n(a.extent(4));n++) - for(IndexType o=0;o(a.extent(5));o++) - for(IndexType p=0;p(a.extent(6));p++) - a(idx,k,l,m,n,o,p) = Rand::draw(gen,begin,end); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType k = 0; k < static_cast(a.extent(1)); k++) + for (IndexType l = 0; l < static_cast(a.extent(2)); l++) + for (IndexType m = 0; m < static_cast(a.extent(3)); m++) + for (IndexType n = 0; n < static_cast(a.extent(4)); + n++) + for (IndexType o = 0; o < static_cast(a.extent(5)); + o++) + for (IndexType p = 0; p < static_cast(a.extent(6)); + p++) + a(idx, k, l, m, n, o, p) = Rand::draw(gen, begin, end); } } rand_pool.free_state(gen); } }; -template -struct fill_random_functor_begin_end{ +template +struct fill_random_functor_begin_end { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; - typename ViewType::const_value_type begin,end; + typename ViewType::const_value_type begin, end; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_begin_end(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type begin_, typename ViewType::const_value_type end_): - a(a_),rand_pool(rand_pool_),begin(begin_),end(end_) {} + typename ViewType::const_value_type begin_, + typename ViewType::const_value_type end_) + : a(a_), rand_pool(rand_pool_), begin(begin_), end(end_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) { - for(IndexType k=0;k(a.extent(1));k++) - for(IndexType l=0;l(a.extent(2));l++) - for(IndexType m=0;m(a.extent(3));m++) - for(IndexType n=0;n(a.extent(4));n++) - for(IndexType o=0;o(a.extent(5));o++) - for(IndexType p=0;p(a.extent(6));p++) - for(IndexType q=0;q(a.extent(7));q++) - a(idx,k,l,m,n,o,p,q) = Rand::draw(gen,begin,end); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType k = 0; k < static_cast(a.extent(1)); k++) + for (IndexType l = 0; l < static_cast(a.extent(2)); l++) + for (IndexType m = 0; m < static_cast(a.extent(3)); m++) + for (IndexType n = 0; n < static_cast(a.extent(4)); + n++) + for (IndexType o = 0; o < static_cast(a.extent(5)); + o++) + for (IndexType p = 0; p < static_cast(a.extent(6)); + p++) + for (IndexType q = 0; + q < static_cast(a.extent(7)); q++) + a(idx, k, l, m, n, o, p, q) = Rand::draw(gen, begin, end); } } rand_pool.free_state(gen); } }; -} +} // namespace Impl -template -void fill_random(ViewType a, RandomPool g, typename ViewType::const_value_type range) { +template +void fill_random(ViewType a, RandomPool g, + typename ViewType::const_value_type range) { int64_t LDA = a.extent(0); - if(LDA>0) - parallel_for((LDA+127)/128,Impl::fill_random_functor_range(a,g,range)); + if (LDA > 0) + parallel_for((LDA + 127) / 128, + Impl::fill_random_functor_range( + a, g, range)); } -template -void fill_random(ViewType a, RandomPool g, typename ViewType::const_value_type begin,typename ViewType::const_value_type end ) { +template +void fill_random(ViewType a, RandomPool g, + typename ViewType::const_value_type begin, + typename ViewType::const_value_type end) { int64_t LDA = a.extent(0); - if(LDA>0) - parallel_for((LDA+127)/128,Impl::fill_random_functor_begin_end(a,g,begin,end)); -} + if (LDA > 0) + parallel_for((LDA + 127) / 128, + Impl::fill_random_functor_begin_end( + a, g, begin, end)); } +} // namespace Kokkos #endif diff --git a/lib/kokkos/algorithms/src/Kokkos_Sort.hpp b/lib/kokkos/algorithms/src/Kokkos_Sort.hpp index 7fb8505fe5327470c6dac96ba17631d460443675..b7a988361f52f382263beb2725758468138fec4f 100644 --- a/lib/kokkos/algorithms/src/Kokkos_Sort.hpp +++ b/lib/kokkos/algorithms/src/Kokkos_Sort.hpp @@ -1,13 +1,14 @@ /* //@HEADER // ************************************************************************ -// -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -36,12 +37,11 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// +// // ************************************************************************ //@HEADER */ - #ifndef KOKKOS_SORT_HPP_ #define KOKKOS_SORT_HPP_ @@ -51,125 +51,107 @@ namespace Kokkos { - namespace Impl { +namespace Impl { - template< class DstViewType , class SrcViewType - , int Rank = DstViewType::Rank > - struct CopyOp; +template +struct CopyOp; - template< class DstViewType , class SrcViewType > - struct CopyOp { - KOKKOS_INLINE_FUNCTION - static void copy(DstViewType const& dst, size_t i_dst, - SrcViewType const& src, size_t i_src ) { - dst(i_dst) = src(i_src); - } - }; +template +struct CopyOp { + KOKKOS_INLINE_FUNCTION + static void copy(DstViewType const& dst, size_t i_dst, SrcViewType const& src, + size_t i_src) { + dst(i_dst) = src(i_src); + } +}; - template< class DstViewType , class SrcViewType > - struct CopyOp { - KOKKOS_INLINE_FUNCTION - static void copy(DstViewType const& dst, size_t i_dst, - SrcViewType const& src, size_t i_src ) { - for(int j = 0;j< (int) dst.extent(1); j++) - dst(i_dst,j) = src(i_src,j); - } - }; +template +struct CopyOp { + KOKKOS_INLINE_FUNCTION + static void copy(DstViewType const& dst, size_t i_dst, SrcViewType const& src, + size_t i_src) { + for (int j = 0; j < (int)dst.extent(1); j++) dst(i_dst, j) = src(i_src, j); + } +}; - template< class DstViewType , class SrcViewType > - struct CopyOp { - KOKKOS_INLINE_FUNCTION - static void copy(DstViewType const& dst, size_t i_dst, - SrcViewType const& src, size_t i_src ) { - for(int j = 0; j +struct CopyOp { + KOKKOS_INLINE_FUNCTION + static void copy(DstViewType const& dst, size_t i_dst, SrcViewType const& src, + size_t i_src) { + for (int j = 0; j < dst.extent(1); j++) + for (int k = 0; k < dst.extent(2); k++) + dst(i_dst, j, k) = src(i_src, j, k); } +}; +} // namespace Impl //---------------------------------------------------------------------------- -template< class KeyViewType - , class BinSortOp - , class Space = typename KeyViewType::device_type - , class SizeType = typename KeyViewType::memory_space::size_type - > +template class BinSort { -public: - - template< class DstViewType , class SrcViewType > + public: + template struct copy_functor { + typedef typename SrcViewType::const_type src_view_type; - typedef typename SrcViewType::const_type src_view_type ; + typedef Impl::CopyOp copy_op; - typedef Impl::CopyOp< DstViewType , src_view_type > copy_op ; + DstViewType dst_values; + src_view_type src_values; + int dst_offset; - DstViewType dst_values ; - src_view_type src_values ; - int dst_offset ; - - copy_functor( DstViewType const & dst_values_ - , int const & dst_offset_ - , SrcViewType const & src_values_ - ) - : dst_values( dst_values_ ) - , src_values( src_values_ ) - , dst_offset( dst_offset_ ) - {} + copy_functor(DstViewType const& dst_values_, int const& dst_offset_, + SrcViewType const& src_values_) + : dst_values(dst_values_), + src_values(src_values_), + dst_offset(dst_offset_) {} KOKKOS_INLINE_FUNCTION - void operator() (const int& i) const { - copy_op::copy(dst_values,i+dst_offset,src_values,i); + void operator()(const int& i) const { + copy_op::copy(dst_values, i + dst_offset, src_values, i); } }; - template< class DstViewType - , class PermuteViewType - , class SrcViewType - > + template struct copy_permute_functor { - // If a Kokkos::View then can generate constant random access // otherwise can only use the constant type. - typedef typename std::conditional - < Kokkos::is_view< SrcViewType >::value - , Kokkos::View< typename SrcViewType::const_data_type - , typename SrcViewType::array_layout - , typename SrcViewType::device_type - , Kokkos::MemoryTraits - > - , typename SrcViewType::const_type - >::type src_view_type ; - - typedef typename PermuteViewType::const_type perm_view_type ; - - typedef Impl::CopyOp< DstViewType , src_view_type > copy_op ; - - DstViewType dst_values ; - perm_view_type sort_order ; - src_view_type src_values ; - int src_offset ; - - copy_permute_functor( DstViewType const & dst_values_ - , PermuteViewType const & sort_order_ - , SrcViewType const & src_values_ - , int const & src_offset_ - ) - : dst_values( dst_values_ ) - , sort_order( sort_order_ ) - , src_values( src_values_ ) - , src_offset( src_offset_ ) - {} + typedef typename std::conditional< + Kokkos::is_view::value, + Kokkos::View >, + typename SrcViewType::const_type>::type src_view_type; + + typedef typename PermuteViewType::const_type perm_view_type; + + typedef Impl::CopyOp copy_op; + + DstViewType dst_values; + perm_view_type sort_order; + src_view_type src_values; + int src_offset; + + copy_permute_functor(DstViewType const& dst_values_, + PermuteViewType const& sort_order_, + SrcViewType const& src_values_, int const& src_offset_) + : dst_values(dst_values_), + sort_order(sort_order_), + src_values(src_values_), + src_offset(src_offset_) {} KOKKOS_INLINE_FUNCTION - void operator() (const int& i) const { - copy_op::copy(dst_values,i,src_values,src_offset+sort_order(i)); + void operator()(const int& i) const { + copy_op::copy(dst_values, i, src_values, src_offset + sort_order(i)); } }; - typedef typename Space::execution_space execution_space; + typedef typename Space::execution_space execution_space; typedef BinSortOp bin_op_type; struct bin_count_tag {}; @@ -177,221 +159,236 @@ public: struct bin_binning_tag {}; struct bin_sort_bins_tag {}; -public: - + public: typedef SizeType size_type; typedef size_type value_type; typedef Kokkos::View offset_type; typedef Kokkos::View bin_count_type; - typedef typename KeyViewType::const_type const_key_view_type ; + typedef typename KeyViewType::const_type const_key_view_type; // If a Kokkos::View then can generate constant random access // otherwise can only use the constant type. - typedef typename std::conditional - < Kokkos::is_view< KeyViewType >::value - , Kokkos::View< typename KeyViewType::const_data_type, - typename KeyViewType::array_layout, - typename KeyViewType::device_type, - Kokkos::MemoryTraits > - , const_key_view_type - >::type const_rnd_key_view_type; + typedef typename std::conditional< + Kokkos::is_view::value, + Kokkos::View >, + const_key_view_type>::type const_rnd_key_view_type; typedef typename KeyViewType::non_const_value_type non_const_key_scalar; - typedef typename KeyViewType::const_value_type const_key_scalar; - - typedef Kokkos::View > bin_count_atomic_type ; + typedef typename KeyViewType::const_value_type const_key_scalar; -private: + typedef Kokkos::View > + bin_count_atomic_type; + private: const_key_view_type keys; const_rnd_key_view_type keys_rnd; -public: - - BinSortOp bin_op ; - offset_type bin_offsets ; - bin_count_atomic_type bin_count_atomic ; - bin_count_type bin_count_const ; - offset_type sort_order ; + public: + BinSortOp bin_op; + offset_type bin_offsets; + bin_count_atomic_type bin_count_atomic; + bin_count_type bin_count_const; + offset_type sort_order; - int range_begin ; - int range_end ; - bool sort_within_bins ; - -public: + int range_begin; + int range_end; + bool sort_within_bins; + public: BinSort() {} //---------------------------------------- - // Constructor: takes the keys, the binning_operator and optionally whether to sort within bins (default false) - BinSort( const_key_view_type keys_ - , int range_begin_ - , int range_end_ - , BinSortOp bin_op_ - , bool sort_within_bins_ = false - ) - : keys(keys_) - , keys_rnd(keys_) - , bin_op(bin_op_) - , bin_offsets() - , bin_count_atomic() - , bin_count_const() - , sort_order() - , range_begin( range_begin_ ) - , range_end( range_end_ ) - , sort_within_bins( sort_within_bins_ ) - { - bin_count_atomic = Kokkos::View("Kokkos::SortImpl::BinSortFunctor::bin_count",bin_op.max_bins()); - bin_count_const = bin_count_atomic; - bin_offsets = offset_type(ViewAllocateWithoutInitializing("Kokkos::SortImpl::BinSortFunctor::bin_offsets"),bin_op.max_bins()); - sort_order = offset_type(ViewAllocateWithoutInitializing("Kokkos::SortImpl::BinSortFunctor::sort_order"),range_end-range_begin); + // Constructor: takes the keys, the binning_operator and optionally whether to + // sort within bins (default false) + BinSort(const_key_view_type keys_, int range_begin_, int range_end_, + BinSortOp bin_op_, bool sort_within_bins_ = false) + : keys(keys_), + keys_rnd(keys_), + bin_op(bin_op_), + bin_offsets(), + bin_count_atomic(), + bin_count_const(), + sort_order(), + range_begin(range_begin_), + range_end(range_end_), + sort_within_bins(sort_within_bins_) { + bin_count_atomic = Kokkos::View( + "Kokkos::SortImpl::BinSortFunctor::bin_count", bin_op.max_bins()); + bin_count_const = bin_count_atomic; + bin_offsets = + offset_type(ViewAllocateWithoutInitializing( + "Kokkos::SortImpl::BinSortFunctor::bin_offsets"), + bin_op.max_bins()); + sort_order = + offset_type(ViewAllocateWithoutInitializing( + "Kokkos::SortImpl::BinSortFunctor::sort_order"), + range_end - range_begin); } - BinSort( const_key_view_type keys_ - , BinSortOp bin_op_ - , bool sort_within_bins_ = false - ) - : BinSort( keys_ , 0 , keys_.extent(0), bin_op_ , sort_within_bins_ ) {} + BinSort(const_key_view_type keys_, BinSortOp bin_op_, + bool sort_within_bins_ = false) + : BinSort(keys_, 0, keys_.extent(0), bin_op_, sort_within_bins_) {} //---------------------------------------- - // Create the permutation vector, the bin_offset array and the bin_count array. Can be called again if keys changed + // Create the permutation vector, the bin_offset array and the bin_count + // array. Can be called again if keys changed void create_permute_vector() { - const size_t len = range_end - range_begin ; - Kokkos::parallel_for ("Kokkos::Sort::BinCount",Kokkos::RangePolicy (0,len),*this); - Kokkos::parallel_scan("Kokkos::Sort::BinOffset",Kokkos::RangePolicy (0,bin_op.max_bins()) ,*this); - - Kokkos::deep_copy(bin_count_atomic,0); - Kokkos::parallel_for ("Kokkos::Sort::BinBinning",Kokkos::RangePolicy (0,len),*this); - - if(sort_within_bins) - Kokkos::parallel_for ("Kokkos::Sort::BinSort",Kokkos::RangePolicy(0,bin_op.max_bins()) ,*this); + const size_t len = range_end - range_begin; + Kokkos::parallel_for( + "Kokkos::Sort::BinCount", + Kokkos::RangePolicy(0, len), *this); + Kokkos::parallel_scan("Kokkos::Sort::BinOffset", + Kokkos::RangePolicy( + 0, bin_op.max_bins()), + *this); + + Kokkos::deep_copy(bin_count_atomic, 0); + Kokkos::parallel_for( + "Kokkos::Sort::BinBinning", + Kokkos::RangePolicy(0, len), *this); + + if (sort_within_bins) + Kokkos::parallel_for( + "Kokkos::Sort::BinSort", + Kokkos::RangePolicy( + 0, bin_op.max_bins()), + *this); } - // Sort a subset of a view with respect to the first dimension using the permutation array - template - void sort( ValuesViewType const & values - , int values_range_begin - , int values_range_end) const - { - typedef - Kokkos::View< typename ValuesViewType::data_type, - typename ValuesViewType::array_layout, - typename ValuesViewType::device_type > - scratch_view_type ; - - const size_t len = range_end - range_begin ; - const size_t values_len = values_range_end - values_range_begin ; + // Sort a subset of a view with respect to the first dimension using the + // permutation array + template + void sort(ValuesViewType const& values, int values_range_begin, + int values_range_end) const { + typedef Kokkos::View + scratch_view_type; + + const size_t len = range_end - range_begin; + const size_t values_len = values_range_end - values_range_begin; if (len != values_len) { - Kokkos::abort("BinSort::sort: values range length != permutation vector length"); + Kokkos::abort( + "BinSort::sort: values range length != permutation vector length"); } #ifdef KOKKOS_ENABLE_DEPRECATED_CODE - scratch_view_type - sorted_values(ViewAllocateWithoutInitializing("Kokkos::SortImpl::BinSortFunctor::sorted_values"), - len, - values.extent(1), - values.extent(2), - values.extent(3), - values.extent(4), - values.extent(5), - values.extent(6), - values.extent(7)); + scratch_view_type sorted_values( + ViewAllocateWithoutInitializing( + "Kokkos::SortImpl::BinSortFunctor::sorted_values"), + len, values.extent(1), values.extent(2), values.extent(3), + values.extent(4), values.extent(5), values.extent(6), values.extent(7)); #else - scratch_view_type - sorted_values(ViewAllocateWithoutInitializing("Kokkos::SortImpl::BinSortFunctor::sorted_values"), - values.rank_dynamic > 0 ? len : KOKKOS_IMPL_CTOR_DEFAULT_ARG, - values.rank_dynamic > 1 ? values.extent(1) : KOKKOS_IMPL_CTOR_DEFAULT_ARG , - values.rank_dynamic > 2 ? values.extent(2) : KOKKOS_IMPL_CTOR_DEFAULT_ARG, - values.rank_dynamic > 3 ? values.extent(3) : KOKKOS_IMPL_CTOR_DEFAULT_ARG, - values.rank_dynamic > 4 ? values.extent(4) : KOKKOS_IMPL_CTOR_DEFAULT_ARG, - values.rank_dynamic > 5 ? values.extent(5) : KOKKOS_IMPL_CTOR_DEFAULT_ARG, - values.rank_dynamic > 6 ? values.extent(6) : KOKKOS_IMPL_CTOR_DEFAULT_ARG, - values.rank_dynamic > 7 ? values.extent(7) : KOKKOS_IMPL_CTOR_DEFAULT_ARG); + scratch_view_type sorted_values( + ViewAllocateWithoutInitializing( + "Kokkos::SortImpl::BinSortFunctor::sorted_values"), + values.rank_dynamic > 0 ? len : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + values.rank_dynamic > 1 ? values.extent(1) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + values.rank_dynamic > 2 ? values.extent(2) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + values.rank_dynamic > 3 ? values.extent(3) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + values.rank_dynamic > 4 ? values.extent(4) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + values.rank_dynamic > 5 ? values.extent(5) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + values.rank_dynamic > 6 ? values.extent(6) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + values.rank_dynamic > 7 ? values.extent(7) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG); #endif { - copy_permute_functor< scratch_view_type /* DstViewType */ - , offset_type /* PermuteViewType */ - , ValuesViewType /* SrcViewType */ - > - functor( sorted_values , sort_order , values, values_range_begin - range_begin ); - - parallel_for("Kokkos::Sort::CopyPermute", Kokkos::RangePolicy(0,len),functor); + copy_permute_functor + functor(sorted_values, sort_order, values, + values_range_begin - range_begin); + + parallel_for("Kokkos::Sort::CopyPermute", + Kokkos::RangePolicy(0, len), functor); } { - copy_functor< ValuesViewType , scratch_view_type > - functor( values , range_begin , sorted_values ); + copy_functor functor( + values, range_begin, sorted_values); - parallel_for("Kokkos::Sort::Copy", Kokkos::RangePolicy(0,len),functor); + parallel_for("Kokkos::Sort::Copy", + Kokkos::RangePolicy(0, len), functor); } Kokkos::fence(); } - template - void sort( ValuesViewType const & values ) const - { - this->sort( values, 0, /*values.extent(0)*/ range_end - range_begin ); + template + void sort(ValuesViewType const& values) const { + this->sort(values, 0, /*values.extent(0)*/ range_end - range_begin); } // Get the permutation vector KOKKOS_INLINE_FUNCTION - offset_type get_permute_vector() const { return sort_order;} + offset_type get_permute_vector() const { return sort_order; } // Get the start offsets for each bin KOKKOS_INLINE_FUNCTION - offset_type get_bin_offsets() const { return bin_offsets;} + offset_type get_bin_offsets() const { return bin_offsets; } // Get the count for each bin KOKKOS_INLINE_FUNCTION - bin_count_type get_bin_count() const {return bin_count_const;} - -public: + bin_count_type get_bin_count() const { return bin_count_const; } + public: KOKKOS_INLINE_FUNCTION - void operator() (const bin_count_tag& tag, const int& i) const { - const int j = range_begin + i ; + void operator()(const bin_count_tag& tag, const int& i) const { + const int j = range_begin + i; bin_count_atomic(bin_op.bin(keys, j))++; } KOKKOS_INLINE_FUNCTION - void operator() (const bin_offset_tag& tag, const int& i, value_type& offset, const bool& final) const { - if(final) { + void operator()(const bin_offset_tag& tag, const int& i, value_type& offset, + const bool& final) const { + if (final) { bin_offsets(i) = offset; } - offset+=bin_count_const(i); + offset += bin_count_const(i); } KOKKOS_INLINE_FUNCTION - void operator() (const bin_binning_tag& tag, const int& i) const { - const int j = range_begin + i ; - const int bin = bin_op.bin(keys,j); + void operator()(const bin_binning_tag& tag, const int& i) const { + const int j = range_begin + i; + const int bin = bin_op.bin(keys, j); const int count = bin_count_atomic(bin)++; - sort_order(bin_offsets(bin) + count) = j ; + sort_order(bin_offsets(bin) + count) = j; } KOKKOS_INLINE_FUNCTION - void operator() (const bin_sort_bins_tag& tag, const int&i ) const { + void operator()(const bin_sort_bins_tag& tag, const int& i) const { auto bin_size = bin_count_const(i); if (bin_size <= 1) return; - int upper_bound = bin_offsets(i)+bin_size; - bool sorted = false; - while(!sorted) { - sorted = true; + int upper_bound = bin_offsets(i) + bin_size; + bool sorted = false; + while (!sorted) { + sorted = true; int old_idx = sort_order(bin_offsets(i)); int new_idx; - for(int k=bin_offsets(i)+1; k +template struct BinOp1D { int max_bins_; double mul_; typename KeyViewType::const_value_type range_; typename KeyViewType::const_value_type min_; - BinOp1D():max_bins_(0),mul_(0.0), - range_(typename KeyViewType::const_value_type()), - min_(typename KeyViewType::const_value_type()) {} + BinOp1D() + : max_bins_(0), + mul_(0.0), + range_(typename KeyViewType::const_value_type()), + min_(typename KeyViewType::const_value_type()) {} - //Construct BinOp with number of bins, minimum value and maxuimum value + // Construct BinOp with number of bins, minimum value and maxuimum value BinOp1D(int max_bins__, typename KeyViewType::const_value_type min, - typename KeyViewType::const_value_type max ) - :max_bins_(max_bins__+1),mul_(1.0*max_bins__/(max-min)),range_(max-min),min_(min) {} - - //Determine bin index from key value - template - KOKKOS_INLINE_FUNCTION - int bin(ViewType& keys, const int& i) const { - return int(mul_*(keys(i)-min_)); + typename KeyViewType::const_value_type max) + : max_bins_(max_bins__ + 1), + mul_(1.0 * max_bins__ / (max - min)), + range_(max - min), + min_(min) {} + + // Determine bin index from key value + template + KOKKOS_INLINE_FUNCTION int bin(ViewType& keys, const int& i) const { + return int(mul_ * (keys(i) - min_)); } - //Return maximum bin index + 1 + // Return maximum bin index + 1 KOKKOS_INLINE_FUNCTION - int max_bins() const { - return max_bins_; - } + int max_bins() const { return max_bins_; } - //Compare to keys within a bin if true new_val will be put before old_val - template - KOKKOS_INLINE_FUNCTION - bool operator()(ViewType& keys, iType1& i1, iType2& i2) const { - return keys(i1) + KOKKOS_INLINE_FUNCTION bool operator()(ViewType& keys, iType1& i1, + iType2& i2) const { + return keys(i1) < keys(i2); } }; -template +template struct BinOp3D { int max_bins_[3]; double mul_[3]; @@ -450,43 +449,42 @@ struct BinOp3D { BinOp3D() {} BinOp3D(int max_bins__[], typename KeyViewType::const_value_type min[], - typename KeyViewType::const_value_type max[] ) - { + typename KeyViewType::const_value_type max[]) { max_bins_[0] = max_bins__[0]; max_bins_[1] = max_bins__[1]; max_bins_[2] = max_bins__[2]; - mul_[0] = 1.0*max_bins__[0]/(max[0]-min[0]); - mul_[1] = 1.0*max_bins__[1]/(max[1]-min[1]); - mul_[2] = 1.0*max_bins__[2]/(max[2]-min[2]); - range_[0] = max[0]-min[0]; - range_[1] = max[1]-min[1]; - range_[2] = max[2]-min[2]; - min_[0] = min[0]; - min_[1] = min[1]; - min_[2] = min[2]; + mul_[0] = 1.0 * max_bins__[0] / (max[0] - min[0]); + mul_[1] = 1.0 * max_bins__[1] / (max[1] - min[1]); + mul_[2] = 1.0 * max_bins__[2] / (max[2] - min[2]); + range_[0] = max[0] - min[0]; + range_[1] = max[1] - min[1]; + range_[2] = max[2] - min[2]; + min_[0] = min[0]; + min_[1] = min[1]; + min_[2] = min[2]; } - template - KOKKOS_INLINE_FUNCTION - int bin(ViewType& keys, const int& i) const { - return int( (((int(mul_[0]*(keys(i,0)-min_[0]))*max_bins_[1]) + - int(mul_[1]*(keys(i,1)-min_[1])))*max_bins_[2]) + - int(mul_[2]*(keys(i,2)-min_[2]))); - } - - KOKKOS_INLINE_FUNCTION - int max_bins() const { - return max_bins_[0]*max_bins_[1]*max_bins_[2]; + template + KOKKOS_INLINE_FUNCTION int bin(ViewType& keys, const int& i) const { + return int((((int(mul_[0] * (keys(i, 0) - min_[0])) * max_bins_[1]) + + int(mul_[1] * (keys(i, 1) - min_[1]))) * + max_bins_[2]) + + int(mul_[2] * (keys(i, 2) - min_[2]))); } - template KOKKOS_INLINE_FUNCTION - bool operator()(ViewType& keys, iType1& i1 , iType2& i2) const { - if (keys(i1,0)>keys(i2,0)) return true; - else if (keys(i1,0)==keys(i2,0)) { - if (keys(i1,1)>keys(i2,1)) return true; - else if (keys(i1,1)==keys(i2,1)) { - if (keys(i1,2)>keys(i2,2)) return true; + int max_bins() const { return max_bins_[0] * max_bins_[1] * max_bins_[2]; } + + template + KOKKOS_INLINE_FUNCTION bool operator()(ViewType& keys, iType1& i1, + iType2& i2) const { + if (keys(i1, 0) > keys(i2, 0)) + return true; + else if (keys(i1, 0) == keys(i2, 0)) { + if (keys(i1, 1) > keys(i2, 1)) + return true; + else if (keys(i1, 1) == keys(i2, 1)) { + if (keys(i1, 2) > keys(i2, 2)) return true; } } return false; @@ -495,85 +493,80 @@ struct BinOp3D { namespace Impl { -template +template bool try_std_sort(ViewType view) { - bool possible = true; - size_t stride[8] = { view.stride_0() - , view.stride_1() - , view.stride_2() - , view.stride_3() - , view.stride_4() - , view.stride_5() - , view.stride_6() - , view.stride_7() - }; - possible = possible && std::is_same::value; - possible = possible && (ViewType::Rank == 1); - possible = possible && (stride[0] == 1); - if(possible) { - std::sort(view.data(),view.data()+view.extent(0)); + bool possible = true; + size_t stride[8] = {view.stride_0(), view.stride_1(), view.stride_2(), + view.stride_3(), view.stride_4(), view.stride_5(), + view.stride_6(), view.stride_7()}; + possible = possible && + std::is_same::value; + possible = possible && (ViewType::Rank == 1); + possible = possible && (stride[0] == 1); + if (possible) { + std::sort(view.data(), view.data() + view.extent(0)); } return possible; } -template +template struct min_max_functor { - typedef Kokkos::MinMaxScalar minmax_scalar; + typedef Kokkos::MinMaxScalar + minmax_scalar; ViewType view; - min_max_functor(const ViewType& view_):view(view_) {} + min_max_functor(const ViewType& view_) : view(view_) {} KOKKOS_INLINE_FUNCTION - void operator() (const size_t& i, minmax_scalar& minmax) const { - if(view(i) < minmax.min_val) minmax.min_val = view(i); - if(view(i) > minmax.max_val) minmax.max_val = view(i); + void operator()(const size_t& i, minmax_scalar& minmax) const { + if (view(i) < minmax.min_val) minmax.min_val = view(i); + if (view(i) > minmax.max_val) minmax.max_val = view(i); } }; -} +} // namespace Impl -template -void sort( ViewType const & view , bool const always_use_kokkos_sort = false) -{ - if(!always_use_kokkos_sort) { - if(Impl::try_std_sort(view)) return; +template +void sort(ViewType const& view, bool const always_use_kokkos_sort = false) { + if (!always_use_kokkos_sort) { + if (Impl::try_std_sort(view)) return; } typedef BinOp1D CompType; Kokkos::MinMaxScalar result; Kokkos::MinMax reducer(result); - parallel_reduce("Kokkos::Sort::FindExtent",Kokkos::RangePolicy(0,view.extent(0)), - Impl::min_max_functor(view),reducer); - if(result.min_val == result.max_val) return; - BinSort bin_sort(view,CompType(view.extent(0)/2,result.min_val,result.max_val),true); + parallel_reduce("Kokkos::Sort::FindExtent", + Kokkos::RangePolicy( + 0, view.extent(0)), + Impl::min_max_functor(view), reducer); + if (result.min_val == result.max_val) return; + BinSort bin_sort( + view, CompType(view.extent(0) / 2, result.min_val, result.max_val), true); bin_sort.create_permute_vector(); bin_sort.sort(view); } -template -void sort( ViewType view - , size_t const begin - , size_t const end - ) -{ - typedef Kokkos::RangePolicy range_policy ; +template +void sort(ViewType view, size_t const begin, size_t const end) { + typedef Kokkos::RangePolicy range_policy; typedef BinOp1D CompType; Kokkos::MinMaxScalar result; Kokkos::MinMax reducer(result); - parallel_reduce("Kokkos::Sort::FindExtent", range_policy( begin , end ) - , Impl::min_max_functor(view),reducer ); + parallel_reduce("Kokkos::Sort::FindExtent", range_policy(begin, end), + Impl::min_max_functor(view), reducer); - if(result.min_val == result.max_val) return; + if (result.min_val == result.max_val) return; - BinSort - bin_sort(view,begin,end,CompType((end-begin)/2,result.min_val,result.max_val),true); + BinSort bin_sort( + view, begin, end, + CompType((end - begin) / 2, result.min_val, result.max_val), true); bin_sort.create_permute_vector(); - bin_sort.sort(view,begin,end); + bin_sort.sort(view, begin, end); } -} +} // namespace Kokkos #endif diff --git a/lib/kokkos/algorithms/unit_tests/CMakeLists.txt b/lib/kokkos/algorithms/unit_tests/CMakeLists.txt index e238b37c8ec933aaec930d1896957e2739b97724..6fb08ce2edb8b8632969b00bd69fdb495ce73cf2 100644 --- a/lib/kokkos/algorithms/unit_tests/CMakeLists.txt +++ b/lib/kokkos/algorithms/unit_tests/CMakeLists.txt @@ -1,18 +1,12 @@ -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -INCLUDE_DIRECTORIES(REQUIRED_DURING_INSTALLATION_TESTING ${CMAKE_CURRENT_SOURCE_DIR}) -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../src ) +#Leave these here for now - I don't need transitive deps anyway +KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) +KOKKOS_INCLUDE_DIRECTORIES(REQUIRED_DURING_INSTALLATION_TESTING ${CMAKE_CURRENT_SOURCE_DIR}) +KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../src ) -IF(NOT KOKKOS_HAS_TRILINOS) - IF(KOKKOS_SEPARATE_LIBS) - set(TEST_LINK_TARGETS kokkoscore) - ELSE() - set(TEST_LINK_TARGETS kokkos) - ENDIF() -ENDIF() SET(GTEST_SOURCE_DIR ${${PARENT_PACKAGE_NAME}_SOURCE_DIR}/tpls/gtest) -INCLUDE_DIRECTORIES(${GTEST_SOURCE_DIR}) +KOKKOS_INCLUDE_DIRECTORIES(${GTEST_SOURCE_DIR}) # mfh 03 Nov 2017: The gtest library used here must have a different # name than that of the gtest library built in KokkosCore. We can't @@ -20,23 +14,20 @@ INCLUDE_DIRECTORIES(${GTEST_SOURCE_DIR}) # possible to build only (e.g.,) KokkosAlgorithms tests, without # building KokkosCore tests. -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DGTEST_HAS_PTHREAD=0") -TRIBITS_ADD_LIBRARY( +KOKKOS_ADD_TEST_LIBRARY( kokkosalgorithms_gtest HEADERS ${GTEST_SOURCE_DIR}/gtest/gtest.h SOURCES ${GTEST_SOURCE_DIR}/gtest/gtest-all.cc - TESTONLY - ) +) +KOKKOS_TARGET_COMPILE_DEFINITIONS(kokkosalgorithms_gtest PUBLIC "-DGTEST_HAS_PTHREAD=0") SET(SOURCES UnitTestMain.cpp TestCuda.cpp ) -SET(LIBRARIES kokkoscore) - -IF(Kokkos_ENABLE_OpenMP) +IF(Kokkos_ENABLE_OPENMP) LIST( APPEND SOURCES TestOpenMP.cpp ) @@ -48,23 +39,19 @@ IF(Kokkos_ENABLE_HPX) ) ENDIF() -IF(Kokkos_ENABLE_Serial) +IF(Kokkos_ENABLE_SERIAL) LIST( APPEND SOURCES TestSerial.cpp ) ENDIF() -IF(Kokkos_ENABLE_Pthread) +IF(Kokkos_ENABLE_PTHREAD) LIST( APPEND SOURCES TestThreads.cpp ) ENDIF() -TRIBITS_ADD_EXECUTABLE_AND_TEST( +KOKKOS_ADD_EXECUTABLE_AND_TEST( UnitTest SOURCES ${SOURCES} - COMM serial mpi - NUM_MPI_PROCS 1 - FAIL_REGULAR_EXPRESSION " FAILED " - TESTONLYLIBS kokkosalgorithms_gtest ${TEST_LINK_TARGETS} - ) +) diff --git a/lib/kokkos/algorithms/unit_tests/TestCuda.cpp b/lib/kokkos/algorithms/unit_tests/TestCuda.cpp index 86fdccd0e784e8bc56472782a87a31a050f468fe..ab727b0326dd2d456845c39d5e8b5173a667d6bc 100644 --- a/lib/kokkos/algorithms/unit_tests/TestCuda.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestCuda.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -57,51 +58,31 @@ namespace Test { -class cuda : public ::testing::Test { -protected: - static void SetUpTestCase() - { - } - static void TearDownTestCase() - { - } -}; - -void cuda_test_random_xorshift64( int num_draws ) -{ +void cuda_test_random_xorshift64(int num_draws) { Impl::test_random >(num_draws); } -void cuda_test_random_xorshift1024( int num_draws ) -{ +void cuda_test_random_xorshift1024(int num_draws) { Impl::test_random >(num_draws); } +#define CUDA_RANDOM_XORSHIFT64(num_draws) \ + TEST(cuda, Random_XorShift64) { cuda_test_random_xorshift64(num_draws); } -#define CUDA_RANDOM_XORSHIFT64( num_draws ) \ - TEST_F( cuda, Random_XorShift64 ) { \ - cuda_test_random_xorshift64(num_draws); \ - } - -#define CUDA_RANDOM_XORSHIFT1024( num_draws ) \ - TEST_F( cuda, Random_XorShift1024 ) { \ - cuda_test_random_xorshift1024(num_draws); \ - } +#define CUDA_RANDOM_XORSHIFT1024(num_draws) \ + TEST(cuda, Random_XorShift1024) { cuda_test_random_xorshift1024(num_draws); } -#define CUDA_SORT_UNSIGNED( size ) \ - TEST_F( cuda, SortUnsigned ) { \ - Impl::test_sort< Kokkos::Cuda, unsigned >(size); \ - } +#define CUDA_SORT_UNSIGNED(size) \ + TEST(cuda, SortUnsigned) { Impl::test_sort(size); } -CUDA_RANDOM_XORSHIFT64( 132141141 ) -CUDA_RANDOM_XORSHIFT1024( 52428813 ) +CUDA_RANDOM_XORSHIFT64(132141141) +CUDA_RANDOM_XORSHIFT1024(52428813) CUDA_SORT_UNSIGNED(171) #undef CUDA_RANDOM_XORSHIFT64 #undef CUDA_RANDOM_XORSHIFT1024 #undef CUDA_SORT_UNSIGNED -} +} // namespace Test #else void KOKKOS_ALGORITHMS_UNITTESTS_TESTCUDA_PREVENT_LINK_ERROR() {} -#endif /* #ifdef KOKKOS_ENABLE_CUDA */ - +#endif /* #ifdef KOKKOS_ENABLE_CUDA */ diff --git a/lib/kokkos/algorithms/unit_tests/TestHPX.cpp b/lib/kokkos/algorithms/unit_tests/TestHPX.cpp index e5b7dbdb7a32c5cee98b33d1169bf5f49e71d1cb..2981e97945cb45065452a8f5330b0b35a9f4c65c 100644 --- a/lib/kokkos/algorithms/unit_tests/TestHPX.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestHPX.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -41,7 +42,6 @@ //@HEADER */ - #include #ifdef KOKKOS_ENABLE_HPX @@ -55,42 +55,33 @@ namespace Test { -class hpx : public ::testing::Test { -protected: - static void SetUpTestCase() - { - std::cout << std::setprecision(5) << std::scientific; - } - - static void TearDownTestCase() - { +#define HPX_RANDOM_XORSHIFT64(num_draws) \ + TEST(hpx, Random_XorShift64) { \ + Impl::test_random< \ + Kokkos::Random_XorShift64_Pool >( \ + num_draws); \ } -}; -#define HPX_RANDOM_XORSHIFT64( num_draws ) \ - TEST_F( hpx, Random_XorShift64 ) { \ - Impl::test_random >(num_draws); \ +#define HPX_RANDOM_XORSHIFT1024(num_draws) \ + TEST(hpx, Random_XorShift1024) { \ + Impl::test_random< \ + Kokkos::Random_XorShift1024_Pool >( \ + num_draws); \ } -#define HPX_RANDOM_XORSHIFT1024( num_draws ) \ - TEST_F( hpx, Random_XorShift1024 ) { \ - Impl::test_random >(num_draws); \ +#define HPX_SORT_UNSIGNED(size) \ + TEST(hpx, SortUnsigned) { \ + Impl::test_sort(size); \ } -#define HPX_SORT_UNSIGNED( size ) \ - TEST_F( hpx, SortUnsigned ) { \ - Impl::test_sort< Kokkos::Experimental::HPX, unsigned >(size); \ - } - -HPX_RANDOM_XORSHIFT64( 10240000 ) -HPX_RANDOM_XORSHIFT1024( 10130144 ) +HPX_RANDOM_XORSHIFT64(10240000) +HPX_RANDOM_XORSHIFT1024(10130144) HPX_SORT_UNSIGNED(171) #undef HPX_RANDOM_XORSHIFT64 #undef HPX_RANDOM_XORSHIFT1024 #undef HPX_SORT_UNSIGNED -} // namespace test +} // namespace Test #else void KOKKOS_ALGORITHMS_UNITTESTS_TESTHPX_PREVENT_LINK_ERROR() {} #endif - diff --git a/lib/kokkos/algorithms/unit_tests/TestOpenMP.cpp b/lib/kokkos/algorithms/unit_tests/TestOpenMP.cpp index c4ddde7b7f71995bb25655cbbb5270b671db7f3e..3a9e306014b27a2c80b1c133b2f61873c2337325 100644 --- a/lib/kokkos/algorithms/unit_tests/TestOpenMP.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestOpenMP.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -41,7 +42,6 @@ //@HEADER */ - #include #ifdef KOKKOS_ENABLE_OPENMP @@ -55,42 +55,31 @@ namespace Test { -class openmp : public ::testing::Test { -protected: - static void SetUpTestCase() - { - std::cout << std::setprecision(5) << std::scientific; - } - - static void TearDownTestCase() - { +#define OPENMP_RANDOM_XORSHIFT64(num_draws) \ + TEST(openmp, Random_XorShift64) { \ + Impl::test_random >( \ + num_draws); \ } -}; -#define OPENMP_RANDOM_XORSHIFT64( num_draws ) \ - TEST_F( openmp, Random_XorShift64 ) { \ - Impl::test_random >(num_draws); \ +#define OPENMP_RANDOM_XORSHIFT1024(num_draws) \ + TEST(openmp, Random_XorShift1024) { \ + Impl::test_random >( \ + num_draws); \ } -#define OPENMP_RANDOM_XORSHIFT1024( num_draws ) \ - TEST_F( openmp, Random_XorShift1024 ) { \ - Impl::test_random >(num_draws); \ +#define OPENMP_SORT_UNSIGNED(size) \ + TEST(openmp, SortUnsigned) { \ + Impl::test_sort(size); \ } -#define OPENMP_SORT_UNSIGNED( size ) \ - TEST_F( openmp, SortUnsigned ) { \ - Impl::test_sort< Kokkos::OpenMP, unsigned >(size); \ - } - -OPENMP_RANDOM_XORSHIFT64( 10240000 ) -OPENMP_RANDOM_XORSHIFT1024( 10130144 ) +OPENMP_RANDOM_XORSHIFT64(10240000) +OPENMP_RANDOM_XORSHIFT1024(10130144) OPENMP_SORT_UNSIGNED(171) #undef OPENMP_RANDOM_XORSHIFT64 #undef OPENMP_RANDOM_XORSHIFT1024 #undef OPENMP_SORT_UNSIGNED -} // namespace test +} // namespace Test #else void KOKKOS_ALGORITHMS_UNITTESTS_TESTOPENMP_PREVENT_LINK_ERROR() {} #endif - diff --git a/lib/kokkos/algorithms/unit_tests/TestROCm.cpp b/lib/kokkos/algorithms/unit_tests/TestROCm.cpp index 15179509bbfc1fe1e193081c8387e237dfa2525c..29814cca3e6c9cc5512d83305e4a6810a781ac79 100644 --- a/lib/kokkos/algorithms/unit_tests/TestROCm.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestROCm.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -57,52 +58,35 @@ namespace Test { -class rocm : public ::testing::Test { -protected: - static void SetUpTestCase() - { - std::cout << std::setprecision(5) << std::scientific; - } - static void TearDownTestCase() - { - } -}; - -void rocm_test_random_xorshift64( int num_draws ) -{ - Impl::test_random >(num_draws); +void rocm_test_random_xorshift64(int num_draws) { + Impl::test_random< + Kokkos::Random_XorShift64_Pool >(num_draws); } -void rocm_test_random_xorshift1024( int num_draws ) -{ - Impl::test_random >(num_draws); +void rocm_test_random_xorshift1024(int num_draws) { + Impl::test_random< + Kokkos::Random_XorShift1024_Pool >(num_draws); } +#define ROCM_RANDOM_XORSHIFT64(num_draws) \ + TEST(rocm, Random_XorShift64) { rocm_test_random_xorshift64(num_draws); } -#define ROCM_RANDOM_XORSHIFT64( num_draws ) \ - TEST_F( rocm, Random_XorShift64 ) { \ - rocm_test_random_xorshift64(num_draws); \ - } - -#define ROCM_RANDOM_XORSHIFT1024( num_draws ) \ - TEST_F( rocm, Random_XorShift1024 ) { \ - rocm_test_random_xorshift1024(num_draws); \ - } +#define ROCM_RANDOM_XORSHIFT1024(num_draws) \ + TEST(rocm, Random_XorShift1024) { rocm_test_random_xorshift1024(num_draws); } -#define ROCM_SORT_UNSIGNED( size ) \ - TEST_F( rocm, SortUnsigned ) { \ - Impl::test_sort< Kokkos::Experimental::ROCm, unsigned >(size); \ +#define ROCM_SORT_UNSIGNED(size) \ + TEST(rocm, SortUnsigned) { \ + Impl::test_sort(size); \ } -ROCM_RANDOM_XORSHIFT64( 132141141 ) -ROCM_RANDOM_XORSHIFT1024( 52428813 ) +ROCM_RANDOM_XORSHIFT64(132141141) +ROCM_RANDOM_XORSHIFT1024(52428813) ROCM_SORT_UNSIGNED(171) #undef ROCM_RANDOM_XORSHIFT64 #undef ROCM_RANDOM_XORSHIFT1024 #undef ROCM_SORT_UNSIGNED -} +} // namespace Test #else void KOKKOS_ALGORITHMS_UNITTESTS_TESTROCM_PREVENT_LINK_ERROR() {} -#endif /* #ifdef KOKKOS_ENABLE_ROCM */ - +#endif /* #ifdef KOKKOS_ENABLE_ROCM */ diff --git a/lib/kokkos/algorithms/unit_tests/TestRandom.hpp b/lib/kokkos/algorithms/unit_tests/TestRandom.hpp index 73bd416f2aba49e311884096c31db295b41f1066..bc55ebfad355bff785ad92d115ac45cff698482f 100644 --- a/lib/kokkos/algorithms/unit_tests/TestRandom.hpp +++ b/lib/kokkos/algorithms/unit_tests/TestRandom.hpp @@ -1,10 +1,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -22,10 +23,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -54,18 +55,19 @@ namespace Test { -namespace Impl{ +namespace Impl { // This test runs the random number generators and uses some statistic tests to // check the 'goodness' of the random numbers: // (i) mean: the mean is expected to be 0.5*RAND_MAX // (ii) variance: the variance is 1/3*mean*mean // (iii) covariance: the covariance is 0 -// (iv) 1-tupledistr: the mean, variance and covariance of a 1D Histrogram of random numbers -// (v) 3-tupledistr: the mean, variance and covariance of a 3D Histrogram of random numbers +// (iv) 1-tupledistr: the mean, variance and covariance of a 1D Histrogram +// of random numbers (v) 3-tupledistr: the mean, variance and covariance of +// a 3D Histrogram of random numbers #define HIST_DIM3D 24 -#define HIST_DIM1D (HIST_DIM3D*HIST_DIM3D*HIST_DIM3D) +#define HIST_DIM1D (HIST_DIM3D * HIST_DIM3D * HIST_DIM3D) struct RandomProperties { uint64_t count; @@ -77,37 +79,37 @@ struct RandomProperties { KOKKOS_INLINE_FUNCTION RandomProperties() { - count = 0; - mean = 0.0; - variance = 0.0; + count = 0; + mean = 0.0; + variance = 0.0; covariance = 0.0; - min = 1e64; - max = -1e64; + min = 1e64; + max = -1e64; } KOKKOS_INLINE_FUNCTION RandomProperties& operator+=(const RandomProperties& add) { - count += add.count; - mean += add.mean; - variance += add.variance; + count += add.count; + mean += add.mean; + variance += add.variance; covariance += add.covariance; - min = add.minmax?add.max:max; + min = add.min < min ? add.min : min; + max = add.max > max ? add.max : max; return *this; } KOKKOS_INLINE_FUNCTION void operator+=(const volatile RandomProperties& add) volatile { - count += add.count; - mean += add.mean; - variance += add.variance; + count += add.count; + mean += add.mean; + variance += add.variance; covariance += add.covariance; - min = add.minmax?add.max:max; + min = add.min < min ? add.min : min; + max = add.max > max ? add.max : max; } }; -template +template struct test_random_functor { typedef typename GeneratorPool::generator_type rnd_type; @@ -123,38 +125,40 @@ struct test_random_functor { // implementations might violate this upper bound, due to rounding // error. Just in case, we leave an extra space at the end of each // dimension, in the View types below. - typedef Kokkos::View type_1d; + typedef Kokkos::View + type_1d; type_1d density_1d; - typedef Kokkos::View type_3d; + typedef Kokkos::View + type_3d; type_3d density_3d; - test_random_functor (GeneratorPool rand_pool_, type_1d d1d, type_3d d3d) : - rand_pool (rand_pool_), - mean (0.5*Kokkos::rand::max ()), - density_1d (d1d), - density_3d (d3d) - {} + test_random_functor(GeneratorPool rand_pool_, type_1d d1d, type_3d d3d) + : rand_pool(rand_pool_), + mean(0.5 * Kokkos::rand::max()), + density_1d(d1d), + density_3d(d3d) {} KOKKOS_INLINE_FUNCTION - void operator() (int i, RandomProperties& prop) const { + void operator()(int i, RandomProperties& prop) const { using Kokkos::atomic_fetch_add; rnd_type rand_gen = rand_pool.get_state(); for (int k = 0; k < 1024; ++k) { - const Scalar tmp = Kokkos::rand::draw(rand_gen); + const Scalar tmp = Kokkos::rand::draw(rand_gen); prop.count++; prop.mean += tmp; - prop.variance += (tmp-mean)*(tmp-mean); - const Scalar tmp2 = Kokkos::rand::draw(rand_gen); + prop.variance += (tmp - mean) * (tmp - mean); + const Scalar tmp2 = Kokkos::rand::draw(rand_gen); prop.count++; prop.mean += tmp2; - prop.variance += (tmp2-mean)*(tmp2-mean); - prop.covariance += (tmp-mean)*(tmp2-mean); - const Scalar tmp3 = Kokkos::rand::draw(rand_gen); + prop.variance += (tmp2 - mean) * (tmp2 - mean); + prop.covariance += (tmp - mean) * (tmp2 - mean); + const Scalar tmp3 = Kokkos::rand::draw(rand_gen); prop.count++; prop.mean += tmp3; - prop.variance += (tmp3-mean)*(tmp3-mean); - prop.covariance += (tmp2-mean)*(tmp3-mean); + prop.variance += (tmp3 - mean) * (tmp3 - mean); + prop.covariance += (tmp2 - mean) * (tmp3 - mean); // NOTE (mfh 03 Nov 2014): Kokkos::rand::max() is supposed to // define an exclusive upper bound on the range of random @@ -169,26 +173,32 @@ struct test_random_functor { // returns values of max(), the histograms will still catch this // indirectly, since none of the other values will be filled in. - const Scalar theMax = Kokkos::rand::max (); - - const uint64_t ind1_1d = static_cast (1.0 * HIST_DIM1D * tmp / theMax); - const uint64_t ind2_1d = static_cast (1.0 * HIST_DIM1D * tmp2 / theMax); - const uint64_t ind3_1d = static_cast (1.0 * HIST_DIM1D * tmp3 / theMax); - - const uint64_t ind1_3d = static_cast (1.0 * HIST_DIM3D * tmp / theMax); - const uint64_t ind2_3d = static_cast (1.0 * HIST_DIM3D * tmp2 / theMax); - const uint64_t ind3_3d = static_cast (1.0 * HIST_DIM3D * tmp3 / theMax); - - atomic_fetch_add (&density_1d(ind1_1d), 1); - atomic_fetch_add (&density_1d(ind2_1d), 1); - atomic_fetch_add (&density_1d(ind3_1d), 1); - atomic_fetch_add (&density_3d(ind1_3d, ind2_3d, ind3_3d), 1); + const Scalar theMax = Kokkos::rand::max(); + + const uint64_t ind1_1d = + static_cast(1.0 * HIST_DIM1D * tmp / theMax); + const uint64_t ind2_1d = + static_cast(1.0 * HIST_DIM1D * tmp2 / theMax); + const uint64_t ind3_1d = + static_cast(1.0 * HIST_DIM1D * tmp3 / theMax); + + const uint64_t ind1_3d = + static_cast(1.0 * HIST_DIM3D * tmp / theMax); + const uint64_t ind2_3d = + static_cast(1.0 * HIST_DIM3D * tmp2 / theMax); + const uint64_t ind3_3d = + static_cast(1.0 * HIST_DIM3D * tmp3 / theMax); + + atomic_fetch_add(&density_1d(ind1_1d), 1); + atomic_fetch_add(&density_1d(ind2_1d), 1); + atomic_fetch_add(&density_1d(ind3_1d), 1); + atomic_fetch_add(&density_3d(ind1_3d, ind2_3d, ind3_3d), 1); } rand_pool.free_state(rand_gen); } }; -template +template struct test_histogram1d_functor { typedef RandomProperties value_type; typedef typename DeviceType::execution_space execution_space; @@ -200,34 +210,29 @@ struct test_histogram1d_functor { // implementations might violate this upper bound, due to rounding // error. Just in case, we leave an extra space at the end of each // dimension, in the View type below. - typedef Kokkos::View type_1d; + typedef Kokkos::View type_1d; type_1d density_1d; double mean; - test_histogram1d_functor (type_1d d1d, int num_draws) : - density_1d (d1d), - mean (1.0*num_draws/HIST_DIM1D*3) - { - } + test_histogram1d_functor(type_1d d1d, int num_draws) + : density_1d(d1d), mean(1.0 * num_draws / HIST_DIM1D * 3) {} - KOKKOS_INLINE_FUNCTION void - operator() (const typename memory_space::size_type i, - RandomProperties& prop) const - { + KOKKOS_INLINE_FUNCTION void operator()( + const typename memory_space::size_type i, RandomProperties& prop) const { typedef typename memory_space::size_type size_type; const double count = density_1d(i); prop.mean += count; prop.variance += 1.0 * (count - mean) * (count - mean); - //prop.covariance += 1.0*count*count; + // prop.covariance += 1.0*count*count; prop.min = count < prop.min ? count : prop.min; prop.max = count > prop.max ? count : prop.max; - if (i < static_cast (HIST_DIM1D-1)) { - prop.covariance += (count - mean) * (density_1d(i+1) - mean); + if (i < static_cast(HIST_DIM1D - 1)) { + prop.covariance += (count - mean) * (density_1d(i + 1) - mean); } } }; -template +template struct test_histogram3d_functor { typedef RandomProperties value_type; typedef typename DeviceType::execution_space execution_space; @@ -239,29 +244,28 @@ struct test_histogram3d_functor { // implementations might violate this upper bound, due to rounding // error. Just in case, we leave an extra space at the end of each // dimension, in the View type below. - typedef Kokkos::View type_3d; + typedef Kokkos::View + type_3d; type_3d density_3d; double mean; - test_histogram3d_functor (type_3d d3d, int num_draws) : - density_3d (d3d), - mean (1.0*num_draws/HIST_DIM1D) - {} + test_histogram3d_functor(type_3d d3d, int num_draws) + : density_3d(d3d), mean(1.0 * num_draws / HIST_DIM1D) {} - KOKKOS_INLINE_FUNCTION void - operator() (const typename memory_space::size_type i, - RandomProperties& prop) const - { + KOKKOS_INLINE_FUNCTION void operator()( + const typename memory_space::size_type i, RandomProperties& prop) const { typedef typename memory_space::size_type size_type; - const double count = density_3d(i/(HIST_DIM3D*HIST_DIM3D), - (i % (HIST_DIM3D*HIST_DIM3D))/HIST_DIM3D, - i % HIST_DIM3D); + const double count = density_3d( + i / (HIST_DIM3D * HIST_DIM3D), + (i % (HIST_DIM3D * HIST_DIM3D)) / HIST_DIM3D, i % HIST_DIM3D); prop.mean += count; prop.variance += (count - mean) * (count - mean); - if (i < static_cast (HIST_DIM1D-1)) { - const double count_next = density_3d((i+1)/(HIST_DIM3D*HIST_DIM3D), - ((i+1)%(HIST_DIM3D*HIST_DIM3D))/HIST_DIM3D, - (i+1)%HIST_DIM3D); + if (i < static_cast(HIST_DIM1D - 1)) { + const double count_next = + density_3d((i + 1) / (HIST_DIM3D * HIST_DIM3D), + ((i + 1) % (HIST_DIM3D * HIST_DIM3D)) / HIST_DIM3D, + (i + 1) % HIST_DIM3D); prop.covariance += (count - mean) * (count_next - mean); } } @@ -270,212 +274,223 @@ struct test_histogram3d_functor { // // Templated test that uses the above functors. // -template +template struct test_random_scalar { typedef typename RandomGenerator::generator_type rnd_type; - int pass_mean,pass_var,pass_covar; - int pass_hist1d_mean,pass_hist1d_var,pass_hist1d_covar; - int pass_hist3d_mean,pass_hist3d_var,pass_hist3d_covar; + int pass_mean, pass_var, pass_covar; + int pass_hist1d_mean, pass_hist1d_var, pass_hist1d_covar; + int pass_hist3d_mean, pass_hist3d_var, pass_hist3d_covar; - test_random_scalar (typename test_random_functor::type_1d& density_1d, - typename test_random_functor::type_3d& density_3d, - RandomGenerator& pool, - unsigned int num_draws) - { + test_random_scalar( + typename test_random_functor::type_1d& density_1d, + typename test_random_functor::type_3d& density_3d, + RandomGenerator& pool, unsigned int num_draws) { + using Kokkos::parallel_reduce; using std::cout; using std::endl; - using Kokkos::parallel_reduce; { cout << " -- Testing randomness properties" << endl; RandomProperties result; typedef test_random_functor functor_type; - parallel_reduce (num_draws/1024, functor_type (pool, density_1d, density_3d), result); - - //printf("Result: %lf %lf %lf\n",result.mean/num_draws/3,result.variance/num_draws/3,result.covariance/num_draws/2); - double tolerance = 1.6*std::sqrt(1.0/num_draws); - double mean_expect = 0.5*Kokkos::rand::max(); - double variance_expect = 1.0/3.0*mean_expect*mean_expect; - double mean_eps = mean_expect/(result.mean/num_draws/3)-1.0; - double variance_eps = variance_expect/(result.variance/num_draws/3)-1.0; - double covariance_eps = result.covariance/num_draws/2/variance_expect; - pass_mean = ((-tolerance < mean_eps) && - ( tolerance > mean_eps)) ? 1:0; - pass_var = ((-1.5*tolerance < variance_eps) && - ( 1.5*tolerance > variance_eps)) ? 1:0; - pass_covar = ((-2.0*tolerance < covariance_eps) && - ( 2.0*tolerance > covariance_eps)) ? 1:0; - cout << "Pass: " << pass_mean - << " " << pass_var - << " " << mean_eps - << " " << variance_eps - << " " << covariance_eps - << " || " << tolerance << endl; + parallel_reduce(num_draws / 1024, + functor_type(pool, density_1d, density_3d), result); + + // printf("Result: %lf %lf + // %lf\n",result.mean/num_draws/3,result.variance/num_draws/3,result.covariance/num_draws/2); + double tolerance = 1.6 * std::sqrt(1.0 / num_draws); + double mean_expect = 0.5 * Kokkos::rand::max(); + double variance_expect = 1.0 / 3.0 * mean_expect * mean_expect; + double mean_eps = mean_expect / (result.mean / num_draws / 3) - 1.0; + double variance_eps = + variance_expect / (result.variance / num_draws / 3) - 1.0; + double covariance_eps = + result.covariance / num_draws / 2 / variance_expect; + pass_mean = ((-tolerance < mean_eps) && (tolerance > mean_eps)) ? 1 : 0; + pass_var = ((-1.5 * tolerance < variance_eps) && + (1.5 * tolerance > variance_eps)) + ? 1 + : 0; + pass_covar = ((-2.0 * tolerance < covariance_eps) && + (2.0 * tolerance > covariance_eps)) + ? 1 + : 0; + cout << "Pass: " << pass_mean << " " << pass_var << " " << mean_eps << " " + << variance_eps << " " << covariance_eps << " || " << tolerance + << endl; } { cout << " -- Testing 1-D histogram" << endl; RandomProperties result; - typedef test_histogram1d_functor functor_type; - parallel_reduce (HIST_DIM1D, functor_type (density_1d, num_draws), result); - - double tolerance = 6*std::sqrt(1.0/HIST_DIM1D); - double mean_expect = 1.0*num_draws*3/HIST_DIM1D; - double variance_expect = 1.0*num_draws*3/HIST_DIM1D*(1.0-1.0/HIST_DIM1D); - double covariance_expect = -1.0*num_draws*3/HIST_DIM1D/HIST_DIM1D; - double mean_eps = mean_expect/(result.mean/HIST_DIM1D)-1.0; - double variance_eps = variance_expect/(result.variance/HIST_DIM1D)-1.0; - double covariance_eps = (result.covariance/HIST_DIM1D - covariance_expect)/mean_expect; - pass_hist1d_mean = ((-0.0001 < mean_eps) && - ( 0.0001 > mean_eps)) ? 1:0; - pass_hist1d_var = ((-0.07 < variance_eps) && - ( 0.07 > variance_eps)) ? 1:0; - pass_hist1d_covar = ((-0.06 < covariance_eps) && - ( 0.06 > covariance_eps)) ? 1:0; - - cout << "Density 1D: " << mean_eps - << " " << variance_eps - << " " << (result.covariance/HIST_DIM1D/HIST_DIM1D) - << " || " << tolerance - << " " << result.min - << " " << result.max - << " || " << result.variance/HIST_DIM1D - << " " << 1.0*num_draws*3/HIST_DIM1D*(1.0-1.0/HIST_DIM1D) - << " || " << result.covariance/HIST_DIM1D - << " " << -1.0*num_draws*3/HIST_DIM1D/HIST_DIM1D - << endl; + typedef test_histogram1d_functor + functor_type; + parallel_reduce(HIST_DIM1D, functor_type(density_1d, num_draws), result); + + double tolerance = 6 * std::sqrt(1.0 / HIST_DIM1D); + double mean_expect = 1.0 * num_draws * 3 / HIST_DIM1D; + double variance_expect = + 1.0 * num_draws * 3 / HIST_DIM1D * (1.0 - 1.0 / HIST_DIM1D); + double covariance_expect = -1.0 * num_draws * 3 / HIST_DIM1D / HIST_DIM1D; + double mean_eps = mean_expect / (result.mean / HIST_DIM1D) - 1.0; + double variance_eps = + variance_expect / (result.variance / HIST_DIM1D) - 1.0; + double covariance_eps = + (result.covariance / HIST_DIM1D - covariance_expect) / mean_expect; + pass_hist1d_mean = ((-0.0001 < mean_eps) && (0.0001 > mean_eps)) ? 1 : 0; + pass_hist1d_var = + ((-0.07 < variance_eps) && (0.07 > variance_eps)) ? 1 : 0; + pass_hist1d_covar = + ((-0.06 < covariance_eps) && (0.06 > covariance_eps)) ? 1 : 0; + + cout << "Density 1D: " << mean_eps << " " << variance_eps << " " + << (result.covariance / HIST_DIM1D / HIST_DIM1D) << " || " + << tolerance << " " << result.min << " " << result.max << " || " + << result.variance / HIST_DIM1D << " " + << 1.0 * num_draws * 3 / HIST_DIM1D * (1.0 - 1.0 / HIST_DIM1D) + << " || " << result.covariance / HIST_DIM1D << " " + << -1.0 * num_draws * 3 / HIST_DIM1D / HIST_DIM1D << endl; } { cout << " -- Testing 3-D histogram" << endl; RandomProperties result; - typedef test_histogram3d_functor functor_type; - parallel_reduce (HIST_DIM1D, functor_type (density_3d, num_draws), result); - - double tolerance = 6*std::sqrt(1.0/HIST_DIM1D); - double mean_expect = 1.0*num_draws/HIST_DIM1D; - double variance_expect = 1.0*num_draws/HIST_DIM1D*(1.0-1.0/HIST_DIM1D); - double covariance_expect = -1.0*num_draws/HIST_DIM1D/HIST_DIM1D; - double mean_eps = mean_expect/(result.mean/HIST_DIM1D)-1.0; - double variance_eps = variance_expect/(result.variance/HIST_DIM1D)-1.0; - double covariance_eps = (result.covariance/HIST_DIM1D - covariance_expect)/mean_expect; - pass_hist3d_mean = ((-tolerance < mean_eps) && - ( tolerance > mean_eps)) ? 1:0; - pass_hist3d_var = ((-1.2*tolerance < variance_eps) && - ( 1.2*tolerance > variance_eps)) ? 1:0; - pass_hist3d_covar = ((-tolerance < covariance_eps) && - ( tolerance > covariance_eps)) ? 1:0; - - cout << "Density 3D: " << mean_eps - << " " << variance_eps - << " " << result.covariance/HIST_DIM1D/HIST_DIM1D - << " || " << tolerance - << " " << result.min - << " " << result.max << endl; + typedef test_histogram3d_functor + functor_type; + parallel_reduce(HIST_DIM1D, functor_type(density_3d, num_draws), result); + + double tolerance = 6 * std::sqrt(1.0 / HIST_DIM1D); + double mean_expect = 1.0 * num_draws / HIST_DIM1D; + double variance_expect = + 1.0 * num_draws / HIST_DIM1D * (1.0 - 1.0 / HIST_DIM1D); + double covariance_expect = -1.0 * num_draws / HIST_DIM1D / HIST_DIM1D; + double mean_eps = mean_expect / (result.mean / HIST_DIM1D) - 1.0; + double variance_eps = + variance_expect / (result.variance / HIST_DIM1D) - 1.0; + double covariance_eps = + (result.covariance / HIST_DIM1D - covariance_expect) / mean_expect; + pass_hist3d_mean = + ((-tolerance < mean_eps) && (tolerance > mean_eps)) ? 1 : 0; + pass_hist3d_var = ((-1.2 * tolerance < variance_eps) && + (1.2 * tolerance > variance_eps)) + ? 1 + : 0; + pass_hist3d_covar = + ((-tolerance < covariance_eps) && (tolerance > covariance_eps)) ? 1 + : 0; + + cout << "Density 3D: " << mean_eps << " " << variance_eps << " " + << result.covariance / HIST_DIM1D / HIST_DIM1D << " || " << tolerance + << " " << result.min << " " << result.max << endl; } } }; template -void test_random(unsigned int num_draws) -{ +void test_random(unsigned int num_draws) { using std::cout; using std::endl; - typename test_random_functor::type_1d density_1d("D1d"); - typename test_random_functor::type_3d density_3d("D3d"); + typename test_random_functor::type_1d density_1d("D1d"); + typename test_random_functor::type_3d density_3d("D3d"); - - uint64_t ticks = std::chrono::high_resolution_clock::now().time_since_epoch().count(); + uint64_t ticks = + std::chrono::high_resolution_clock::now().time_since_epoch().count(); cout << "Test Seed:" << ticks << endl; RandomGenerator pool(ticks); cout << "Test Scalar=int" << endl; - test_random_scalar test_int(density_1d,density_3d,pool,num_draws); - ASSERT_EQ( test_int.pass_mean,1); - ASSERT_EQ( test_int.pass_var,1); - ASSERT_EQ( test_int.pass_covar,1); - ASSERT_EQ( test_int.pass_hist1d_mean,1); - ASSERT_EQ( test_int.pass_hist1d_var,1); - ASSERT_EQ( test_int.pass_hist1d_covar,1); - ASSERT_EQ( test_int.pass_hist3d_mean,1); - ASSERT_EQ( test_int.pass_hist3d_var,1); - ASSERT_EQ( test_int.pass_hist3d_covar,1); - deep_copy(density_1d,0); - deep_copy(density_3d,0); + test_random_scalar test_int(density_1d, density_3d, + pool, num_draws); + ASSERT_EQ(test_int.pass_mean, 1); + ASSERT_EQ(test_int.pass_var, 1); + ASSERT_EQ(test_int.pass_covar, 1); + ASSERT_EQ(test_int.pass_hist1d_mean, 1); + ASSERT_EQ(test_int.pass_hist1d_var, 1); + ASSERT_EQ(test_int.pass_hist1d_covar, 1); + ASSERT_EQ(test_int.pass_hist3d_mean, 1); + ASSERT_EQ(test_int.pass_hist3d_var, 1); + ASSERT_EQ(test_int.pass_hist3d_covar, 1); + deep_copy(density_1d, 0); + deep_copy(density_3d, 0); cout << "Test Scalar=unsigned int" << endl; - test_random_scalar test_uint(density_1d,density_3d,pool,num_draws); - ASSERT_EQ( test_uint.pass_mean,1); - ASSERT_EQ( test_uint.pass_var,1); - ASSERT_EQ( test_uint.pass_covar,1); - ASSERT_EQ( test_uint.pass_hist1d_mean,1); - ASSERT_EQ( test_uint.pass_hist1d_var,1); - ASSERT_EQ( test_uint.pass_hist1d_covar,1); - ASSERT_EQ( test_uint.pass_hist3d_mean,1); - ASSERT_EQ( test_uint.pass_hist3d_var,1); - ASSERT_EQ( test_uint.pass_hist3d_covar,1); - deep_copy(density_1d,0); - deep_copy(density_3d,0); + test_random_scalar test_uint( + density_1d, density_3d, pool, num_draws); + ASSERT_EQ(test_uint.pass_mean, 1); + ASSERT_EQ(test_uint.pass_var, 1); + ASSERT_EQ(test_uint.pass_covar, 1); + ASSERT_EQ(test_uint.pass_hist1d_mean, 1); + ASSERT_EQ(test_uint.pass_hist1d_var, 1); + ASSERT_EQ(test_uint.pass_hist1d_covar, 1); + ASSERT_EQ(test_uint.pass_hist3d_mean, 1); + ASSERT_EQ(test_uint.pass_hist3d_var, 1); + ASSERT_EQ(test_uint.pass_hist3d_covar, 1); + deep_copy(density_1d, 0); + deep_copy(density_3d, 0); cout << "Test Scalar=int64_t" << endl; - test_random_scalar test_int64(density_1d,density_3d,pool,num_draws); - ASSERT_EQ( test_int64.pass_mean,1); - ASSERT_EQ( test_int64.pass_var,1); - ASSERT_EQ( test_int64.pass_covar,1); - ASSERT_EQ( test_int64.pass_hist1d_mean,1); - ASSERT_EQ( test_int64.pass_hist1d_var,1); - ASSERT_EQ( test_int64.pass_hist1d_covar,1); - ASSERT_EQ( test_int64.pass_hist3d_mean,1); - ASSERT_EQ( test_int64.pass_hist3d_var,1); - ASSERT_EQ( test_int64.pass_hist3d_covar,1); - deep_copy(density_1d,0); - deep_copy(density_3d,0); + test_random_scalar test_int64( + density_1d, density_3d, pool, num_draws); + ASSERT_EQ(test_int64.pass_mean, 1); + ASSERT_EQ(test_int64.pass_var, 1); + ASSERT_EQ(test_int64.pass_covar, 1); + ASSERT_EQ(test_int64.pass_hist1d_mean, 1); + ASSERT_EQ(test_int64.pass_hist1d_var, 1); + ASSERT_EQ(test_int64.pass_hist1d_covar, 1); + ASSERT_EQ(test_int64.pass_hist3d_mean, 1); + ASSERT_EQ(test_int64.pass_hist3d_var, 1); + ASSERT_EQ(test_int64.pass_hist3d_covar, 1); + deep_copy(density_1d, 0); + deep_copy(density_3d, 0); cout << "Test Scalar=uint64_t" << endl; - test_random_scalar test_uint64(density_1d,density_3d,pool,num_draws); - ASSERT_EQ( test_uint64.pass_mean,1); - ASSERT_EQ( test_uint64.pass_var,1); - ASSERT_EQ( test_uint64.pass_covar,1); - ASSERT_EQ( test_uint64.pass_hist1d_mean,1); - ASSERT_EQ( test_uint64.pass_hist1d_var,1); - ASSERT_EQ( test_uint64.pass_hist1d_covar,1); - ASSERT_EQ( test_uint64.pass_hist3d_mean,1); - ASSERT_EQ( test_uint64.pass_hist3d_var,1); - ASSERT_EQ( test_uint64.pass_hist3d_covar,1); - deep_copy(density_1d,0); - deep_copy(density_3d,0); + test_random_scalar test_uint64( + density_1d, density_3d, pool, num_draws); + ASSERT_EQ(test_uint64.pass_mean, 1); + ASSERT_EQ(test_uint64.pass_var, 1); + ASSERT_EQ(test_uint64.pass_covar, 1); + ASSERT_EQ(test_uint64.pass_hist1d_mean, 1); + ASSERT_EQ(test_uint64.pass_hist1d_var, 1); + ASSERT_EQ(test_uint64.pass_hist1d_covar, 1); + ASSERT_EQ(test_uint64.pass_hist3d_mean, 1); + ASSERT_EQ(test_uint64.pass_hist3d_var, 1); + ASSERT_EQ(test_uint64.pass_hist3d_covar, 1); + deep_copy(density_1d, 0); + deep_copy(density_3d, 0); cout << "Test Scalar=float" << endl; - test_random_scalar test_float(density_1d,density_3d,pool,num_draws); - ASSERT_EQ( test_float.pass_mean,1); - ASSERT_EQ( test_float.pass_var,1); - ASSERT_EQ( test_float.pass_covar,1); - ASSERT_EQ( test_float.pass_hist1d_mean,1); - ASSERT_EQ( test_float.pass_hist1d_var,1); - ASSERT_EQ( test_float.pass_hist1d_covar,1); - ASSERT_EQ( test_float.pass_hist3d_mean,1); - ASSERT_EQ( test_float.pass_hist3d_var,1); - ASSERT_EQ( test_float.pass_hist3d_covar,1); - deep_copy(density_1d,0); - deep_copy(density_3d,0); + test_random_scalar test_float(density_1d, density_3d, + pool, num_draws); + ASSERT_EQ(test_float.pass_mean, 1); + ASSERT_EQ(test_float.pass_var, 1); + ASSERT_EQ(test_float.pass_covar, 1); + ASSERT_EQ(test_float.pass_hist1d_mean, 1); + ASSERT_EQ(test_float.pass_hist1d_var, 1); + ASSERT_EQ(test_float.pass_hist1d_covar, 1); + ASSERT_EQ(test_float.pass_hist3d_mean, 1); + ASSERT_EQ(test_float.pass_hist3d_var, 1); + ASSERT_EQ(test_float.pass_hist3d_covar, 1); + deep_copy(density_1d, 0); + deep_copy(density_3d, 0); cout << "Test Scalar=double" << endl; - test_random_scalar test_double(density_1d,density_3d,pool,num_draws); - ASSERT_EQ( test_double.pass_mean,1); - ASSERT_EQ( test_double.pass_var,1); - ASSERT_EQ( test_double.pass_covar,1); - ASSERT_EQ( test_double.pass_hist1d_mean,1); - ASSERT_EQ( test_double.pass_hist1d_var,1); - ASSERT_EQ( test_double.pass_hist1d_covar,1); - ASSERT_EQ( test_double.pass_hist3d_mean,1); - ASSERT_EQ( test_double.pass_hist3d_var,1); - ASSERT_EQ( test_double.pass_hist3d_covar,1); -} + test_random_scalar test_double( + density_1d, density_3d, pool, num_draws); + ASSERT_EQ(test_double.pass_mean, 1); + ASSERT_EQ(test_double.pass_var, 1); + ASSERT_EQ(test_double.pass_covar, 1); + ASSERT_EQ(test_double.pass_hist1d_mean, 1); + ASSERT_EQ(test_double.pass_hist1d_var, 1); + ASSERT_EQ(test_double.pass_hist1d_covar, 1); + ASSERT_EQ(test_double.pass_hist3d_mean, 1); + ASSERT_EQ(test_double.pass_hist3d_var, 1); + ASSERT_EQ(test_double.pass_hist3d_covar, 1); } +} // namespace Impl -} // namespace Test +} // namespace Test -#endif //KOKKOS_TEST_UNORDERED_MAP_HPP +#endif // KOKKOS_TEST_UNORDERED_MAP_HPP diff --git a/lib/kokkos/algorithms/unit_tests/TestSerial.cpp b/lib/kokkos/algorithms/unit_tests/TestSerial.cpp index 9cf998f7732628bf71e8bc8ea90fcf6e41a27f3d..2eacdc2677184988c226bbaca13827a4b55cccff 100644 --- a/lib/kokkos/algorithms/unit_tests/TestSerial.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestSerial.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -52,49 +53,36 @@ #include #include - //---------------------------------------------------------------------------- - namespace Test { -class serial : public ::testing::Test { -protected: - static void SetUpTestCase() - { - } - - static void TearDownTestCase () - { +#define SERIAL_RANDOM_XORSHIFT64(num_draws) \ + TEST(serial, Random_XorShift64) { \ + Impl::test_random >( \ + num_draws); \ } -}; -#define SERIAL_RANDOM_XORSHIFT64( num_draws ) \ - TEST_F( serial, Random_XorShift64 ) { \ - Impl::test_random >(num_draws); \ +#define SERIAL_RANDOM_XORSHIFT1024(num_draws) \ + TEST(serial, Random_XorShift1024) { \ + Impl::test_random >( \ + num_draws); \ } -#define SERIAL_RANDOM_XORSHIFT1024( num_draws ) \ - TEST_F( serial, Random_XorShift1024 ) { \ - Impl::test_random >(num_draws); \ +#define SERIAL_SORT_UNSIGNED(size) \ + TEST(serial, SortUnsigned) { \ + Impl::test_sort(size); \ } -#define SERIAL_SORT_UNSIGNED( size ) \ - TEST_F( serial, SortUnsigned ) { \ - Impl::test_sort< Kokkos::Serial, unsigned >(size); \ - } - -SERIAL_RANDOM_XORSHIFT64( 10240000 ) -SERIAL_RANDOM_XORSHIFT1024( 10130144 ) +SERIAL_RANDOM_XORSHIFT64(10240000) +SERIAL_RANDOM_XORSHIFT1024(10130144) SERIAL_SORT_UNSIGNED(171) #undef SERIAL_RANDOM_XORSHIFT64 #undef SERIAL_RANDOM_XORSHIFT1024 #undef SERIAL_SORT_UNSIGNED -} // namespace Test +} // namespace Test #else void KOKKOS_ALGORITHMS_UNITTESTS_TESTSERIAL_PREVENT_LINK_ERROR() {} -#endif // KOKKOS_ENABLE_SERIAL - - +#endif // KOKKOS_ENABLE_SERIAL diff --git a/lib/kokkos/algorithms/unit_tests/TestSort.hpp b/lib/kokkos/algorithms/unit_tests/TestSort.hpp index 5fd7f09b50ef617b797f39305a3318e71ea354da..310a93c93d1d55baad4d33d9d34540ca523c9433 100644 --- a/lib/kokkos/algorithms/unit_tests/TestSort.hpp +++ b/lib/kokkos/algorithms/unit_tests/TestSort.hpp @@ -1,10 +1,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -22,10 +23,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -43,235 +44,248 @@ #define KOKKOS_ALGORITHMS_UNITTESTS_TESTSORT_HPP #include -#include -#include -#include -#include +#include +#include +#include +#include namespace Test { -namespace Impl{ +namespace Impl { -template +template struct is_sorted_struct { typedef unsigned int value_type; typedef ExecutionSpace execution_space; - Kokkos::View keys; + Kokkos::View keys; - is_sorted_struct(Kokkos::View keys_):keys(keys_) {} + is_sorted_struct(Kokkos::View keys_) : keys(keys_) {} KOKKOS_INLINE_FUNCTION - void operator() (int i, unsigned int& count) const { - if(keys(i)>keys(i+1)) count++; + void operator()(int i, unsigned int& count) const { + if (keys(i) > keys(i + 1)) count++; } }; -template +template struct sum { typedef double value_type; typedef ExecutionSpace execution_space; - Kokkos::View keys; + Kokkos::View keys; - sum(Kokkos::View keys_):keys(keys_) {} + sum(Kokkos::View keys_) : keys(keys_) {} KOKKOS_INLINE_FUNCTION - void operator() (int i, double& count) const { - count+=keys(i); - } + void operator()(int i, double& count) const { count += keys(i); } }; -template +template struct bin3d_is_sorted_struct { typedef unsigned int value_type; typedef ExecutionSpace execution_space; - Kokkos::View keys; + Kokkos::View keys; int max_bins; Scalar min; Scalar max; - bin3d_is_sorted_struct(Kokkos::View keys_,int max_bins_,Scalar min_,Scalar max_): - keys(keys_),max_bins(max_bins_),min(min_),max(max_) { - } + bin3d_is_sorted_struct(Kokkos::View keys_, + int max_bins_, Scalar min_, Scalar max_) + : keys(keys_), max_bins(max_bins_), min(min_), max(max_) {} KOKKOS_INLINE_FUNCTION - void operator() (int i, unsigned int& count) const { - int ix1 = int ((keys(i,0)-min)/max * max_bins); - int iy1 = int ((keys(i,1)-min)/max * max_bins); - int iz1 = int ((keys(i,2)-min)/max * max_bins); - int ix2 = int ((keys(i+1,0)-min)/max * max_bins); - int iy2 = int ((keys(i+1,1)-min)/max * max_bins); - int iz2 = int ((keys(i+1,2)-min)/max * max_bins); - - if (ix1>ix2) count++; - else if(ix1==ix2) { - if (iy1>iy2) count++; - else if ((iy1==iy2) && (iz1>iz2)) count++; + void operator()(int i, unsigned int& count) const { + int ix1 = int((keys(i, 0) - min) / max * max_bins); + int iy1 = int((keys(i, 1) - min) / max * max_bins); + int iz1 = int((keys(i, 2) - min) / max * max_bins); + int ix2 = int((keys(i + 1, 0) - min) / max * max_bins); + int iy2 = int((keys(i + 1, 1) - min) / max * max_bins); + int iz2 = int((keys(i + 1, 2) - min) / max * max_bins); + + if (ix1 > ix2) + count++; + else if (ix1 == ix2) { + if (iy1 > iy2) + count++; + else if ((iy1 == iy2) && (iz1 > iz2)) + count++; } } }; -template +template struct sum3D { typedef double value_type; typedef ExecutionSpace execution_space; - Kokkos::View keys; + Kokkos::View keys; - sum3D(Kokkos::View keys_):keys(keys_) {} + sum3D(Kokkos::View keys_) : keys(keys_) {} KOKKOS_INLINE_FUNCTION - void operator() (int i, double& count) const { - count+=keys(i,0); - count+=keys(i,1); - count+=keys(i,2); + void operator()(int i, double& count) const { + count += keys(i, 0); + count += keys(i, 1); + count += keys(i, 2); } }; -template -void test_1D_sort(unsigned int n,bool force_kokkos) { - typedef Kokkos::View KeyViewType; - KeyViewType keys("Keys",n); +template +void test_1D_sort(unsigned int n, bool force_kokkos) { + typedef Kokkos::View KeyViewType; + KeyViewType keys("Keys", n); // Test sorting array with all numbers equal - Kokkos::deep_copy(keys,KeyType(1)); - Kokkos::sort(keys,force_kokkos); + Kokkos::deep_copy(keys, KeyType(1)); + Kokkos::sort(keys, force_kokkos); Kokkos::Random_XorShift64_Pool g(1931); - Kokkos::fill_random(keys,g,Kokkos::Random_XorShift64_Pool::generator_type::MAX_URAND); + Kokkos::fill_random(keys, g, + Kokkos::Random_XorShift64_Pool< + ExecutionSpace>::generator_type::MAX_URAND); - double sum_before = 0.0; - double sum_after = 0.0; + double sum_before = 0.0; + double sum_after = 0.0; unsigned int sort_fails = 0; - Kokkos::parallel_reduce(n,sum(keys),sum_before); + Kokkos::parallel_reduce(n, sum(keys), sum_before); - Kokkos::sort(keys,force_kokkos); + Kokkos::sort(keys, force_kokkos); - Kokkos::parallel_reduce(n,sum(keys),sum_after); - Kokkos::parallel_reduce(n-1,is_sorted_struct(keys),sort_fails); + Kokkos::parallel_reduce(n, sum(keys), sum_after); + Kokkos::parallel_reduce( + n - 1, is_sorted_struct(keys), sort_fails); - double ratio = sum_before/sum_after; + double ratio = sum_before / sum_after; double epsilon = 1e-10; - unsigned int equal_sum = (ratio > (1.0-epsilon)) && (ratio < (1.0+epsilon)) ? 1 : 0; + unsigned int equal_sum = + (ratio > (1.0 - epsilon)) && (ratio < (1.0 + epsilon)) ? 1 : 0; - ASSERT_EQ(sort_fails,0); - ASSERT_EQ(equal_sum,1); + ASSERT_EQ(sort_fails, 0); + ASSERT_EQ(equal_sum, 1); } -template +template void test_3D_sort(unsigned int n) { - typedef Kokkos::View KeyViewType; + typedef Kokkos::View KeyViewType; - KeyViewType keys("Keys",n*n*n); + KeyViewType keys("Keys", n * n * n); Kokkos::Random_XorShift64_Pool g(1931); - Kokkos::fill_random(keys,g,100.0); + Kokkos::fill_random(keys, g, 100.0); - double sum_before = 0.0; - double sum_after = 0.0; + double sum_before = 0.0; + double sum_after = 0.0; unsigned int sort_fails = 0; - Kokkos::parallel_reduce(keys.extent(0),sum3D(keys),sum_before); + Kokkos::parallel_reduce(keys.extent(0), sum3D(keys), + sum_before); int bin_1d = 1; - while( bin_1d*bin_1d*bin_1d*4< (int) keys.extent(0) ) bin_1d*=2; - int bin_max[3] = {bin_1d,bin_1d,bin_1d}; - typename KeyViewType::value_type min[3] = {0,0,0}; - typename KeyViewType::value_type max[3] = {100,100,100}; - - typedef Kokkos::BinOp3D< KeyViewType > BinOp; - BinOp bin_op(bin_max,min,max); - Kokkos::BinSort< KeyViewType , BinOp > - Sorter(keys,bin_op,false); + while (bin_1d * bin_1d * bin_1d * 4 < (int)keys.extent(0)) bin_1d *= 2; + int bin_max[3] = {bin_1d, bin_1d, bin_1d}; + typename KeyViewType::value_type min[3] = {0, 0, 0}; + typename KeyViewType::value_type max[3] = {100, 100, 100}; + + typedef Kokkos::BinOp3D BinOp; + BinOp bin_op(bin_max, min, max); + Kokkos::BinSort Sorter(keys, bin_op, false); Sorter.create_permute_vector(); - Sorter.template sort< KeyViewType >(keys); + Sorter.template sort(keys); - Kokkos::parallel_reduce(keys.extent(0),sum3D(keys),sum_after); - Kokkos::parallel_reduce(keys.extent(0)-1,bin3d_is_sorted_struct(keys,bin_1d,min[0],max[0]),sort_fails); + Kokkos::parallel_reduce(keys.extent(0), sum3D(keys), + sum_after); + Kokkos::parallel_reduce(keys.extent(0) - 1, + bin3d_is_sorted_struct( + keys, bin_1d, min[0], max[0]), + sort_fails); - double ratio = sum_before/sum_after; + double ratio = sum_before / sum_after; double epsilon = 1e-10; - unsigned int equal_sum = (ratio > (1.0-epsilon)) && (ratio < (1.0+epsilon)) ? 1 : 0; + unsigned int equal_sum = + (ratio > (1.0 - epsilon)) && (ratio < (1.0 + epsilon)) ? 1 : 0; - if ( sort_fails ) - printf("3D Sort Sum: %f %f Fails: %u\n",sum_before,sum_after,sort_fails); + if (sort_fails) + printf("3D Sort Sum: %f %f Fails: %u\n", sum_before, sum_after, sort_fails); - ASSERT_EQ(sort_fails,0); - ASSERT_EQ(equal_sum,1); + ASSERT_EQ(sort_fails, 0); + ASSERT_EQ(equal_sum, 1); } //---------------------------------------------------------------------------- -template -void test_dynamic_view_sort(unsigned int n ) -{ - typedef Kokkos::Experimental::DynamicView KeyDynamicViewType; - typedef Kokkos::View KeyViewType; +template +void test_dynamic_view_sort(unsigned int n) { + typedef Kokkos::Experimental::DynamicView + KeyDynamicViewType; + typedef Kokkos::View KeyViewType; - const size_t upper_bound = 2 * n ; + const size_t upper_bound = 2 * n; const size_t min_chunk_size = 1024; KeyDynamicViewType keys("Keys", min_chunk_size, upper_bound); keys.resize_serial(n); - KeyViewType keys_view("KeysTmp", n ); + KeyViewType keys_view("KeysTmp", n); // Test sorting array with all numbers equal - Kokkos::deep_copy(keys_view,KeyType(1)); - Kokkos::deep_copy(keys,keys_view); - Kokkos::sort(keys, 0 /* begin */ , n /* end */ ); + Kokkos::deep_copy(keys_view, KeyType(1)); + Kokkos::deep_copy(keys, keys_view); + Kokkos::sort(keys, 0 /* begin */, n /* end */); Kokkos::Random_XorShift64_Pool g(1931); - Kokkos::fill_random(keys_view,g,Kokkos::Random_XorShift64_Pool::generator_type::MAX_URAND); + Kokkos::fill_random(keys_view, g, + Kokkos::Random_XorShift64_Pool< + ExecutionSpace>::generator_type::MAX_URAND); ExecutionSpace().fence(); - Kokkos::deep_copy(keys,keys_view); - //ExecutionSpace().fence(); + Kokkos::deep_copy(keys, keys_view); + // ExecutionSpace().fence(); - double sum_before = 0.0; - double sum_after = 0.0; + double sum_before = 0.0; + double sum_after = 0.0; unsigned int sort_fails = 0; - Kokkos::parallel_reduce(n,sum(keys_view),sum_before); + Kokkos::parallel_reduce(n, sum(keys_view), + sum_before); - Kokkos::sort(keys, 0 /* begin */ , n /* end */ ); + Kokkos::sort(keys, 0 /* begin */, n /* end */); - ExecutionSpace().fence(); // Need this fence to prevent BusError with Cuda - Kokkos::deep_copy( keys_view , keys ); - //ExecutionSpace().fence(); + ExecutionSpace().fence(); // Need this fence to prevent BusError with Cuda + Kokkos::deep_copy(keys_view, keys); + // ExecutionSpace().fence(); - Kokkos::parallel_reduce(n,sum(keys_view),sum_after); - Kokkos::parallel_reduce(n-1,is_sorted_struct(keys_view),sort_fails); + Kokkos::parallel_reduce(n, sum(keys_view), + sum_after); + Kokkos::parallel_reduce( + n - 1, is_sorted_struct(keys_view), sort_fails); - double ratio = sum_before/sum_after; + double ratio = sum_before / sum_after; double epsilon = 1e-10; - unsigned int equal_sum = (ratio > (1.0-epsilon)) && (ratio < (1.0+epsilon)) ? 1 : 0; - - if ( sort_fails != 0 || equal_sum != 1 ) { - std::cout << " N = " << n - << " ; sum_before = " << sum_before - << " ; sum_after = " << sum_after - << " ; ratio = " << ratio - << std::endl ; + unsigned int equal_sum = + (ratio > (1.0 - epsilon)) && (ratio < (1.0 + epsilon)) ? 1 : 0; + + if (sort_fails != 0 || equal_sum != 1) { + std::cout << " N = " << n << " ; sum_before = " << sum_before + << " ; sum_after = " << sum_after << " ; ratio = " << ratio + << std::endl; } - ASSERT_EQ(sort_fails,0); - ASSERT_EQ(equal_sum,1); + ASSERT_EQ(sort_fails, 0); + ASSERT_EQ(equal_sum, 1); } //---------------------------------------------------------------------------- -template -void test_issue_1160() -{ +template +void test_issue_1160() { Kokkos::View element_("element", 10); Kokkos::View x_("x", 10); Kokkos::View v_("y", 10); auto h_element = Kokkos::create_mirror_view(element_); - auto h_x = Kokkos::create_mirror_view(x_); - auto h_v = Kokkos::create_mirror_view(v_); + auto h_x = Kokkos::create_mirror_view(x_); + auto h_v = Kokkos::create_mirror_view(v_); h_element(0) = 9; h_element(1) = 8; @@ -292,20 +306,21 @@ void test_issue_1160() Kokkos::deep_copy(v_, h_v); typedef decltype(element_) KeyViewType; - typedef Kokkos::BinOp1D< KeyViewType > BinOp; + typedef Kokkos::BinOp1D BinOp; int begin = 3; - int end = 8; - auto max = h_element(begin); - auto min = h_element(end - 1); + int end = 8; + auto max = h_element(begin); + auto min = h_element(end - 1); BinOp binner(end - begin, min, max); - Kokkos::BinSort Sorter(element_,begin,end,binner,false); + Kokkos::BinSort Sorter(element_, begin, end, binner, + false); Sorter.create_permute_vector(); - Sorter.sort(element_,begin,end); + Sorter.sort(element_, begin, end); - Sorter.sort(x_,begin,end); - Sorter.sort(v_,begin,end); + Sorter.sort(x_, begin, end); + Sorter.sort(v_, begin, end); Kokkos::deep_copy(h_element, element_); Kokkos::deep_copy(h_x, x_); @@ -330,18 +345,17 @@ void test_issue_1160() //---------------------------------------------------------------------------- -template -void test_sort(unsigned int N) -{ - test_1D_sort(N*N*N, true); - test_1D_sort(N*N*N, false); +template +void test_sort(unsigned int N) { + test_1D_sort(N * N * N, true); + test_1D_sort(N * N * N, false); #if !defined(KOKKOS_ENABLE_ROCM) - test_3D_sort(N); - test_dynamic_view_sort(N*N); + test_3D_sort(N); + test_dynamic_view_sort(N * N); #endif test_issue_1160(); } -} -} +} // namespace Impl +} // namespace Test #endif /* KOKKOS_ALGORITHMS_UNITTESTS_TESTSORT_HPP */ diff --git a/lib/kokkos/algorithms/unit_tests/TestThreads.cpp b/lib/kokkos/algorithms/unit_tests/TestThreads.cpp index 99cdb7da92a253d9469b956c27ee06f212421c0d..c75e6e8dfba9f8d69617b8ff44b4c095a9e55537 100644 --- a/lib/kokkos/algorithms/unit_tests/TestThreads.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestThreads.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -52,51 +53,36 @@ #include #include - //---------------------------------------------------------------------------- - namespace Test { -class threads : public ::testing::Test { -protected: - static void SetUpTestCase() - { - std::cout << std::setprecision(5) << std::scientific; +#define THREADS_RANDOM_XORSHIFT64(num_draws) \ + TEST(threads, Random_XorShift64) { \ + Impl::test_random >( \ + num_draws); \ } - static void TearDownTestCase() - { +#define THREADS_RANDOM_XORSHIFT1024(num_draws) \ + TEST(threads, Random_XorShift1024) { \ + Impl::test_random >( \ + num_draws); \ } -}; -#define THREADS_RANDOM_XORSHIFT64( num_draws ) \ - TEST_F( threads, Random_XorShift64 ) { \ - Impl::test_random >(num_draws); \ +#define THREADS_SORT_UNSIGNED(size) \ + TEST(threads, SortUnsigned) { \ + Impl::test_sort(size); \ } -#define THREADS_RANDOM_XORSHIFT1024( num_draws ) \ - TEST_F( threads, Random_XorShift1024 ) { \ - Impl::test_random >(num_draws); \ - } - -#define THREADS_SORT_UNSIGNED( size ) \ - TEST_F( threads, SortUnsigned ) { \ - Impl::test_sort< Kokkos::Threads, double >(size); \ - } - - -THREADS_RANDOM_XORSHIFT64( 10240000 ) -THREADS_RANDOM_XORSHIFT1024( 10130144 ) +THREADS_RANDOM_XORSHIFT64(10240000) +THREADS_RANDOM_XORSHIFT1024(10130144) THREADS_SORT_UNSIGNED(171) #undef THREADS_RANDOM_XORSHIFT64 #undef THREADS_RANDOM_XORSHIFT1024 #undef THREADS_SORT_UNSIGNED -} // namespace Test +} // namespace Test #else void KOKKOS_ALGORITHMS_UNITTESTS_TESTTHREADS_PREVENT_LINK_ERROR() {} #endif - - diff --git a/lib/kokkos/algorithms/unit_tests/UnitTestMain.cpp b/lib/kokkos/algorithms/unit_tests/UnitTestMain.cpp index 8feb08332fa4d976395edfaf59b8e8653484c0f2..e245aad35fc33a595a16f711dbd4a63a0c7f8948 100644 --- a/lib/kokkos/algorithms/unit_tests/UnitTestMain.cpp +++ b/lib/kokkos/algorithms/unit_tests/UnitTestMain.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -45,10 +46,9 @@ #include int main(int argc, char *argv[]) { - Kokkos::initialize(argc,argv); - ::testing::InitGoogleTest(&argc,argv); + Kokkos::initialize(argc, argv); + ::testing::InitGoogleTest(&argc, argv); int result = RUN_ALL_TESTS(); Kokkos::finalize(); return result; } - diff --git a/lib/kokkos/benchmarks/atomic/main.cpp b/lib/kokkos/benchmarks/atomic/main.cpp index d86d196249b337882993544101d93df35e6ccb22..5f0977f75484e4103e74c4ecaf2d0ad6ec2f3acb 100644 --- a/lib/kokkos/benchmarks/atomic/main.cpp +++ b/lib/kokkos/benchmarks/atomic/main.cpp @@ -1,124 +1,120 @@ -#include -#include -#include +#include +#include +#include -template -double test_atomic(int L, int N, int M,int K,int R,Kokkos::View offsets) { - Kokkos::View output("Output",N); +template +double test_atomic(int L, int N, int M, int K, int R, + Kokkos::View offsets) { + Kokkos::View output("Output", N); Kokkos::Impl::Timer timer; - for(int r = 0; r -double test_no_atomic(int L, int N, int M,int K,int R,Kokkos::View offsets) { - Kokkos::View output("Output",N); +template +double test_no_atomic(int L, int N, int M, int K, int R, + Kokkos::View offsets) { + Kokkos::View output("Output", N); Kokkos::Impl::Timer timer; - for(int r = 0; r\n"); - printf("Example Input GPU:\n"); - printf(" Histogram : 1000000 1000 1 1000 1 10 1\n"); - printf(" MD Force : 100000 100000 100 1000 20 10 4\n"); - printf(" Matrix Assembly : 100000 1000000 50 1000 20 10 4\n"); - Kokkos::finalize(); - return 0; - } + Kokkos::initialize(argc, argv); + { + if (argc < 8) { + printf("Arguments: L N M D K R T\n"); + printf(" L: Number of iterations to run\n"); + printf(" N: Length of array to do atomics into\n"); + printf(" M: Number of atomics per iteration to do\n"); + printf(" D: Distance from index i to do atomics into (randomly)\n"); + printf(" K: Number of FMAD per atomic\n"); + printf(" R: Number of repeats of the experiments\n"); + printf(" T: Type of atomic\n"); + printf(" 1 - int\n"); + printf(" 2 - long\n"); + printf(" 3 - float\n"); + printf(" 4 - double\n"); + printf(" 5 - complex\n"); + printf("Example Input GPU:\n"); + printf(" Histogram : 1000000 1000 1 1000 1 10 1\n"); + printf(" MD Force : 100000 100000 100 1000 20 10 4\n"); + printf(" Matrix Assembly : 100000 1000000 50 1000 20 10 4\n"); + Kokkos::finalize(); + return 0; + } + int L = atoi(argv[1]); + int N = atoi(argv[2]); + int M = atoi(argv[3]); + int D = atoi(argv[4]); + int K = atoi(argv[5]); + int R = atoi(argv[6]); + int type = atoi(argv[7]); - int L = atoi(argv[1]); - int N = atoi(argv[2]); - int M = atoi(argv[3]); - int D = atoi(argv[4]); - int K = atoi(argv[5]); - int R = atoi(argv[6]); - int type = atoi(argv[7]); - - Kokkos::View offsets("Offsets",L,M); - Kokkos::Random_XorShift64_Pool<> pool(12371); - Kokkos::fill_random(offsets,pool,D); - double time = 0; - if(type==1) - time = test_atomic(L,N,M,K,R,offsets); - if(type==2) - time = test_atomic(L,N,M,K,R,offsets); - if(type==3) - time = test_atomic(L,N,M,K,R,offsets); - if(type==4) - time = test_atomic(L,N,M,K,R,offsets); - if(type==5) - time = test_atomic >(L,N,M,K,R,offsets); + Kokkos::View offsets("Offsets", L, M); + Kokkos::Random_XorShift64_Pool<> pool(12371); + Kokkos::fill_random(offsets, pool, D); + double time = 0; + if (type == 1) time = test_atomic(L, N, M, K, R, offsets); + if (type == 2) time = test_atomic(L, N, M, K, R, offsets); + if (type == 3) time = test_atomic(L, N, M, K, R, offsets); + if (type == 4) time = test_atomic(L, N, M, K, R, offsets); + if (type == 5) + time = test_atomic >(L, N, M, K, R, offsets); - double time2 = 1; - if(type==1) - time2 = test_no_atomic(L,N,M,K,R,offsets); - if(type==2) - time2 = test_no_atomic(L,N,M,K,R,offsets); - if(type==3) - time2 = test_no_atomic(L,N,M,K,R,offsets); - if(type==4) - time2 = test_no_atomic(L,N,M,K,R,offsets); - if(type==5) - time2 = test_no_atomic >(L,N,M,K,R,offsets); + double time2 = 1; + if (type == 1) time2 = test_no_atomic(L, N, M, K, R, offsets); + if (type == 2) time2 = test_no_atomic(L, N, M, K, R, offsets); + if (type == 3) time2 = test_no_atomic(L, N, M, K, R, offsets); + if (type == 4) time2 = test_no_atomic(L, N, M, K, R, offsets); + if (type == 5) + time2 = test_no_atomic >(L, N, M, K, R, offsets); - int size = 0; - if(type==1) size = sizeof(int); - if(type==2) size = sizeof(long); - if(type==3) size = sizeof(float); - if(type==4) size = sizeof(double); - if(type==5) size = sizeof(Kokkos::complex); + int size = 0; + if (type == 1) size = sizeof(int); + if (type == 2) size = sizeof(long); + if (type == 3) size = sizeof(float); + if (type == 4) size = sizeof(double); + if (type == 5) size = sizeof(Kokkos::complex); - printf("%i\n",size); - printf("Time: %s %i %i %i %i %i %i (t_atomic: %e t_nonatomic: %e ratio: %lf )( GUpdates/s: %lf GB/s: %lf )\n", - (type==1)?"int": ( - (type==2)?"long": ( - (type==3)?"float": ( - (type==4)?"double":"complex"))), - L,N,M,D,K,R,time,time2,time/time2, - 1.e-9*L*R*M/time, 1.0*L*R*M*2*size/time/1024/1024/1024); -} + printf("%i\n", size); + printf( + "Time: %s %i %i %i %i %i %i (t_atomic: %e t_nonatomic: %e ratio: %lf " + ")( GUpdates/s: %lf GB/s: %lf )\n", + (type == 1) + ? "int" + : ((type == 2) + ? "long" + : ((type == 3) ? "float" + : ((type == 4) ? "double" : "complex"))), + L, N, M, D, K, R, time, time2, time / time2, 1.e-9 * L * R * M / time, + 1.0 * L * R * M * 2 * size / time / 1024 / 1024 / 1024); + } Kokkos::finalize(); } - diff --git a/lib/kokkos/benchmarks/bytes_and_flops/bench.hpp b/lib/kokkos/benchmarks/bytes_and_flops/bench.hpp index 59b4d50c441eb5d4c58c99c53eee7f9c6b9adbd1..62d7ef4a4cf387191c6d0276c4ea360c289d4de5 100644 --- a/lib/kokkos/benchmarks/bytes_and_flops/bench.hpp +++ b/lib/kokkos/benchmarks/bytes_and_flops/bench.hpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -41,59 +42,52 @@ //@HEADER */ -#include -#include +#include +#include -template +template struct Run { -static void run(int N, int K, int R, int F, int T, int S); + static void run(int N, int K, int R, int F, int T, int S); }; -template +template struct RunStride { -static void run_1(int N, int K, int R, int F, int T, int S); -static void run_2(int N, int K, int R, int F, int T, int S); -static void run_3(int N, int K, int R, int F, int T, int S); -static void run_4(int N, int K, int R, int F, int T, int S); -static void run_5(int N, int K, int R, int F, int T, int S); -static void run_6(int N, int K, int R, int F, int T, int S); -static void run_7(int N, int K, int R, int F, int T, int S); -static void run_8(int N, int K, int R, int F, int T, int S); -static void run(int N, int K, int R, int U, int F, int T, int S); + static void run_1(int N, int K, int R, int F, int T, int S); + static void run_2(int N, int K, int R, int F, int T, int S); + static void run_3(int N, int K, int R, int F, int T, int S); + static void run_4(int N, int K, int R, int F, int T, int S); + static void run_5(int N, int K, int R, int F, int T, int S); + static void run_6(int N, int K, int R, int F, int T, int S); + static void run_7(int N, int K, int R, int F, int T, int S); + static void run_8(int N, int K, int R, int F, int T, int S); + static void run(int N, int K, int R, int U, int F, int T, int S); }; #define STRIDE 1 -#include +#include #undef STRIDE #define STRIDE 2 -#include +#include #undef STRIDE #define STRIDE 4 -#include +#include #undef STRIDE #define STRIDE 8 -#include +#include #undef STRIDE #define STRIDE 16 -#include +#include #undef STRIDE #define STRIDE 32 -#include +#include #undef STRIDE -template +template void run_stride_unroll(int N, int K, int R, int D, int U, int F, int T, int S) { - if(D == 1) - RunStride::run(N,K,R,U,F,T,S); - if(D == 2) - RunStride::run(N,K,R,U,F,T,S); - if(D == 4) - RunStride::run(N,K,R,U,F,T,S); - if(D == 8) - RunStride::run(N,K,R,U,F,T,S); - if(D == 16) - RunStride::run(N,K,R,U,F,T,S); - if(D == 32) - RunStride::run(N,K,R,U,F,T,S); + if (D == 1) RunStride::run(N, K, R, U, F, T, S); + if (D == 2) RunStride::run(N, K, R, U, F, T, S); + if (D == 4) RunStride::run(N, K, R, U, F, T, S); + if (D == 8) RunStride::run(N, K, R, U, F, T, S); + if (D == 16) RunStride::run(N, K, R, U, F, T, S); + if (D == 32) RunStride::run(N, K, R, U, F, T, S); } - diff --git a/lib/kokkos/benchmarks/bytes_and_flops/bench_stride.hpp b/lib/kokkos/benchmarks/bytes_and_flops/bench_stride.hpp index 6509c654e71b4fdfd5366e4699972d7d19fe7807..64817fe9dc4b5d1efcad168d9b2e0915da6d492a 100644 --- a/lib/kokkos/benchmarks/bytes_and_flops/bench_stride.hpp +++ b/lib/kokkos/benchmarks/bytes_and_flops/bench_stride.hpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -41,84 +42,82 @@ //@HEADER */ - #define UNROLL 1 -#include +#include #undef UNROLL #define UNROLL 2 -#include +#include #undef UNROLL #define UNROLL 3 -#include +#include #undef UNROLL #define UNROLL 4 -#include +#include #undef UNROLL #define UNROLL 5 -#include +#include #undef UNROLL #define UNROLL 6 -#include +#include #undef UNROLL #define UNROLL 7 -#include +#include #undef UNROLL #define UNROLL 8 -#include +#include #undef UNROLL -template -struct RunStride { -static void run_1(int N, int K, int R, int F, int T, int S) { - Run::run(N,K,R,F,T,S); -} -static void run_2(int N, int K, int R, int F, int T, int S) { - Run::run(N,K,R,F,T,S); -} -static void run_3(int N, int K, int R, int F, int T, int S) { - Run::run(N,K,R,F,T,S); -} -static void run_4(int N, int K, int R, int F, int T, int S) { - Run::run(N,K,R,F,T,S); -} -static void run_5(int N, int K, int R, int F, int T, int S) { - Run::run(N,K,R,F,T,S); -} -static void run_6(int N, int K, int R, int F, int T, int S) { - Run::run(N,K,R,F,T,S); -} -static void run_7(int N, int K, int R, int F, int T, int S) { - Run::run(N,K,R,F,T,S); -} -static void run_8(int N, int K, int R, int F, int T, int S) { - Run::run(N,K,R,F,T,S); -} - -static void run(int N, int K, int R, int U, int F, int T, int S) { - if(U==1) { - run_1(N,K,R,F,T,S); +template +struct RunStride { + static void run_1(int N, int K, int R, int F, int T, int S) { + Run::run(N, K, R, F, T, S); } - if(U==2) { - run_2(N,K,R,F,T,S); + static void run_2(int N, int K, int R, int F, int T, int S) { + Run::run(N, K, R, F, T, S); } - if(U==3) { - run_3(N,K,R,F,T,S); + static void run_3(int N, int K, int R, int F, int T, int S) { + Run::run(N, K, R, F, T, S); } - if(U==4) { - run_4(N,K,R,F,T,S); + static void run_4(int N, int K, int R, int F, int T, int S) { + Run::run(N, K, R, F, T, S); } - if(U==5) { - run_5(N,K,R,F,T,S); + static void run_5(int N, int K, int R, int F, int T, int S) { + Run::run(N, K, R, F, T, S); } - if(U==6) { - run_6(N,K,R,F,T,S); + static void run_6(int N, int K, int R, int F, int T, int S) { + Run::run(N, K, R, F, T, S); } - if(U==7) { - run_7(N,K,R,F,T,S); + static void run_7(int N, int K, int R, int F, int T, int S) { + Run::run(N, K, R, F, T, S); + } + static void run_8(int N, int K, int R, int F, int T, int S) { + Run::run(N, K, R, F, T, S); } - if(U==8) { - run_8(N,K,R,F,T,S); - } -} -}; + static void run(int N, int K, int R, int U, int F, int T, int S) { + if (U == 1) { + run_1(N, K, R, F, T, S); + } + if (U == 2) { + run_2(N, K, R, F, T, S); + } + if (U == 3) { + run_3(N, K, R, F, T, S); + } + if (U == 4) { + run_4(N, K, R, F, T, S); + } + if (U == 5) { + run_5(N, K, R, F, T, S); + } + if (U == 6) { + run_6(N, K, R, F, T, S); + } + if (U == 7) { + run_7(N, K, R, F, T, S); + } + if (U == 8) { + run_8(N, K, R, F, T, S); + } + } +}; diff --git a/lib/kokkos/benchmarks/bytes_and_flops/bench_unroll_stride.hpp b/lib/kokkos/benchmarks/bytes_and_flops/bench_unroll_stride.hpp index c6651da1e7cf2e6d1a233dc90e9adf1211decf69..00ce635a489f677ff43b05c782856dcdfa1cafa9 100644 --- a/lib/kokkos/benchmarks/bytes_and_flops/bench_unroll_stride.hpp +++ b/lib/kokkos/benchmarks/bytes_and_flops/bench_unroll_stride.hpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -41,108 +42,110 @@ //@HEADER */ -template -struct Run { -static void run(int N, int K, int R, int F, int T, int S) { - Kokkos::View A("A",N,K); - Kokkos::View B("B",N,K); - Kokkos::View C("C",N,K); +template +struct Run { + static void run(int N, int K, int R, int F, int T, int S) { + Kokkos::View A("A", N, K); + Kokkos::View B("B", N, K); + Kokkos::View C("C", N, K); - Kokkos::deep_copy(A,Scalar(1.5)); - Kokkos::deep_copy(B,Scalar(2.5)); - Kokkos::deep_copy(C,Scalar(3.5)); + Kokkos::deep_copy(A, Scalar(1.5)); + Kokkos::deep_copy(B, Scalar(2.5)); + Kokkos::deep_copy(C, Scalar(3.5)); - Kokkos::Timer timer; - Kokkos::parallel_for("BenchmarkKernel",Kokkos::TeamPolicy<>(N,T).set_scratch_size(0,Kokkos::PerTeam(S)), - KOKKOS_LAMBDA ( const Kokkos::TeamPolicy<>::member_type& team) { - const int n = team.league_rank(); - for(int r=0; r1) - Scalar a2 = a1*1.3; -#endif -#if(UNROLL>2) - Scalar a3 = a2*1.1; -#endif -#if(UNROLL>3) - Scalar a4 = a3*1.1; -#endif -#if(UNROLL>4) - Scalar a5 = a4*1.3; -#endif -#if(UNROLL>5) - Scalar a6 = a5*1.1; -#endif -#if(UNROLL>6) - Scalar a7 = a6*1.1; -#endif -#if(UNROLL>7) - Scalar a8 = a7*1.1; + Kokkos::Timer timer; + Kokkos::parallel_for( + "BenchmarkKernel", + Kokkos::TeamPolicy<>(N, T).set_scratch_size(0, Kokkos::PerTeam(S)), + KOKKOS_LAMBDA(const Kokkos::TeamPolicy<>::member_type& team) { + const int n = team.league_rank(); + for (int r = 0; r < R; r++) { + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0, K), [&](const int& i) { + Scalar a1 = A(n, i, 0); + const Scalar b = B(n, i, 0); +#if (UNROLL > 1) + Scalar a2 = a1 * 1.3; +#endif +#if (UNROLL > 2) + Scalar a3 = a2 * 1.1; +#endif +#if (UNROLL > 3) + Scalar a4 = a3 * 1.1; +#endif +#if (UNROLL > 4) + Scalar a5 = a4 * 1.3; +#endif +#if (UNROLL > 5) + Scalar a6 = a5 * 1.1; +#endif +#if (UNROLL > 6) + Scalar a7 = a6 * 1.1; +#endif +#if (UNROLL > 7) + Scalar a8 = a7 * 1.1; #endif - - for(int f = 0; f1) - a2 += b*a2; + for (int f = 0; f < F; f++) { + a1 += b * a1; +#if (UNROLL > 1) + a2 += b * a2; #endif -#if(UNROLL>2) - a3 += b*a3; +#if (UNROLL > 2) + a3 += b * a3; #endif -#if(UNROLL>3) - a4 += b*a4; +#if (UNROLL > 3) + a4 += b * a4; #endif -#if(UNROLL>4) - a5 += b*a5; +#if (UNROLL > 4) + a5 += b * a5; #endif -#if(UNROLL>5) - a6 += b*a6; +#if (UNROLL > 5) + a6 += b * a6; #endif -#if(UNROLL>6) - a7 += b*a7; +#if (UNROLL > 6) + a7 += b * a7; #endif -#if(UNROLL>7) - a8 += b*a8; +#if (UNROLL > 7) + a8 += b * a8; #endif - - - } -#if(UNROLL==1) - C(n,i,0) = a1; + } +#if (UNROLL == 1) + C(n, i, 0) = a1; #endif -#if(UNROLL==2) - C(n,i,0) = a1+a2; +#if (UNROLL == 2) + C(n, i, 0) = a1 + a2; #endif -#if(UNROLL==3) - C(n,i,0) = a1+a2+a3; +#if (UNROLL == 3) + C(n, i, 0) = a1 + a2 + a3; #endif -#if(UNROLL==4) - C(n,i,0) = a1+a2+a3+a4; +#if (UNROLL == 4) + C(n, i, 0) = a1 + a2 + a3 + a4; #endif -#if(UNROLL==5) - C(n,i,0) = a1+a2+a3+a4+a5; +#if (UNROLL == 5) + C(n, i, 0) = a1 + a2 + a3 + a4 + a5; #endif -#if(UNROLL==6) - C(n,i,0) = a1+a2+a3+a4+a5+a6; +#if (UNROLL == 6) + C(n, i, 0) = a1 + a2 + a3 + a4 + a5 + a6; #endif -#if(UNROLL==7) - C(n,i,0) = a1+a2+a3+a4+a5+a6+a7; +#if (UNROLL == 7) + C(n, i, 0) = a1 + a2 + a3 + a4 + a5 + a6 + a7; #endif -#if(UNROLL==8) - C(n,i,0) = a1+a2+a3+a4+a5+a6+a7+a8; +#if (UNROLL == 8) + C(n, i, 0) = a1 + a2 + a3 + a4 + a5 + a6 + a7 + a8; #endif + }); + } + }); + Kokkos::fence(); + double seconds = timer.seconds(); - }); - } - }); - Kokkos::fence(); - double seconds = timer.seconds(); - - double bytes = 1.0*N*K*R*3*sizeof(Scalar); - double flops = 1.0*N*K*R*(F*2*UNROLL + 2*(UNROLL-1)); - printf("NKRUFTS: %i %i %i %i %i %i %i Time: %lfs Bandwidth: %lfGiB/s GFlop/s: %lf\n",N,K,R,UNROLL,F,T,S,seconds,1.0*bytes/seconds/1024/1024/1024,1.e-9*flops/seconds); -} + double bytes = 1.0 * N * K * R * 3 * sizeof(Scalar); + double flops = 1.0 * N * K * R * (F * 2 * UNROLL + 2 * (UNROLL - 1)); + printf( + "NKRUFTS: %i %i %i %i %i %i %i Time: %lfs Bandwidth: %lfGiB/s GFlop/s: " + "%lf\n", + N, K, R, UNROLL, F, T, S, seconds, + 1.0 * bytes / seconds / 1024 / 1024 / 1024, 1.e-9 * flops / seconds); + } }; - diff --git a/lib/kokkos/benchmarks/bytes_and_flops/main.cpp b/lib/kokkos/benchmarks/bytes_and_flops/main.cpp index 4f46b38717df237f0d33c0ac101d105f41f3e9d8..c21a16200e4ed0cf48e8d9e4b1dd25d9753e95c5 100644 --- a/lib/kokkos/benchmarks/bytes_and_flops/main.cpp +++ b/lib/kokkos/benchmarks/bytes_and_flops/main.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -41,25 +42,27 @@ //@HEADER */ -#include -#include -#include -#include +#include +#include +#include +#include int main(int argc, char* argv[]) { Kokkos::initialize(); - - if(argc<10) { + if (argc < 10) { printf("Arguments: N K R D U F T S\n"); printf(" P: Precision (1==float, 2==double)\n"); printf(" N,K: dimensions of the 2D array to allocate\n"); printf(" R: how often to loop through the K dimension with each team\n"); printf(" D: distance between loaded elements (stride)\n"); printf(" U: how many independent flops to do per load\n"); - printf(" F: how many times to repeat the U unrolled operations before reading next element\n"); + printf( + " F: how many times to repeat the U unrolled operations before " + "reading next element\n"); printf(" T: team size\n"); - printf(" S: shared memory per team (used to control occupancy on GPUs)\n"); + printf( + " S: shared memory per team (used to control occupancy on GPUs)\n"); printf("Example Input GPU:\n"); printf(" Bandwidth Bound : 2 100000 1024 1 1 1 1 256 6000\n"); printf(" Cache Bound : 2 100000 1024 64 1 1 1 512 20000\n"); @@ -70,7 +73,6 @@ int main(int argc, char* argv[]) { return 0; } - int P = atoi(argv[1]); int N = atoi(argv[2]); int K = atoi(argv[3]); @@ -81,17 +83,25 @@ int main(int argc, char* argv[]) { int T = atoi(argv[8]); int S = atoi(argv[9]); - if(U>8) {printf("U must be 1-8\n"); return 0;} - if( (D!=1) && (D!=2) && (D!=4) && (D!=8) && (D!=16) && (D!=32)) {printf("D must be one of 1,2,4,8,16,32\n"); return 0;} - if( (P!=1) && (P!=2) ) {printf("P must be one of 1,2\n"); return 0;} + if (U > 8) { + printf("U must be 1-8\n"); + return 0; + } + if ((D != 1) && (D != 2) && (D != 4) && (D != 8) && (D != 16) && (D != 32)) { + printf("D must be one of 1,2,4,8,16,32\n"); + return 0; + } + if ((P != 1) && (P != 2)) { + printf("P must be one of 1,2\n"); + return 0; + } - if(P==1) { - run_stride_unroll(N,K,R,D,U,F,T,S); + if (P == 1) { + run_stride_unroll(N, K, R, D, U, F, T, S); } - if(P==2) { - run_stride_unroll(N,K,R,D,U,F,T,S); + if (P == 2) { + run_stride_unroll(N, K, R, D, U, F, T, S); } Kokkos::finalize(); } - diff --git a/lib/kokkos/benchmarks/gather/gather.hpp b/lib/kokkos/benchmarks/gather/gather.hpp index bbbd65850f75c00a928383d3c6ae2f9af27d7d95..239614184ba13aad35fe54190ce5eaf507c61fde 100644 --- a/lib/kokkos/benchmarks/gather/gather.hpp +++ b/lib/kokkos/benchmarks/gather/gather.hpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -41,52 +42,44 @@ //@HEADER */ -template +template struct RunGather { static void run(int N, int K, int D, int R, int F); }; #define UNROLL 1 -#include +#include #undef UNROLL #define UNROLL 2 -#include +#include #undef UNROLL #define UNROLL 3 -#include +#include #undef UNROLL #define UNROLL 4 -#include +#include #undef UNROLL #define UNROLL 5 -#include +#include #undef UNROLL #define UNROLL 6 -#include +#include #undef UNROLL #define UNROLL 7 -#include +#include #undef UNROLL #define UNROLL 8 -#include +#include #undef UNROLL -template +template void run_gather_test(int N, int K, int D, int R, int U, int F) { - if(U == 1) - RunGather::run(N,K,D,R,F); - if(U == 2) - RunGather::run(N,K,D,R,F); - if(U == 3) - RunGather::run(N,K,D,R,F); - if(U == 4) - RunGather::run(N,K,D,R,F); - if(U == 5) - RunGather::run(N,K,D,R,F); - if(U == 6) - RunGather::run(N,K,D,R,F); - if(U == 7) - RunGather::run(N,K,D,R,F); - if(U == 8) - RunGather::run(N,K,D,R,F); + if (U == 1) RunGather::run(N, K, D, R, F); + if (U == 2) RunGather::run(N, K, D, R, F); + if (U == 3) RunGather::run(N, K, D, R, F); + if (U == 4) RunGather::run(N, K, D, R, F); + if (U == 5) RunGather::run(N, K, D, R, F); + if (U == 6) RunGather::run(N, K, D, R, F); + if (U == 7) RunGather::run(N, K, D, R, F); + if (U == 8) RunGather::run(N, K, D, R, F); } diff --git a/lib/kokkos/benchmarks/gather/gather_unroll.hpp b/lib/kokkos/benchmarks/gather/gather_unroll.hpp index 1d9c99adf9e9909a21be2899f60000713e0208a4..4dc046f99c33125f86d7674f61b38b77caff99c2 100644 --- a/lib/kokkos/benchmarks/gather/gather_unroll.hpp +++ b/lib/kokkos/benchmarks/gather/gather_unroll.hpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -41,129 +42,132 @@ //@HEADER */ -#include -#include +#include +#include -template -struct RunGather { -static void run(int N, int K, int D, int R, int F) { - Kokkos::View connectivity("Connectivity",N,K); - Kokkos::View A_in("Input",N); - Kokkos::View B_in("Input",N); - Kokkos::View C("Output",N); +template +struct RunGather { + static void run(int N, int K, int D, int R, int F) { + Kokkos::View connectivity("Connectivity", N, K); + Kokkos::View A_in("Input", N); + Kokkos::View B_in("Input", N); + Kokkos::View C("Output", N); - Kokkos::Random_XorShift64_Pool<> rand_pool(12313); + Kokkos::Random_XorShift64_Pool<> rand_pool(12313); - Kokkos::deep_copy(A_in,1.5); - Kokkos::deep_copy(B_in,2.0); + Kokkos::deep_copy(A_in, 1.5); + Kokkos::deep_copy(B_in, 2.0); - Kokkos::View > A(A_in); - Kokkos::View > B(B_in); - - Kokkos::parallel_for("InitKernel",N, - KOKKOS_LAMBDA (const int& i) { - auto rand_gen = rand_pool.get_state(); - for( int jj=0; jj > A( + A_in); + Kokkos::View > B( + B_in); + Kokkos::parallel_for( + "InitKernel", N, KOKKOS_LAMBDA(const int& i) { + auto rand_gen = rand_pool.get_state(); + for (int jj = 0; jj < K; jj++) { + connectivity(i, jj) = (rand_gen.rand(D) + i - D / 2 + N) % N; + } + rand_pool.free_state(rand_gen); + }); + Kokkos::fence(); - Kokkos::Timer timer; - for(int r = 0; r1) - Scalar a2 = a1*Scalar(1.3); + Kokkos::Timer timer; + for (int r = 0; r < R; r++) { + Kokkos::parallel_for( + "BenchmarkKernel", N, KOKKOS_LAMBDA(const int& i) { + Scalar c = Scalar(0.0); + for (int jj = 0; jj < K; jj++) { + const int j = connectivity(i, jj); + Scalar a1 = A(j); + const Scalar b = B(j); +#if (UNROLL > 1) + Scalar a2 = a1 * Scalar(1.3); #endif -#if(UNROLL>2) - Scalar a3 = a2*Scalar(1.1); +#if (UNROLL > 2) + Scalar a3 = a2 * Scalar(1.1); #endif -#if(UNROLL>3) - Scalar a4 = a3*Scalar(1.1); +#if (UNROLL > 3) + Scalar a4 = a3 * Scalar(1.1); #endif -#if(UNROLL>4) - Scalar a5 = a4*Scalar(1.3); +#if (UNROLL > 4) + Scalar a5 = a4 * Scalar(1.3); #endif -#if(UNROLL>5) - Scalar a6 = a5*Scalar(1.1); +#if (UNROLL > 5) + Scalar a6 = a5 * Scalar(1.1); #endif -#if(UNROLL>6) - Scalar a7 = a6*Scalar(1.1); +#if (UNROLL > 6) + Scalar a7 = a6 * Scalar(1.1); #endif -#if(UNROLL>7) - Scalar a8 = a7*Scalar(1.1); +#if (UNROLL > 7) + Scalar a8 = a7 * Scalar(1.1); #endif - - for(int f = 0; f1) - a2 += b*a2; + for (int f = 0; f < F; f++) { + a1 += b * a1; +#if (UNROLL > 1) + a2 += b * a2; #endif -#if(UNROLL>2) - a3 += b*a3; +#if (UNROLL > 2) + a3 += b * a3; #endif -#if(UNROLL>3) - a4 += b*a4; +#if (UNROLL > 3) + a4 += b * a4; #endif -#if(UNROLL>4) - a5 += b*a5; +#if (UNROLL > 4) + a5 += b * a5; #endif -#if(UNROLL>5) - a6 += b*a6; +#if (UNROLL > 5) + a6 += b * a6; #endif -#if(UNROLL>6) - a7 += b*a7; +#if (UNROLL > 6) + a7 += b * a7; #endif -#if(UNROLL>7) - a8 += b*a8; +#if (UNROLL > 7) + a8 += b * a8; #endif - - - } -#if(UNROLL==1) - c += a1; + } +#if (UNROLL == 1) + c += a1; #endif -#if(UNROLL==2) - c += a1+a2; +#if (UNROLL == 2) + c += a1 + a2; #endif -#if(UNROLL==3) - c += a1+a2+a3; +#if (UNROLL == 3) + c += a1 + a2 + a3; #endif -#if(UNROLL==4) - c += a1+a2+a3+a4; +#if (UNROLL == 4) + c += a1 + a2 + a3 + a4; #endif -#if(UNROLL==5) - c += a1+a2+a3+a4+a5; +#if (UNROLL == 5) + c += a1 + a2 + a3 + a4 + a5; #endif -#if(UNROLL==6) - c += a1+a2+a3+a4+a5+a6; +#if (UNROLL == 6) + c += a1 + a2 + a3 + a4 + a5 + a6; #endif -#if(UNROLL==7) - c += a1+a2+a3+a4+a5+a6+a7; +#if (UNROLL == 7) + c += a1 + a2 + a3 + a4 + a5 + a6 + a7; #endif -#if(UNROLL==8) - c += a1+a2+a3+a4+a5+a6+a7+a8; +#if (UNROLL == 8) + c += a1 + a2 + a3 + a4 + a5 + a6 + a7 + a8; #endif + } + C(i) = c; + }); + Kokkos::fence(); + } + double seconds = timer.seconds(); - } - C(i) = c ; - }); - Kokkos::fence(); + double bytes = 1.0 * N * K * R * (2 * sizeof(Scalar) + sizeof(int)) + + 1.0 * N * R * sizeof(Scalar); + double flops = 1.0 * N * K * R * (F * 2 * UNROLL + 2 * (UNROLL - 1)); + double gather_ops = 1.0 * N * K * R * 2; + printf( + "SNKDRUF: %i %i %i %i %i %i %i Time: %lfs Bandwidth: %lfGiB/s GFlop/s: " + "%lf GGather/s: %lf\n", + sizeof(Scalar) / 4, N, K, D, R, UNROLL, F, seconds, + 1.0 * bytes / seconds / 1024 / 1024 / 1024, 1.e-9 * flops / seconds, + 1.e-9 * gather_ops / seconds); } - double seconds = timer.seconds(); - - double bytes = 1.0*N*K*R*(2*sizeof(Scalar)+sizeof(int)) + 1.0*N*R*sizeof(Scalar); - double flops = 1.0*N*K*R*(F*2*UNROLL + 2*(UNROLL-1)); - double gather_ops = 1.0*N*K*R*2; - printf("SNKDRUF: %i %i %i %i %i %i %i Time: %lfs Bandwidth: %lfGiB/s GFlop/s: %lf GGather/s: %lf\n",sizeof(Scalar)/4,N,K,D,R,UNROLL,F,seconds,1.0*bytes/seconds/1024/1024/1024,1.e-9*flops/seconds,1.e-9*gather_ops/seconds); -} }; diff --git a/lib/kokkos/benchmarks/gather/main.cpp b/lib/kokkos/benchmarks/gather/main.cpp index ca5238e7fdb527e56080f5b76eb6fa7c9487fcd3..6a2db3e024dff5f338aebf2260fc8910aa9764bc 100644 --- a/lib/kokkos/benchmarks/gather/main.cpp +++ b/lib/kokkos/benchmarks/gather/main.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -41,23 +42,26 @@ //@HEADER */ -#include -#include -#include -#include +#include +#include +#include +#include int main(int argc, char* argv[]) { - Kokkos::initialize(argc,argv); + Kokkos::initialize(argc, argv); - if(argc<8) { + if (argc < 8) { printf("Arguments: S N K D\n"); - printf(" S: Scalar Type Size (1==float, 2==double, 4=complex)\n"); + printf( + " S: Scalar Type Size (1==float, 2==double, 4=complex)\n"); printf(" N: Number of entities\n"); printf(" K: Number of things to gather per entity\n"); printf(" D: Max distance of gathered things of an entity\n"); printf(" R: how often to loop through the K dimension with each team\n"); printf(" U: how many independent flops to do per load\n"); - printf(" F: how many times to repeat the U unrolled operations before reading next element\n"); + printf( + " F: how many times to repeat the U unrolled operations before " + "reading next element\n"); printf("Example Input GPU:\n"); printf(" Bandwidth Bound : 2 10000000 1 1 10 1 1\n"); printf(" Cache Bound : 2 10000000 64 1 10 1 1\n"); @@ -68,7 +72,6 @@ int main(int argc, char* argv[]) { return 0; } - int S = atoi(argv[1]); int N = atoi(argv[2]); int K = atoi(argv[3]); @@ -77,17 +80,22 @@ int main(int argc, char* argv[]) { int U = atoi(argv[6]); int F = atoi(argv[7]); - if( (S!=1) && (S!=2) && (S!=4)) {printf("S must be one of 1,2,4\n"); return 0;} - if( N(N,K,D,R,U,F); + if ((S != 1) && (S != 2) && (S != 4)) { + printf("S must be one of 1,2,4\n"); + return 0; + } + if (N < D) { + printf("N must be larger or equal to D\n"); + return 0; + } + if (S == 1) { + run_gather_test(N, K, D, R, U, F); } - if(S==2) { - run_gather_test(N,K,D,R,U,F); + if (S == 2) { + run_gather_test(N, K, D, R, U, F); } - if(S==4) { - run_gather_test >(N,K,D,R,U,F); + if (S == 4) { + run_gather_test >(N, K, D, R, U, F); } Kokkos::finalize(); } - diff --git a/lib/kokkos/benchmarks/gups/gups-kokkos.cc b/lib/kokkos/benchmarks/gups/gups-kokkos.cc index 4602adda7937cc0dfc8939527353f8936961fed3..9ac59be4a6b348d28d19757f8f0acb7c59686fe2 100644 --- a/lib/kokkos/benchmarks/gups/gups-kokkos.cc +++ b/lib/kokkos/benchmarks/gups/gups-kokkos.cc @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR diff --git a/lib/kokkos/benchmarks/policy_performance/main.cpp b/lib/kokkos/benchmarks/policy_performance/main.cpp index 2f5395734afdcfce41cc43c9b2322de4615a74b1..332e5574dabe3a22e0834f351524ba09487730a5 100644 --- a/lib/kokkos/benchmarks/policy_performance/main.cpp +++ b/lib/kokkos/benchmarks/policy_performance/main.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -44,67 +45,86 @@ #include #include "policy_perf_test.hpp" -int main(int argc, char* argv[] ) { - Kokkos::initialize(argc,argv); +int main(int argc, char* argv[]) { + Kokkos::initialize(argc, argv); - if(argc<10) { + if (argc < 10) { printf(" Ten arguments are needed to run this program:\n"); - printf(" (1)team_range, (2)thread_range, (3)vector_range, (4)outer_repeat, (5)thread_repeat, (6)vector_repeat, (7)team_size, (8)vector_size, (9)schedule, (10)test_type\n"); + printf( + " (1)team_range, (2)thread_range, (3)vector_range, (4)outer_repeat, " + "(5)thread_repeat, (6)vector_repeat, (7)team_size, (8)vector_size, " + "(9)schedule, (10)test_type\n"); printf(" team_range: number of teams (league_size)\n"); printf(" thread_range: range for nested TeamThreadRange parallel_*\n"); printf(" vector_range: range for nested ThreadVectorRange parallel_*\n"); printf(" outer_repeat: number of repeats for outer parallel_* call\n"); - printf(" thread_repeat: number of repeats for TeamThreadRange parallel_* call\n"); - printf(" vector_repeat: number of repeats for ThreadVectorRange parallel_* call\n"); + printf( + " thread_repeat: number of repeats for TeamThreadRange parallel_* " + "call\n"); + printf( + " vector_repeat: number of repeats for ThreadVectorRange parallel_* " + "call\n"); printf(" team_size: number of team members (team_size)\n"); printf(" vector_size: desired vectorization (if possible)\n"); printf(" schedule: 1 == Static 2 == Dynamic\n"); - printf(" test_type: 3-digit code XYZ for testing (nested) parallel_*\n"); - printf(" code key: XYZ X in {1,2,3,4,5}, Y in {0,1,2}, Z in {0,1,2}\n"); + printf( + " test_type: 3-digit code XYZ for testing (nested) parallel_*\n"); + printf( + " code key: XYZ X in {1,2,3,4,5}, Y in {0,1,2}, Z in " + "{0,1,2}\n"); printf(" TeamPolicy:\n"); - printf(" X: 0 = none (never used, makes no sense); 1 = parallel_for; 2 = parallel_reduce\n"); - printf(" Y: 0 = none; 1 = parallel_for; 2 = parallel_reduce\n"); - printf(" Z: 0 = none; 1 = parallel_for; 2 = parallel_reduce\n"); + printf( + " X: 0 = none (never used, makes no sense); 1 = " + "parallel_for; 2 = parallel_reduce\n"); + printf( + " Y: 0 = none; 1 = parallel_for; 2 = " + "parallel_reduce\n"); + printf( + " Z: 0 = none; 1 = parallel_for; 2 = " + "parallel_reduce\n"); printf(" RangePolicy:\n"); - printf(" X: 3 = parallel_for; 4 = parallel_reduce; 5 = parallel_scan\n"); + printf( + " X: 3 = parallel_for; 4 = parallel_reduce; 5 = " + "parallel_scan\n"); printf(" Y: 0 = none\n"); printf(" Z: 0 = none\n"); printf(" Example Input:\n"); - printf(" 100000 32 32 100 100 100 8 1 1 100\n"); + printf(" 100000 32 32 100 100 100 8 1 1 100\n"); Kokkos::finalize(); return 0; } - int team_range = atoi(argv[1]); + int team_range = atoi(argv[1]); int thread_range = atoi(argv[2]); int vector_range = atoi(argv[3]); - int outer_repeat = atoi(argv[4]); + int outer_repeat = atoi(argv[4]); int thread_repeat = atoi(argv[5]); int vector_repeat = atoi(argv[6]); - int team_size = atoi(argv[7]); + int team_size = atoi(argv[7]); int vector_size = atoi(argv[8]); - int schedule = atoi(argv[9]); - int test_type = atoi(argv[10]); + int schedule = atoi(argv[9]); + int test_type = atoi(argv[10]); - int disable_verbose_output = 0; - if ( argc > 11 ) { + int disable_verbose_output = 0; + if (argc > 11) { disable_verbose_output = atoi(argv[11]); } - if ( schedule != 1 && schedule != 2 ) { + if (schedule != 1 && schedule != 2) { printf("schedule: %d\n", schedule); printf("Options for schedule are: 1 == Static 2 == Dynamic\n"); Kokkos::finalize(); return -1; } - if ( test_type != 100 && test_type != 110 && test_type != 111 && test_type != 112 && test_type != 120 && test_type != 121 && test_type != 122 - && test_type != 200 && test_type != 210 && test_type != 211 && test_type != 212 && test_type != 220 && test_type != 221 && test_type != 222 - && test_type != 300 && test_type != 400 && test_type != 500 - ) - { + if (test_type != 100 && test_type != 110 && test_type != 111 && + test_type != 112 && test_type != 120 && test_type != 121 && + test_type != 122 && test_type != 200 && test_type != 210 && + test_type != 211 && test_type != 212 && test_type != 220 && + test_type != 221 && test_type != 222 && test_type != 300 && + test_type != 400 && test_type != 500) { printf("Incorrect test_type option\n"); Kokkos::finalize(); return -2; @@ -112,56 +132,85 @@ int main(int argc, char* argv[] ) { double result = 0.0; - Kokkos::parallel_reduce( "parallel_reduce warmup", Kokkos::TeamPolicy<>(10,1), - KOKKOS_LAMBDA(const Kokkos::TeamPolicy<>::member_type team, double& lval) { - lval += 1; - }, result); + Kokkos::parallel_reduce( + "parallel_reduce warmup", Kokkos::TeamPolicy<>(10, 1), + KOKKOS_LAMBDA(const Kokkos::TeamPolicy<>::member_type team, + double& lval) { lval += 1; }, + result); - typedef Kokkos::View view_type_1d; - typedef Kokkos::View view_type_2d; + typedef Kokkos::View view_type_1d; + typedef Kokkos::View view_type_2d; typedef Kokkos::View view_type_3d; // Allocate view without initializing - // Call a 'warmup' test with 1 repeat - this will initialize the corresponding view appropriately for test and should obey first-touch etc - // Second call to test is the one we actually care about and time - view_type_1d v_1( Kokkos::ViewAllocateWithoutInitializing("v_1"), team_range*team_size); - view_type_2d v_2( Kokkos::ViewAllocateWithoutInitializing("v_2"), team_range*team_size, thread_range); - view_type_3d v_3( Kokkos::ViewAllocateWithoutInitializing("v_3"), team_range*team_size, thread_range, vector_range); + // Call a 'warmup' test with 1 repeat - this will initialize the corresponding + // view appropriately for test and should obey first-touch etc Second call to + // test is the one we actually care about and time + view_type_1d v_1(Kokkos::ViewAllocateWithoutInitializing("v_1"), + team_range * team_size); + view_type_2d v_2(Kokkos::ViewAllocateWithoutInitializing("v_2"), + team_range * team_size, thread_range); + view_type_3d v_3(Kokkos::ViewAllocateWithoutInitializing("v_3"), + team_range * team_size, thread_range, vector_range); double result_computed = 0.0; - double result_expect = 0.0; - double time = 0.0; + double result_expect = 0.0; + double time = 0.0; - if(schedule==1) { - if ( test_type != 500 ) { + if (schedule == 1) { + if (test_type != 500) { // warmup - no repeat of loops - test_policy,int>(team_range,thread_range,vector_range,1,1,1,team_size,vector_size,test_type,v_1,v_2,v_3,result_computed,result_expect,time); - test_policy,int>(team_range,thread_range,vector_range,outer_repeat,thread_repeat,vector_repeat,team_size,vector_size,test_type,v_1,v_2,v_3,result_computed,result_expect,time); - } - else { + test_policy, int>( + team_range, thread_range, vector_range, 1, 1, 1, team_size, + vector_size, test_type, v_1, v_2, v_3, result_computed, result_expect, + time); + test_policy, int>( + team_range, thread_range, vector_range, outer_repeat, thread_repeat, + vector_repeat, team_size, vector_size, test_type, v_1, v_2, v_3, + result_computed, result_expect, time); + } else { // parallel_scan: initialize 1d view for parallel_scan - test_policy,int>(team_range,thread_range,vector_range,1,1,1,team_size,vector_size,100,v_1,v_2,v_3,result_computed,result_expect,time); - test_policy,int>(team_range,thread_range,vector_range,outer_repeat,thread_repeat,vector_repeat,team_size,vector_size,test_type,v_1,v_2,v_3,result_computed,result_expect,time); + test_policy, int>( + team_range, thread_range, vector_range, 1, 1, 1, team_size, + vector_size, 100, v_1, v_2, v_3, result_computed, result_expect, + time); + test_policy, int>( + team_range, thread_range, vector_range, outer_repeat, thread_repeat, + vector_repeat, team_size, vector_size, test_type, v_1, v_2, v_3, + result_computed, result_expect, time); } } - if(schedule==2) { - if ( test_type != 500 ) { + if (schedule == 2) { + if (test_type != 500) { // warmup - no repeat of loops - test_policy,int>(team_range,thread_range,vector_range,1,1,1,team_size,vector_size,test_type,v_1,v_2,v_3,result_computed,result_expect,time); - test_policy,int>(team_range,thread_range,vector_range,outer_repeat,thread_repeat,vector_repeat,team_size,vector_size,test_type,v_1,v_2,v_3,result_computed,result_expect,time); - } - else { + test_policy, int>( + team_range, thread_range, vector_range, 1, 1, 1, team_size, + vector_size, test_type, v_1, v_2, v_3, result_computed, result_expect, + time); + test_policy, int>( + team_range, thread_range, vector_range, outer_repeat, thread_repeat, + vector_repeat, team_size, vector_size, test_type, v_1, v_2, v_3, + result_computed, result_expect, time); + } else { // parallel_scan: initialize 1d view for parallel_scan - test_policy,int>(team_range,thread_range,vector_range,1,1,1,team_size,vector_size,100,v_1,v_2,v_3,result_computed,result_expect,time); - test_policy,int>(team_range,thread_range,vector_range,outer_repeat,thread_repeat,vector_repeat,team_size,vector_size,test_type,v_1,v_2,v_3,result_computed,result_expect,time); + test_policy, int>( + team_range, thread_range, vector_range, 1, 1, 1, team_size, + vector_size, 100, v_1, v_2, v_3, result_computed, result_expect, + time); + test_policy, int>( + team_range, thread_range, vector_range, outer_repeat, thread_repeat, + vector_repeat, team_size, vector_size, test_type, v_1, v_2, v_3, + result_computed, result_expect, time); } } - if ( disable_verbose_output == 0 ) { - printf("%7i %4i %2i %9i %4i %4i %4i %2i %1i %3i %e %e %lf\n",team_range,thread_range,vector_range,outer_repeat,thread_repeat,vector_repeat,team_size,vector_size,schedule,test_type,result_computed,result_expect,time); - } - else { - printf("%lf\n",time); + if (disable_verbose_output == 0) { + printf("%7i %4i %2i %9i %4i %4i %4i %2i %1i %3i %e %e %lf\n", team_range, + thread_range, vector_range, outer_repeat, thread_repeat, + vector_repeat, team_size, vector_size, schedule, test_type, + result_computed, result_expect, time); + } else { + printf("%lf\n", time); } Kokkos::finalize(); diff --git a/lib/kokkos/benchmarks/policy_performance/policy_perf_test.hpp b/lib/kokkos/benchmarks/policy_performance/policy_perf_test.hpp index 1ab437928de761bed614efc8a22d8c4d38fcf38f..7a1500891f7af1d2ff5ce61ebb83637d20c43cb4 100644 --- a/lib/kokkos/benchmarks/policy_performance/policy_perf_test.hpp +++ b/lib/kokkos/benchmarks/policy_performance/policy_perf_test.hpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -43,297 +44,375 @@ #include -template < class ViewType > +template struct ParallelScanFunctor { using value_type = double; ViewType v; - ParallelScanFunctor( const ViewType & v_ ) - : v(v_) - {} + ParallelScanFunctor(const ViewType& v_) : v(v_) {} KOKKOS_INLINE_FUNCTION - void operator()( const int idx, value_type& val, const bool& final ) const - { - // inclusive scan - val += v(idx); - if ( final ) { - v(idx) = val; - } + void operator()(const int idx, value_type& val, const bool& final) const { + // inclusive scan + val += v(idx); + if (final) { + v(idx) = val; } + } }; -template +template void test_policy(int team_range, int thread_range, int vector_range, - int outer_repeat, int thread_repeat, int inner_repeat, - int team_size, int vector_size, int test_type, - ViewType1 &v1, ViewType2 &v2, ViewType3 &v3, - double &result, double &result_expect, double &time) { - - typedef Kokkos::TeamPolicy t_policy; + int outer_repeat, int thread_repeat, int inner_repeat, + int team_size, int vector_size, int test_type, ViewType1& v1, + ViewType2& v2, ViewType3& v3, double& result, + double& result_expect, double& time) { + typedef Kokkos::TeamPolicy t_policy; typedef typename t_policy::member_type t_team; Kokkos::Timer timer; - for(int orep = 0; orep(v1) + Kokkos::parallel_scan("500 outer scan", team_size * team_range, + ParallelScanFunctor(v1) #if 0 // This does not compile with pre Cuda 8.0 - see Github Issue #913 for explanation KOKKOS_LAMBDA (const int idx, double& val, const bool& final) { @@ -345,11 +424,12 @@ void test_policy(int team_range, int thread_range, int vector_range, } #endif ); - // result = v1( team_size*team_range - 1 ); // won't work with Cuda - need to copy result back to host to print - // result_expect = 0.5*(team_size*team_range)*(team_size*team_range-1); + // result = v1( team_size*team_range - 1 ); // won't work with Cuda - need + // to copy result back to host to print result_expect = + // 0.5*(team_size*team_range)*(team_size*team_range-1); } - } // end outer for loop + } // end outer for loop time = timer.seconds(); -} //end test_policy +} // end test_policy diff --git a/lib/kokkos/benchmarks/stream/stream-kokkos.cc b/lib/kokkos/benchmarks/stream/stream-kokkos.cc index 370995432e5833490b5cd7b42331d676ef5b8f63..6ce789dd82049a81bf9ed2ff5ba2318093c948af 100644 --- a/lib/kokkos/benchmarks/stream/stream-kokkos.cc +++ b/lib/kokkos/benchmarks/stream/stream-kokkos.cc @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR diff --git a/lib/kokkos/bin/nvcc_wrapper b/lib/kokkos/bin/nvcc_wrapper index 94bc72854e74211cd3ed1c2876100731311dddfe..8a23d0d620508529bef99f0e74b80a2a0093cb6f 100755 --- a/lib/kokkos/bin/nvcc_wrapper +++ b/lib/kokkos/bin/nvcc_wrapper @@ -85,11 +85,11 @@ first_xcompiler_arg=1 temp_dir=${TMPDIR:-/tmp} -# Check if we have an optimization argument already -optimization_applied=0 +# optimization flag added as a command-line argument +optimization_flag="" -# Check if we have -std=c++X or --std=c++X already -stdcxx_applied=0 +# std standard flag added as a command-line argument +std_flag="" # Run nvcc a second time to generate dependencies if needed depfile_separate=0 @@ -99,6 +99,10 @@ depfile_target_arg="" # Option to remove duplicate libraries and object files remove_duplicate_link_files=0 +function warn_std_flag() { + echo "nvcc_wrapper - *warning* you have set multiple standard flags (-std=c++1* or --std=c++1*), only the last is used because nvcc can only accept a single std setting" +} + #echo "Arguments: $# $@" while [ $# -gt 0 ] @@ -130,12 +134,16 @@ do ;; # Ensure we only have one optimization flag because NVCC doesn't allow muliple -O*) - if [ $optimization_applied -eq 1 ]; then - echo "nvcc_wrapper - *warning* you have set multiple optimization flags (-O*), only the first is used because nvcc can only accept a single optimization setting." + if [ -n "$optimization_flag" ]; then + echo "nvcc_wrapper - *warning* you have set multiple optimization flags (-O*), only the last is used because nvcc can only accept a single optimization setting." + shared_args=${shared_args/ $optimization_flag/} + fi + if [ "$1" = "-O" ]; then + optimization_flag="-O2" else - shared_args="$shared_args $1" - optimization_applied=1 + optimization_flag=$1 fi + shared_args="$shared_args $optimization_flag" ;; #Handle shared args (valid for both nvcc and the host compiler) -D*) @@ -171,7 +179,7 @@ do shift ;; #Handle known nvcc args - --dryrun|--verbose|--keep|--keep-dir*|-G|--relocatable-device-code*|-lineinfo|-expt-extended-lambda|--resource-usage|-Xptxas*) + --dryrun|--verbose|--keep|--keep-dir*|-G|--relocatable-device-code*|-lineinfo|-expt-extended-lambda|--resource-usage|-Xptxas*|--fmad*) cuda_args="$cuda_args $1" ;; #Handle more known nvcc args @@ -179,21 +187,43 @@ do cuda_args="$cuda_args $1" ;; #Handle known nvcc args that have an argument - -rdc|-maxrregcount|--default-stream) + -rdc|-maxrregcount|--default-stream|-Xnvlink|--fmad) cuda_args="$cuda_args $1 $2" shift ;; -rdc=*|-maxrregcount*|--maxrregcount*) cuda_args="$cuda_args $1" ;; - #Handle c++11 - --std=c++11|-std=c++11|--std=c++14|-std=c++14|--std=c++1y|-std=c++1y|--std=c++17|-std=c++17|--std=c++1z|-std=c++1z) - if [ $stdcxx_applied -eq 1 ]; then - echo "nvcc_wrapper - *warning* you have set multiple optimization flags (-std=c++1* or --std=c++1*), only the first is used because nvcc can only accept a single std setting" - else - shared_args="$shared_args $1" - stdcxx_applied=1 + #Handle unsupported standard flags + --std=c++1y|-std=c++1y|--std=c++1z|-std=c++1z|--std=gnu++1y|-std=gnu++1y|--std=gnu++1z|-std=gnu++1z|--std=c++2a|-std=c++2a|--std=c++17|-std=c++17) + fallback_std_flag="-std=c++14" + # this is hopefully just occurring in a downstream project during CMake feature tests + # we really have no choice here but to accept the flag and change to an accepted C++ standard + echo "nvcc_wrapper does not accept standard flags $1 since partial standard flags and standards after C++14 are not supported. nvcc_wrapper will use $fallback_std_flag instead. It is undefined behavior to use this flag. This should only be occurring during CMake configuration." + if [ -n "$std_flag" ]; then + warn_std_flag + shared_args=${shared_args/ $std_flag/} + fi + std_flag=$fallback_std_flag + shared_args="$shared_args $std_flag" + ;; + -std=gnu*) + corrected_std_flag=${1/gnu/c} + echo "nvcc_wrapper has been given GNU extension standard flag $1 - reverting flag to $corrected_std_flag" + if [ -n "$std_flag" ]; then + warn_std_flag + shared_args=${shared_args/ $std_flag/} + fi + std_flag=$corrected_std_flag + shared_args="$shared_args $std_flag" + ;; + --std=c++11|-std=c++11|--std=c++14|-std=c++14) + if [ -n "$std_flag" ]; then + warn_std_flag + shared_args=${shared_args/ $std_flag/} fi + std_flag=$1 + shared_args="$shared_args $std_flag" ;; #strip of -std=c++98 due to nvcc warnings and Tribits will place both -std=c++11 and -std=c++98 @@ -308,16 +338,6 @@ do shift done -#Check if nvcc exists -if [ $host_only -ne 1 ]; then - var=$(which nvcc ) - if [ $? -gt 0 ]; then - echo "Could not find nvcc in PATH" - exit $? - fi -fi - - # Only print host compiler version if [ $get_host_version -eq 1 ]; then $host_compiler --version @@ -372,6 +392,9 @@ if [ $first_xcompiler_arg -eq 0 ]; then nvcc_command="$nvcc_command -Xcompiler $xcompiler_args" fi +#Replace all commas in xcompiler_args with a space for the host only command +xcompiler_args=${xcompiler_args//,/" "} + #Compose host only command host_command="$host_compiler $shared_args $host_only_args $compile_arg $output_arg $xcompiler_args $host_linker_args $shared_versioned_libraries_host" diff --git a/lib/kokkos/cm_generate_makefile.bash b/lib/kokkos/cm_generate_makefile.bash new file mode 100755 index 0000000000000000000000000000000000000000..fd7cfe2d325bb817aeeaa0027fcb99d0169de95c --- /dev/null +++ b/lib/kokkos/cm_generate_makefile.bash @@ -0,0 +1,339 @@ +#!/bin/bash + +update_kokkos_devices() { + SEARCH_TEXT="*$1*" + if [[ $KOKKOS_DEVICES == $SEARCH_TEXT ]]; then + echo kokkos devices already includes $SEARCH_TEXT + else + if [ "$KOKKOS_DEVICES" = "" ]; then + KOKKOS_DEVICES="$1" + echo reseting kokkos devices to $KOKKOS_DEVICES + else + KOKKOS_DEVICES="${KOKKOS_DEVICES},$1" + echo appending to kokkos devices $KOKKOS_DEVICES + fi + fi +} + +get_kokkos_device_list() { + KOKKOS_DEVICE_CMD= + PARSE_DEVICES_LST=$(echo $KOKKOS_DEVICES | tr "," "\n") + for DEVICE_ in $PARSE_DEVICES_LST + do + UC_DEVICE=$(echo $DEVICE_ | tr "[:lower:]" "[:upper:]") + KOKKOS_DEVICE_CMD="-DKokkos_ENABLE_${UC_DEVICE}=ON ${KOKKOS_DEVICE_CMD}" + done +} + +get_kokkos_arch_list() { + KOKKOS_ARCH_CMD= + PARSE_ARCH_LST=$(echo $KOKKOS_ARCH | tr "," "\n") + for ARCH_ in $PARSE_ARCH_LST + do + UC_ARCH=$(echo $ARCH_ | tr "[:lower:]" "[:upper:]") + KOKKOS_ARCH_CMD="-DKokkos_ARCH_${UC_ARCH}=ON ${KOKKOS_ARCH_CMD}" + done +} + +get_kokkos_cuda_option_list() { + echo parsing KOKKOS_CUDA_OPTIONS=$KOKKOS_CUDA_OPTIONS + KOKKOS_CUDA_OPTION_CMD= + PARSE_CUDA_LST=$(echo $KOKKOS_CUDA_OPTIONS | tr "," "\n") + for CUDA_ in $PARSE_CUDA_LST + do + CUDA_OPT_NAME= + if [ "${CUDA_}" == "enable_lambda" ]; then + CUDA_OPT_NAME=CUDA_LAMBDA + elif [ "${CUDA_}" == "rdc" ]; then + CUDA_OPT_NAME=CUDA_RELOCATABLE_DEVICE_CODE + elif [ "${CUDA_}" == "force_uvm" ]; then + CUDA_OPT_NAME=CUDA_UVM + elif [ "${CUDA_}" == "use_ldg" ]; then + CUDA_OPT_NAME=CUDA_LDG_INTRINSIC + else + echo "${CUDA_} is not a valid cuda options..." + fi + if [ "${CUDA_OPT_NAME}" != "" ]; then + KOKKOS_CUDA_OPTION_CMD="-DKokkos_ENABLE_${CUDA_OPT_NAME}=ON ${KOKKOS_CUDA_OPTION_CMD}" + fi + done +} + +get_kokkos_option_list() { + echo parsing KOKKOS_OPTIONS=$KOKKOS_OPTIONS + KOKKOS_OPTION_CMD= + PARSE_OPTIONS_LST=$(echo $KOKKOS_OPTIONS | tr "," "\n") + for OPT_ in $PARSE_OPTIONS_LST + do + UC_OPT_=$(echo $OPT_ | tr "[:lower:]" "[:upper:]") + if [[ "$UC_OPT_" == *DISABLE* ]]; then + FLIP_OPT_=${UC_OPT_/DISABLE/ENABLE} + KOKKOS_OPTION_CMD="-DKokkos_${FLIP_OPT_}=OFF ${KOKKOS_OPTION_CMD}" + elif [[ "$UC_OPT_" == *ENABLE* ]]; then + KOKKOS_OPTION_CMD="-DKokkos_${UC_OPT_}=ON ${KOKKOS_OPTION_CMD}" + else + KOKKOS_OPTION_CMD="-DKokkos_ENABLE_${UC_OPT_}=ON ${KOKKOS_OPTION_CMD}" + fi + done +} + +display_help_text() { + + echo "Kokkos configure options:" + echo "" + echo "--kokkos-path=/Path/To/Kokkos: Path to the Kokkos root directory." + echo "--prefix=/Install/Path: Path to install the Kokkos library." + echo "" + echo "--with-cuda[=/Path/To/Cuda]: Enable Cuda and set path to Cuda Toolkit." + echo "--with-openmp: Enable OpenMP backend." + echo "--with-pthread: Enable Pthreads backend." + echo "--with-serial: Enable Serial backend." + echo "--with-devices: Explicitly add a set of backends." + echo "" + echo "--arch=[OPT]: Set target architectures. Options are:" + echo " [AMD]" + echo " AMDAVX = AMD CPU" + echo " EPYC = AMD EPYC Zen-Core CPU" + echo " [ARM]" + echo " ARMv80 = ARMv8.0 Compatible CPU" + echo " ARMv81 = ARMv8.1 Compatible CPU" + echo " ARMv8-ThunderX = ARMv8 Cavium ThunderX CPU" + echo " ARMv8-TX2 = ARMv8 Cavium ThunderX2 CPU" + echo " [IBM]" + echo " BGQ = IBM Blue Gene Q" + echo " Power7 = IBM POWER7 and POWER7+ CPUs" + echo " Power8 = IBM POWER8 CPUs" + echo " Power9 = IBM POWER9 CPUs" + echo " [Intel]" + echo " WSM = Intel Westmere CPUs" + echo " SNB = Intel Sandy/Ivy Bridge CPUs" + echo " HSW = Intel Haswell CPUs" + echo " BDW = Intel Broadwell Xeon E-class CPUs" + echo " SKX = Intel Sky Lake Xeon E-class HPC CPUs (AVX512)" + echo " [Intel Xeon Phi]" + echo " KNC = Intel Knights Corner Xeon Phi" + echo " KNL = Intel Knights Landing Xeon Phi" + echo " [NVIDIA]" + echo " Kepler30 = NVIDIA Kepler generation CC 3.0" + echo " Kepler32 = NVIDIA Kepler generation CC 3.2" + echo " Kepler35 = NVIDIA Kepler generation CC 3.5" + echo " Kepler37 = NVIDIA Kepler generation CC 3.7" + echo " Maxwell50 = NVIDIA Maxwell generation CC 5.0" + echo " Maxwell52 = NVIDIA Maxwell generation CC 5.2" + echo " Maxwell53 = NVIDIA Maxwell generation CC 5.3" + echo " Pascal60 = NVIDIA Pascal generation CC 6.0" + echo " Pascal61 = NVIDIA Pascal generation CC 6.1" + echo " Volta70 = NVIDIA Volta generation CC 7.0" + echo " Volta72 = NVIDIA Volta generation CC 7.2" + echo "" + echo "--compiler=/Path/To/Compiler Set the compiler." + echo "--debug,-dbg: Enable Debugging." + echo "--cxxflags=[FLAGS] Overwrite CXXFLAGS for library build and test" + echo " build. This will still set certain required" + echo " flags via KOKKOS_CXXFLAGS (such as -fopenmp," + echo " --std=c++11, etc.)." + echo "--cxxstandard=[FLAGS] Overwrite KOKKOS_CXX_STANDARD for library build and test" + echo " c++11 (default), c++14, c++17, c++1y, c++1z, c++2a" + echo "--ldflags=[FLAGS] Overwrite LDFLAGS for library build and test" + echo " build. This will still set certain required" + echo " flags via KOKKOS_LDFLAGS (such as -fopenmp," + echo " -lpthread, etc.)." + echo "--with-gtest=/Path/To/Gtest: Set path to gtest. (Used in unit and performance" + echo " tests.)" + echo "--with-hwloc=/Path/To/Hwloc: Set path to hwloc library." + echo "--with-memkind=/Path/To/MemKind: Set path to memkind library." + echo "--with-options=[OPT]: Additional options to Kokkos:" + echo " compiler_warnings" + echo " aggressive_vectorization = add ivdep on loops" + echo " disable_profiling = do not compile with profiling hooks" + echo " " + echo "--with-cuda-options=[OPT]: Additional options to CUDA:" + echo " force_uvm, use_ldg, enable_lambda, rdc" + echo "--with-hpx-options=[OPT]: Additional options to HPX:" + echo " enable_async_dispatch" + echo "--gcc-toolchain=/Path/To/GccRoot: Set the gcc toolchain to use with clang (e.g. /usr)" + echo "--make-j=[NUM]: DEPRECATED: call make with appropriate" + echo " -j flag" + +} + +while [[ $# > 0 ]] +do + key="$1" + + case $key in + --kokkos-path*) + KOKKOS_PATH="${key#*=}" + ;; + --hpx-path*) + HPX_PATH="${key#*=}" + ;; + --prefix*) + PREFIX="${key#*=}" + ;; + --with-cuda) + update_kokkos_devices Cuda + CUDA_PATH_NVCC=$(command -v nvcc) + CUDA_PATH=${CUDA_PATH_NVCC%/bin/nvcc} + ;; + # Catch this before '--with-cuda*' + --with-cuda-options*) + KOKKOS_CUDA_OPTIONS="${key#*=}" + ;; + --with-cuda*) + update_kokkos_devices Cuda + CUDA_PATH="${key#*=}" + ;; + --with-openmp) + update_kokkos_devices OpenMP + ;; + --with-pthread) + update_kokkos_devices Pthread + ;; + --with-serial) + update_kokkos_devices Serial + ;; + --with-hpx-options*) + KOKKOS_HPX_OPT="${key#*=}" + ;; + --with-hpx*) + update_kokkos_devices HPX + if [ -z "$HPX_PATH" ]; then + HPX_PATH="${key#*=}" + fi + ;; + --with-devices*) + DEVICES="${key#*=}" + PARSE_DEVICES=$(echo $DEVICES | tr "," "\n") + for DEVICE_ in $PARSE_DEVICES + do + update_kokkos_devices $DEVICE_ + done + ;; + --with-gtest*) + GTEST_PATH="${key#*=}" + ;; + --with-hwloc*) + HWLOC_PATH="${key#*=}" + ;; + --with-memkind*) + MEMKIND_PATH="${key#*=}" + ;; + --arch*) + KOKKOS_ARCH="${key#*=}" + ;; + --cxxflags*) + KOKKOS_CXXFLAGS="${key#*=}" + KOKKOS_CXXFLAGS=${KOKKOS_CXXFLAGS//,/ } + ;; + --cxxstandard*) + KOKKOS_CXX_STANDARD="${key#*=}" + ;; + --ldflags*) + KOKKOS_LDFLAGS="${key#*=}" + ;; + --debug|-dbg) + KOKKOS_DEBUG=yes + ;; + --make-j*) + echo "Warning: ${key} is deprecated" + echo "Call make with appropriate -j flag" + ;; + --compiler*) + COMPILER="${key#*=}" + CNUM=$(command -v ${COMPILER} 2>&1 >/dev/null | grep "no ${COMPILER}" | wc -l) + if [ ${CNUM} -gt 0 ]; then + echo "Invalid compiler by --compiler command: '${COMPILER}'" + exit + fi + if [[ ! -n ${COMPILER} ]]; then + echo "Empty compiler specified by --compiler command." + exit + fi + CNUM=$(command -v ${COMPILER} | grep ${COMPILER} | wc -l) + if [ ${CNUM} -eq 0 ]; then + echo "Invalid compiler by --compiler command: '${COMPILER}'" + exit + fi + # ... valid compiler, ensure absolute path set + WCOMPATH=$(command -v $COMPILER) + COMPDIR=$(dirname $WCOMPATH) + COMPNAME=$(basename $WCOMPATH) + COMPILER=${COMPDIR}/${COMPNAME} + ;; + --with-options*) + KOKKOS_OPTIONS="${key#*=}" + ;; + --gcc-toolchain*) + KOKKOS_GCC_TOOLCHAIN="${key#*=}" + ;; + --help) + display_help_text + exit 0 + ;; + *) + echo "warning: ignoring unknown option $key" + ;; + esac + + shift +done + + +if [ "$COMPILER" == "" ]; then + COMPILER_CMD= +else + COMPILER_CMD=-DCMAKE_CXX_COMPILER=$COMPILER +fi + +if [ "$KOKKOS_DEBUG" == "" ]; then + KOKKOS_DEBUG_CMD=-DCMAKE_BUILD_TYPE=RELEASE +else + KOKKOS_DEBUG_CMD=-DCMAKE_BUILD_TYPE=DEBUG +fi + +if [ ! -e ${KOKKOS_PATH}/CMakeLists.txt ]; then + if [ "${KOKKOS_PATH}" == "" ]; then + CM_SCRIPT=$0 + KOKKOS_PATH=`dirname $CM_SCRIPT` + if [ ! -e ${KOKKOS_PATH}/CMakeLists.txt ]; then + echo "${KOKKOS_PATH} repository appears to not be complete. please verify and try again" + exit 0 + fi + else + echo "KOKKOS_PATH does not appear to be set properly. please specify in location of CMakeLists.txt" + display_help_text + exit 0 + fi +fi + +get_kokkos_device_list +get_kokkos_option_list +get_kokkos_arch_list +get_kokkos_cuda_option_list + +## if HPX is enabled, we need to enforce cxx standard = 14 +if [[ ${KOKKOS_DEVICE_CMD} == *Kokkos_ENABLE_HPX* ]]; then + if [ "${KOKKOS_CXX_STANDARD}" == "" ] || [ ${#KOKKOS_CXX_STANDARD} -lt 14 ]; then + echo CXX Standard must be 14 or higher for HPX to work. + KOKKOS_CXX_STANDARD=14 + fi +fi + +if [ "$KOKKOS_CXX_STANDARD" == "" ]; then + STANDARD_CMD= +else + STANDARD_CMD=-DKokkos_CXX_STANDARD=${KOKKOS_CXX_STANDARD} +fi + +if [[ ${COMPILER} == *clang* ]]; then + gcc_path=$(which g++ | awk --field-separator='/bin/g++' '{printf $1}' ) + KOKKOS_CXXFLAGS="${KOKKOS_CXXFLAGS} --gcc-toolchain=${gcc_path}" + + if [ ! "${CUDA_PATH}" == "" ]; then + KOKKOS_CXXFLAGS="${KOKKOS_CXXFLAGS} --cuda-path=${CUDA_PATH}" + fi +fi + +echo cmake $COMPILER_CMD -DCMAKE_CXX_FLAGS="${KOKKOS_CXXFLAGS}" -DCMAKE_EXE_LINKER_FLAGS="${KOKKOS_LDFLAGS}" -DCMAKE_INSTALL_PREFIX=${PREFIX} ${KOKKOS_DEVICE_CMD} ${KOKKOS_ARCH_CMD} -DKokkos_ENABLE_TESTS=ON ${KOKKOS_OPTION_CMD} ${KOKKOS_CUDA_OPTION_CMD} -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_CXX_EXTENSIONS=OFF ${STANDARD_CMD} ${KOKKOS_DEBUG_CMD} ${KOKKOS_PATH} +cmake $COMPILER_CMD -DCMAKE_CXX_FLAGS="${KOKKOS_CXXFLAGS//\"}" -DCMAKE_EXE_LINKER_FLAGS="${KOKKOS_LDFLAGS//\"}" -DCMAKE_INSTALL_PREFIX=${PREFIX} ${KOKKOS_DEVICE_CMD} ${KOKKOS_ARCH_CMD} -DKokkos_ENABLE_TESTS=ON ${KOKKOS_OPTION_CMD} ${KOKKOS_CUDA_OPTION_CMD} -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_CXX_EXTENSIONS=OFF ${STANDARD_CMD} ${KOKKOS_DEBUG_CMD} ${KOKKOS_PATH} diff --git a/lib/kokkos/cmake/KokkosConfig.cmake.in b/lib/kokkos/cmake/KokkosConfig.cmake.in index fc099a494ce25af9068e113a688904e06458fcbe..6f4607687e066b3a1bee5d467803d0f39c99810c 100644 --- a/lib/kokkos/cmake/KokkosConfig.cmake.in +++ b/lib/kokkos/cmake/KokkosConfig.cmake.in @@ -1,18 +1,14 @@ -# - Config file for the Kokkos package -# It defines the following variables -# Kokkos_INCLUDE_DIRS - include directories for Kokkos -# Kokkos_LIBRARIES - libraries to link against - # Compute paths -GET_FILENAME_COMPONENT(Kokkos_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) -SET(Kokkos_INCLUDE_DIRS "@CONF_INCLUDE_DIRS@") +@PACKAGE_INIT@ -# Our library dependencies (contains definitions for IMPORTED targets) -IF(NOT TARGET kokkos AND NOT Kokkos_BINARY_DIR) - INCLUDE("${Kokkos_CMAKE_DIR}/KokkosTargets.cmake") -ENDIF() +#Find dependencies +INCLUDE(CMakeFindDependencyMacro) -# These are IMPORTED targets created by KokkosTargets.cmake -SET(Kokkos_LIBRARY_DIRS @INSTALL_LIB_DIR@) -SET(Kokkos_LIBRARIES @Kokkos_LIBRARIES_NAMES@) -SET(Kokkos_TPL_LIBRARIES @KOKKOS_LIBS@) +#This needs to go above the KokkosTargets in case +#the Kokkos targets depend in some way on the TPL imports +@KOKKOS_TPL_EXPORTS@ + +GET_FILENAME_COMPONENT(Kokkos_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) +INCLUDE("${Kokkos_CMAKE_DIR}/KokkosTargets.cmake") +INCLUDE("${Kokkos_CMAKE_DIR}/KokkosConfigCommon.cmake") +UNSET(Kokkos_CMAKE_DIR) diff --git a/lib/kokkos/cmake/KokkosConfigCommon.cmake.in b/lib/kokkos/cmake/KokkosConfigCommon.cmake.in new file mode 100644 index 0000000000000000000000000000000000000000..da9c61976c596e7318bd7a055ced6c6b527d2f60 --- /dev/null +++ b/lib/kokkos/cmake/KokkosConfigCommon.cmake.in @@ -0,0 +1,87 @@ +SET(Kokkos_DEVICES @KOKKOS_ENABLED_DEVICES@) +SET(Kokkos_OPTIONS @KOKKOS_ENABLED_OPTIONS@) +SET(Kokkos_TPLS @KOKKOS_ENABLED_TPLS@) +SET(Kokkos_ARCH @KOKKOS_ENABLED_ARCH_LIST@) + +# These are needed by KokkosKernels +FOREACH(DEV ${Kokkos_DEVICES}) + SET(Kokkos_ENABLE_${DEV} ON) +ENDFOREACH() + +IF(NOT Kokkos_FIND_QUIETLY) + MESSAGE(STATUS "Enabled Kokkos devices: ${Kokkos_DEVICES}") +ENDIF() + +IF (Kokkos_ENABLE_CUDA AND ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.14.0") + #If we are building CUDA, we have tricked CMake because we declare a CXX project + #If the default C++ standard for a given compiler matches the requested + #standard, then CMake just omits the -std flag in later versions of CMake + #This breaks CUDA compilation (CUDA compiler can have a different default + #-std then the underlying host compiler by itself). Setting this variable + #forces CMake to always add the -std flag even if it thinks it doesn't need it + SET(CMAKE_CXX_STANDARD_DEFAULT 98 CACHE INTERNAL "" FORCE) +ENDIF() + +SET(KOKKOS_USE_CXX_EXTENSIONS @KOKKOS_USE_CXX_EXTENSIONS@) +IF (NOT DEFINED CMAKE_CXX_EXTENSIONS OR CMAKE_CXX_EXTENSIONS) + IF (NOT KOKKOS_USE_CXX_EXTENSIONS) + MESSAGE(WARNING "The installed Kokkos configuration does not support CXX extensions. Forcing -DCMAKE_CXX_EXTENSIONS=Off") + SET(CMAKE_CXX_EXTENSIONS OFF CACHE BOOL "" FORCE) + ENDIF() +ENDIF() + +include(FindPackageHandleStandardArgs) + +# This function makes sure that Kokkos was built with the requested backends +# and target architectures and generates a fatal error if it was not. +# +# kokkos_check( +# [DEVICES ...] # Set of backends (e.g. "OpenMP" and/or "Cuda") +# [ARCH ...] # Target architectures (e.g. "Power9" and/or "Volta70") +# [OPTIONS ...] # Optional settings (e.g. "PROFILING") +# [TPLS ...] # Third party libraries +# [RETURN_VALUE ] # Set a variable that indicates the result of the +# # check instead of a fatal error +# ) +function(kokkos_check) + set(ALLOWED_ARGS DEVICES ARCH OPTIONS TPLS) + cmake_parse_arguments(KOKKOS_CHECK "" "RETURN_VALUE" "${ALLOWED_ARGS}" ${ARGN}) + foreach(_arg ${KOKKOS_CHECK_UNPARSED_ARGUMENTS}) + message(SEND_ERROR "Argument '${_arg}' passed to kokkos_check() was not recognized") + endforeach() + # Get the list of keywords that were actually passed to the function. + set(REQUESTED_ARGS) + foreach(arg ${ALLOWED_ARGS}) + if(KOKKOS_CHECK_${arg}) + list(APPEND REQUESTED_ARGS ${arg}) + endif() + endforeach() + set(KOKKOS_CHECK_SUCCESS TRUE) + foreach(arg ${REQUESTED_ARGS}) + # Define variables named after the required arguments that are provided by + # the Kokkos install. + foreach(requested ${KOKKOS_CHECK_${arg}}) + foreach(provided ${Kokkos_${arg}}) + STRING(TOUPPER ${requested} REQUESTED_UC) + STRING(TOUPPER ${provided} PROVIDED_UC) + if(PROVIDED_UC STREQUAL REQUESTED_UC) + string(REPLACE ";" " " ${requested} "${KOKKOS_CHECK_${arg}}") + endif() + endforeach() + endforeach() + # Somewhat divert the CMake function below from its original purpose and + # use it to check that there are variables defined for all required + # arguments. Success or failure messages will be displayed but we are + # responsible for signaling failure and skip the build system generation. + find_package_handle_standard_args("Kokkos_${arg}" DEFAULT_MSG + ${KOKKOS_CHECK_${arg}}) + if(NOT Kokkos_${arg}_FOUND) + set(KOKKOS_CHECK_SUCCESS FALSE) + endif() + endforeach() + if(NOT KOKKOS_CHECK_SUCCESS AND NOT KOKKOS_CHECK_RETURN_VALUE) + message(FATAL_ERROR "Kokkos does NOT provide all backends and/or architectures requested") + else() + set(${KOKKOS_CHECK_RETURN_VALUE} ${KOKKOS_CHECK_SUCCESS} PARENT_SCOPE) + endif() +endfunction() diff --git a/lib/kokkos/cmake/KokkosCore_config.h.in b/lib/kokkos/cmake/KokkosCore_config.h.in new file mode 100644 index 0000000000000000000000000000000000000000..084afba8a803b16031fdd6b1fb9e96c419d90fff --- /dev/null +++ b/lib/kokkos/cmake/KokkosCore_config.h.in @@ -0,0 +1,89 @@ + +#if !defined(KOKKOS_MACROS_HPP) || defined(KOKKOS_CORE_CONFIG_H) +#error "Do not include KokkosCore_config.h directly; include Kokkos_Macros.hpp instead." +#else +#define KOKKOS_CORE_CONFIG_H +#endif + +/* Execution Spaces */ +#cmakedefine KOKKOS_ENABLE_SERIAL +#cmakedefine KOKKOS_ENABLE_OPENMP +#cmakedefine KOKKOS_ENABLE_THREADS +#cmakedefine KOKKOS_ENABLE_CUDA +#cmakedefine KOKKOS_ENABLE_HPX +#cmakedefine KOKKOS_ENABLE_MEMKIND +#cmakedefine KOKKOS_ENABLE_LIBRT + +#ifndef __CUDA_ARCH__ +#cmakedefine KOKKOS_ENABLE_TM +#cmakedefine KOKKOS_USE_ISA_X86_64 +#cmakedefine KOKKOS_USE_ISA_KNC +#cmakedefine KOKKOS_USE_ISA_POWERPCLE +#cmakedefine KOKKOS_USE_ISA_POWERPCBE +#endif + +/* General Settings */ +#cmakedefine KOKKOS_ENABLE_CXX11 +#cmakedefine KOKKOS_ENABLE_CXX14 +#cmakedefine KOKKOS_ENABLE_CXX17 +#cmakedefine KOKKOS_ENABLE_CXX20 + +#cmakedefine KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE +#cmakedefine KOKKOS_ENABLE_CUDA_UVM +#cmakedefine KOKKOS_ENABLE_CUDA_LAMBDA +#cmakedefine KOKKOS_ENABLE_CUDA_CONSTEXPR +#cmakedefine KOKKOS_ENABLE_CUDA_LDG_INTRINSIC +#cmakedefine KOKKOS_ENABLE_HPX_ASYNC_DISPATCH +#cmakedefine KOKKOS_ENABLE_DEBUG +#cmakedefine KOKKOS_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK +#cmakedefine KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK +#cmakedefine KOKKOS_ENABLE_COMPILER_WARNINGS +#cmakedefine KOKKOS_ENABLE_PROFILING +#cmakedefine KOKKOS_ENABLE_PROFILING_LOAD_PRINT +#cmakedefine KOKKOS_ENABLE_DEPRECATED_CODE +#cmakedefine KOKKOS_ENABLE_ETI +#cmakedefine KOKKOS_ENABLE_LARGE_MEM_TESTS +#cmakedefine KOKKOS_ENABLE_DUALVIEW_MODIFY_CHECK +#cmakedefine KOKKOS_ENABLE_COMPLEX_ALIGN +#cmakedefine KOKKOS_OPT_RANGE_AGGRESSIVE_VECTORIZATION + +/* TPL Settings */ +#cmakedefine KOKKOS_ENABLE_HWLOC +#cmakedefine KOKKOS_USE_LIBRT +#cmakedefine KOKKOS_ENABLE_HWBSPACE + +#cmakedefine KOKKOS_IMPL_CUDA_CLANG_WORKAROUND + +#cmakedefine KOKKOS_COMPILER_CUDA_VERSION @KOKKOS_COMPILER_CUDA_VERSION@ + +#cmakedefine KOKKOS_ARCH_SSE42 +#cmakedefine KOKKOS_ARCH_ARMV80 +#cmakedefine KOKKOS_ARCH_ARMV8_THUNDERX +#cmakedefine KOKKOS_ARCH_ARMV81 +#cmakedefine KOKKOS_ARCH_ARMV8_THUNDERX2 +#cmakedefine KOKKOS_ARCH_AMD_AVX2 +#cmakedefine KOKKOS_ARCH_AVX +#cmakedefine KOKKOS_ARCH_AVX2 +#cmakedefine KOKKOS_ARCH_AVX512XEON +#cmakedefine KOKKOS_ARCH_KNC +#cmakedefine KOKKOS_ARCH_AVX512MIC +#cmakedefine KOKKOS_ARCH_POWER7 +#cmakedefine KOKKOS_ARCH_POWER8 +#cmakedefine KOKKOS_ARCH_POWER9 +#cmakedefine KOKKOS_ARCH_KEPLER +#cmakedefine KOKKOS_ARCH_KEPLER30 +#cmakedefine KOKKOS_ARCH_KEPLER32 +#cmakedefine KOKKOS_ARCH_KEPLER35 +#cmakedefine KOKKOS_ARCH_KEPLER37 +#cmakedefine KOKKOS_ARCH_MAXWELL +#cmakedefine KOKKOS_ARCH_MAXWELL50 +#cmakedefine KOKKOS_ARCH_MAXWELL52 +#cmakedefine KOKKOS_ARCH_MAXWELL53 +#cmakedefine KOKKOS_ARCH_PASCAL +#cmakedefine KOKKOS_ARCH_PASCAL60 +#cmakedefine KOKKOS_ARCH_PASCAL61 +#cmakedefine KOKKOS_ARCH_VOLTA +#cmakedefine KOKKOS_ARCH_VOLTA70 +#cmakedefine KOKKOS_ARCH_VOLTA72 +#cmakedefine KOKKOS_ARCH_TURING75 +#cmakedefine KOKKOS_ARCH_AMD_EPYC diff --git a/lib/kokkos/cmake/Makefile.generate_cmake_settings b/lib/kokkos/cmake/Makefile.generate_cmake_settings deleted file mode 100644 index da076b23db2697f4abb874cd5c154c08e99ad6cb..0000000000000000000000000000000000000000 --- a/lib/kokkos/cmake/Makefile.generate_cmake_settings +++ /dev/null @@ -1,8 +0,0 @@ -ifndef KOKKOS_PATH - MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) - KOKKOS_PATH = $(subst Makefile,,$(MAKEFILE_PATH)).. -endif - -include $(KOKKOS_PATH)/Makefile.kokkos -include $(KOKKOS_PATH)/core/src/Makefile.generate_header_lists -include $(KOKKOS_PATH)/core/src/Makefile.generate_build_files diff --git a/lib/kokkos/cmake/Modules/FindHWLOC.cmake b/lib/kokkos/cmake/Modules/FindHWLOC.cmake deleted file mode 100644 index 60df8084d80815f79f0215c17b40f80db4c791c0..0000000000000000000000000000000000000000 --- a/lib/kokkos/cmake/Modules/FindHWLOC.cmake +++ /dev/null @@ -1,20 +0,0 @@ -#.rst: -# FindHWLOC -# ---------- -# -# Try to find HWLOC, based on KOKKOS_HWLOC_DIR -# -# The following variables are defined: -# -# HWLOC_FOUND - System has HWLOC -# HWLOC_INCLUDE_DIR - HWLOC include directory -# HWLOC_LIBRARIES - Libraries needed to use HWLOC - -find_path(HWLOC_INCLUDE_DIR hwloc.h PATHS "${KOKKOS_HWLOC_DIR}/include") -find_library(HWLOC_LIBRARIES hwloc PATHS "${KOKKOS_HWLOC_DIR}/lib") - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(HWLOC DEFAULT_MSG - HWLOC_INCLUDE_DIR HWLOC_LIBRARIES) - -mark_as_advanced(HWLOC_INCLUDE_DIR HWLOC_LIBRARIES) diff --git a/lib/kokkos/cmake/Modules/FindMemkind.cmake b/lib/kokkos/cmake/Modules/FindMemkind.cmake deleted file mode 100644 index 245fb44c19a2ff71a389c5f139cb3f6ac38f924e..0000000000000000000000000000000000000000 --- a/lib/kokkos/cmake/Modules/FindMemkind.cmake +++ /dev/null @@ -1,20 +0,0 @@ -#.rst: -# FindMemkind -# ---------- -# -# Try to find Memkind. -# -# The following variables are defined: -# -# MEMKIND_FOUND - System has Memkind -# MEMKIND_INCLUDE_DIR - Memkind include directory -# MEMKIND_LIBRARIES - Libraries needed to use Memkind - -find_path(MEMKIND_INCLUDE_DIR memkind.h) -find_library(MEMKIND_LIBRARIES memkind) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(Memkind DEFAULT_MSG - MEMKIND_INCLUDE_DIR MEMKIND_LIBRARIES) - -mark_as_advanced(MEMKIND_INCLUDE_DIR MEMKIND_LIBRARIES) diff --git a/lib/kokkos/cmake/Modules/FindQthreads.cmake b/lib/kokkos/cmake/Modules/FindQthreads.cmake deleted file mode 100644 index a254b0e996d23e01897f5f186a316c285f64e9ee..0000000000000000000000000000000000000000 --- a/lib/kokkos/cmake/Modules/FindQthreads.cmake +++ /dev/null @@ -1,20 +0,0 @@ -#.rst: -# FindQthreads -# ---------- -# -# Try to find Qthreads. -# -# The following variables are defined: -# -# QTHREADS_FOUND - System has Qthreads -# QTHREADS_INCLUDE_DIR - Qthreads include directory -# QTHREADS_LIBRARIES - Libraries needed to use Qthreads - -find_path(QTHREADS_INCLUDE_DIR qthread.h) -find_library(QTHREADS_LIBRARIES qthread) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(Qthreads DEFAULT_MSG - QTHREADS_INCLUDE_DIR QTHREADS_LIBRARIES) - -mark_as_advanced(QTHREADS_INCLUDE_DIR QTHREADS_LIBRARIES) diff --git a/lib/kokkos/cmake/Modules/FindTPLCUDA.cmake b/lib/kokkos/cmake/Modules/FindTPLCUDA.cmake new file mode 100644 index 0000000000000000000000000000000000000000..36aefcdb44fdc52c2ee62b50973ad28b02a8fbf5 --- /dev/null +++ b/lib/kokkos/cmake/Modules/FindTPLCUDA.cmake @@ -0,0 +1,13 @@ + +IF (KOKKOS_CXX_COMPILER_ID STREQUAL Clang) + KOKKOS_FIND_IMPORTED(CUDA INTERFACE + LIBRARIES cudart cuda + LIBRARY_PATHS ENV LD_LIBRARY_PATH ENV CUDA_PATH + ALLOW_SYSTEM_PATH_FALLBACK + ) +ELSE() + KOKKOS_CREATE_IMPORTED_TPL(CUDA INTERFACE + LINK_LIBRARIES cuda + ) +ENDIF() + diff --git a/lib/kokkos/cmake/Modules/FindTPLHPX.cmake b/lib/kokkos/cmake/Modules/FindTPLHPX.cmake new file mode 100644 index 0000000000000000000000000000000000000000..c8b3bc4c9b84505eceff8ba3453501f9bb5d1e01 --- /dev/null +++ b/lib/kokkos/cmake/Modules/FindTPLHPX.cmake @@ -0,0 +1,15 @@ + +FIND_PACKAGE(HPX REQUIRED) +#as of right now, HPX doesn't export correctly +#so let's convert it to an interface target +KOKKOS_CREATE_IMPORTED_TPL(HPX INTERFACE + LINK_LIBRARIES ${HPX_LIBRARIES} + INCLUDES ${HPX_INCLUDE_DIRS} +) +#this is a bit funky since this is a CMake target +#but HPX doesn't export itself correctly +KOKKOS_EXPORT_CMAKE_TPL(HPX) + +#I would prefer all of this gets replaced with +#KOKKOS_IMPORT_CMAKE_TPL(HPX) + diff --git a/lib/kokkos/cmake/Modules/FindTPLHWLOC.cmake b/lib/kokkos/cmake/Modules/FindTPLHWLOC.cmake new file mode 100644 index 0000000000000000000000000000000000000000..cf763b7e5bb585ed77e8dc1fb3b015566a0326f9 --- /dev/null +++ b/lib/kokkos/cmake/Modules/FindTPLHWLOC.cmake @@ -0,0 +1 @@ +KOKKOS_FIND_IMPORTED(HWLOC HEADER hwloc.h LIBRARY hwloc) diff --git a/lib/kokkos/cmake/Modules/FindTPLLIBDL.cmake b/lib/kokkos/cmake/Modules/FindTPLLIBDL.cmake new file mode 100644 index 0000000000000000000000000000000000000000..5fc6a693035cea5e05b379f5fac9d50bcaeb3f7a --- /dev/null +++ b/lib/kokkos/cmake/Modules/FindTPLLIBDL.cmake @@ -0,0 +1 @@ +KOKKOS_FIND_IMPORTED(LIBDL HEADER dlfcn.h LIBRARY dl) diff --git a/lib/kokkos/cmake/Modules/FindTPLLIBNUMA.cmake b/lib/kokkos/cmake/Modules/FindTPLLIBNUMA.cmake new file mode 100644 index 0000000000000000000000000000000000000000..811db5851b9ee359ad996a743bf8a0ac283512f6 --- /dev/null +++ b/lib/kokkos/cmake/Modules/FindTPLLIBNUMA.cmake @@ -0,0 +1 @@ +KOKKOS_FIND_IMPORTED(LIBNUMA HEADER numa.h LIBRARY numa) diff --git a/lib/kokkos/cmake/Modules/FindTPLLIBRT.cmake b/lib/kokkos/cmake/Modules/FindTPLLIBRT.cmake new file mode 100644 index 0000000000000000000000000000000000000000..e75da56b5b5324050236ee0ee4c6847452d5b3cf --- /dev/null +++ b/lib/kokkos/cmake/Modules/FindTPLLIBRT.cmake @@ -0,0 +1 @@ +KOKKOS_FIND_IMPORTED(LIBRT HEADER time.h LIBRARY rt) diff --git a/lib/kokkos/cmake/Modules/FindTPLMEMKIND.cmake b/lib/kokkos/cmake/Modules/FindTPLMEMKIND.cmake new file mode 100644 index 0000000000000000000000000000000000000000..20aaff22955ce9ad026c51b870bf04b7d8b0df42 --- /dev/null +++ b/lib/kokkos/cmake/Modules/FindTPLMEMKIND.cmake @@ -0,0 +1 @@ +KOKKOS_FIND_IMPORTED(MEMKIND HEADER memkind.h LIBRARY memkind) diff --git a/lib/kokkos/cmake/Modules/FindTPLPTHREAD.cmake b/lib/kokkos/cmake/Modules/FindTPLPTHREAD.cmake new file mode 100644 index 0000000000000000000000000000000000000000..b4b8c34122a67ba73fc991ed52e35176c7df2fa1 --- /dev/null +++ b/lib/kokkos/cmake/Modules/FindTPLPTHREAD.cmake @@ -0,0 +1,17 @@ + +TRY_COMPILE(KOKKOS_HAS_PTHREAD_ARG + ${KOKKOS_TOP_BUILD_DIR}/tpl_tests + ${KOKKOS_SOURCE_DIR}/cmake/compile_tests/pthread.cpp + LINK_LIBRARIES -pthread + COMPILE_DEFINITIONS -pthread) + +INCLUDE(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(PTHREAD DEFAULT_MSG KOKKOS_HAS_PTHREAD_ARG) + +KOKKOS_CREATE_IMPORTED_TPL(PTHREAD + INTERFACE #this is not a real library with a real location + COMPILE_OPTIONS -pthread + LINK_OPTIONS -pthread) + + + diff --git a/lib/kokkos/cmake/README.md b/lib/kokkos/cmake/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0543a18e89cc9ed70472ee3381d2fc2834e14d7b --- /dev/null +++ b/lib/kokkos/cmake/README.md @@ -0,0 +1,331 @@ +![Kokkos](https://avatars2.githubusercontent.com/u/10199860?s=200&v=4) + +# Developing Kokkos + +This document contains a build system overview for developers with information on adding new CMake options that could influence +* Header configuration macros +* Optional features +* Third-partly libraries +* Compiler and linker flags +For build system details for users, refer to the [build instructions](../BUILD.md). + +## Build System + +Kokkos uses CMake to configure, build, and install. +Rather than being a completely straightforward use of modern CMake, +Kokkos has several extra complications, primarily due to: +* Kokkos must support linking to an installed version or in-tree builds as a subdirectory of a larger project. +* Kokkos must configure a special compiler `nvcc_wrapper` that allows `nvcc` to accept all C++ flags (which `nvcc` currently does not). +* Kokkos must work as a part of TriBITS, a CMake library providing a particular build idiom for Trilinos. +* Kokkos has many pre-existing users. We need to be careful about breaking previous versions or generating meaningful error messags if we do break backwards compatibility. + +If you are looking at the build system code wondering why certain decisions were made: we have had to balance many competing requirements and certain technical debt. Everything in the build system was done for a reason, trying to adhere as closely as possible to modern CMake best practices while meeting all pre-existing. customer requirements. + +### Modern CMake Philosophy + +Modern CMake relies on understanding the principle of *building* and *using* a code project. +What preprocessor, compiler, and linker flags do I need to *build* my project? +What flags does a downstream project that links to me need to *use* my project? +In CMake terms, flags that are only needed for building are `PRIVATE`. +Only Kokkos needs these flags, not a package that depends on Kokkos. +Flags that must be used in a downstream project are `PUBLIC`. +Kokkos must tell other projects to use them. + +In Kokkos, almost everything is a public flag since Kokkos is driven by headers and Kokkos is in charge of optimizing your code to achieve performance portability! +Include paths, C++ standard flags, architecture-specific optimizations, or OpenMP and CUDA flags are all examples of flags that Kokkos configures and adds to your project. + +Modern CMake now automatically propagates flags through the `target_link_libraries` command. +Suppose you have a library `stencil` that needs to build with Kokkos. +Consider the following CMake code: + +```` +find_package(Kokkos) +add_library(stencil stencil.cpp) +target_link_libraries(stencil Kokkos::kokkos) +```` + +This locates the Kokkos package, adds your library, and tells CMake to link Kokkos to your library. +All public build flags get added automatically through the `target_link_libraries` command. +There is nothing to do. You can be happily oblivious to how Kokkos was configured. +Everything should just work. + +As a Kokkos developer who wants to add new public compiler flags, how do you ensure that CMake does this properly? Modern CMake works through targets and properties. +Each target has a set of standard properties: +* `INTERFACE_COMPILE_OPTIONS` contains all the compiler options that Kokkos should add to downstream projects +* `INTERFACE_INCLUDE_DIRECTORIES` contains all the directories downstream projects must include from Kokkos +* `INTERFACE_COMPILE_DEFINITIONS` contains the list of preprocessor `-D` flags +* `INTERFACE_LINK_LIBRARIES` contains all the libraries downstream projects need to link +* `INTERFACE_COMPILE_FEATURES` essentially adds compiler flags, but with extra complications. Features names are specific to CMake. More later. + +CMake makes it easy to append to these properties using: +* `target_compile_options(kokkos PUBLIC -fmyflag)` +* `target_include_directories(kokkos PUBLIC mySpecialFolder)` +* `target_compile_definitions(kokkos PUBLIC -DmySpecialFlag=0)` +* `target_link_libraries(kokkos PUBLIC mySpecialLibrary)` +* `target_compile_features(kokkos PUBLIC mySpecialFeature)` +Note that all of these use `PUBLIC`! Almost every Kokkos flag is not private to Kokkos, but must also be used by downstream projects. + + +### Compiler Features and Compiler Options +Compiler options are flags like `-fopenmp` that do not need to be "resolved." +The flag is either on or off. +Compiler features are more fine-grained and require conflicting requests to be resolved. +Suppose I have +```` +add_library(A a.cpp) +target_compile_features(A PUBLIC cxx_std_11) +```` +then another target +```` +add_library(B b.cpp) +target_compile_features(B PUBLIC cxx_std_14) +target_link_libraries(A B) +```` +I have requested two different features. +CMake understands the requests and knows that `cxx_std_11` is a subset of `cxx_std_14`. +CMake then picks C++14 for library `B`. +CMake would not have been able to do feature resolution if we had directly done: +```` +target_compile_options(A PUBLIC -std=c++11) +```` + +### Adding Kokkos Options +After configuring for the first time, +CMake creates a cache of configure variables in `CMakeCache.txt`. +Reconfiguring in the folder "restarts" from those variables. +All flags passed as `-DKokkos_SOME_OPTION=X` to `cmake` become variables in the cache. +All Kokkos options begin with camel case `Kokkos_` followed by an upper case option name. + +CMake best practice is to avoid cache variables, if possible. +In essence, you want the minimal amount of state cached between configurations. +And never, ever have behavior influenced by multiple cache variables. +If you want to change the Kokkos configuration, have a single unique variable that needs to be changed. +Never require two cache variables to be changed. + +Kokkos provides a function `KOKKOS_OPTION` for defining valid cache-level variables, +proofreading them, and defining local project variables. +The most common variables are called `Kokkos_ENABLE_X`, +for which a helper function `KOKKOS_ENABLE_OPTION` is provided, e.g. +```` +KOKKOS_ENABLE_OPTION(TESTS OFF "Whether to build tests") +```` +The function checks if `-DKokkos_ENABLE_TESTS` was given, +whether it was given with the wrong case, e.g. `-DKokkos_Enable_Tests`, +and then defines a regular (non-cache) variable `KOKKOS_ENABLE_TESTS` to `ON` or `OFF` +depending on the given default and whether the option was specified. + +### Defining Kokkos Config Macros + +Sometimes you may want to add `#define Kokkos_X` macros to the config header. +This is straightforward with CMake. +Suppose you want to define an optional macro `KOKKOS_SUPER_SCIENCE`. +Simply go into `KokkosCore_config.h.in` and add +```` +#cmakedefine KOKKOS_SUPER_SCIENCE +```` +I can either add +```` +KOKKOS_OPTION(SUPER_SCIENCE ON "Whether to do some super science") +```` +to directly set the variable as a command-line `-D` option. +Alternatively, based on other logic, I could add to a `CMakeLists.txt` +```` +SET(KOKKOS_SUPER_SCIENCE ON) +```` +If not set as a command-line option (cache variable), you must make sure the variable is visible in the top-level scope. +If set in a function, you would need: +```` +SET(KOKKOS_SUPER_SCIENCE ON PARENT_SCOPE) +```` + +### Third-Party Libraries +In much the same way that compiler flags transitively propagate to dependent projects, +modern CMake allows us to propagate dependent libraries. +If Kokkos depends on, e.g. `hwloc` the downstream project will also need to link `hwloc`. +There are three stages in adding a new third-party library (TPL): +* Finding: find the desired library on the system and verify the installation is correct +* Importing: create a CMake target, if necessary, that is compatible with `target_link_libraries`. This is mostly relevant for TPLs not installed with CMake. +* Exporting: make the desired library visible to downstream projects + +TPLs are somewhat complicated by whether the library was installed with CMake or some other build system. +If CMake, our lives are greatly simplified. We simply use `find_package` to locate the installed CMake project then call `target_link_libraries(kokkoscore PUBLIC/PRIVATE TPL)`. For libaries not installed with CMake, the process is a bit more complex. +It is up to the Kokkos developers to "convert" the library into a CMake target as if it had been installed as a valid modern CMake target with properties. +There are helper functions for simplifying the process of importing TPLs in Kokkos, but we walk through the process in detail to clearly illustrate the steps involved. + +#### TPL Search Order + +There are several options for where CMake could try to find a TPL. +If there are multiple installations of the same TPL on the system, +the search order is critical for making sure the correct TPL is found. +There are 3 possibilities that could be used: + +1. Default system paths like /usr +1. User-provided paths through options `_ROOT` and `Kokkos__DIR` +1. Additional paths not in the CMake default list or provided by the user that Kokkos decides to add. For example, Kokkos may query `nvcc` or `LD_LIBRARY_PATH` for where to find CUDA libraries. + +The following is the search order that Kokkos follows. Note: This differs from the default search order used by CMake `find_library` and `find_header`. CMake prefers default system paths over user-provided paths. +For Kokkos (and package managers in general), it is better to prefer user-provided paths since this usually indicates a specific version we want. + +1. `_ROOT` +1. `Kokkos__DIR` +1. Paths added by Kokkos CMake logic +1. Default system paths (if allowed) + +Default system paths are allowed in two cases. First, none of the other options are given so the only place to look is system paths. Second, if explicitly given permission, configure will look in system paths. +The rationale for this logic is that if you specify a custom location, you usually *only* want to look in that location. +If you do not find the TPL where you expect it, you should error out rather than grab another random match. + + +#### Finding TPLs + +If finding a TPL that is not a modern CMake project, refer to the `FindHWLOC.cmake` file in `cmake/Modules` for an example. +You will usually need to verify expected headers with `find_path` +```` +find_path(TPL_INCLUDE_DIR mytpl.h PATHS "${KOKKOS_MYTPL_DIR}/include") +```` +This insures that the library header is in the expected include directory and defines the variable `TPL_INCLUDE_DIR` with a valid path if successful. +Similarly, you can verify a library +```` +find_library(TPL_LIBRARY mytpl PATHS "${KOKKOS_MYTPL_DIR/lib") +```` +that then defines the variable `TPL_LIBRARY` with a valid path if successful. +CMake provides a utility for checking if the `find_path` and `find_library` calls were successful that emulates the behavior of `find_package` for a CMake target. +```` +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(MYTPL DEFAULT_MSG + MYTPL_INCLUDE_DIR MYTPL_LIBRARY) +```` +If the find failed, CMake will print standard error messages explaining the failure. + +#### Importing TPLs + +The installed TPL must be adapted into a CMake target. +CMake allows libraries to be added that are built externally as follows: +```` +add_library(Kokkos::mytpl UNKNOWN IMPORTED) +```` +Importantly, we use a `Kokkos::` namespace to avoid name conflicts and identify this specifically as the version imported by Kokkos. +Because we are importing a non-CMake target, we must populate all the target properties that would have been automatically populated for a CMake target. +```` +set_target_properties(Kokkos::mytpl PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${MYTPL_INCLUDE_DIR}" + IMPORTED_LOCATION "${MYTPL_LIBRARY}" +) +```` + +#### Exporting TPLs + +Kokkos may now depend on the target `Kokkos::mytpl` as a `PUBLIC` library (remember building and using). +This means that downstream projects must also know about `Kokkos::myptl` - so Kokkos must export them. +In the `KokkosConfig.cmake.in` file, we need to add code like the following: +```` +set(MYTPL_LIBRARY @MYTPL_LIBRARY@) +set(MYTPL_INCLUDE_DIR @MYTPL_INCLUDE_DIR@) +add_library(Kokkos::mytpl UNKNOWN IMPORTED) +set_target_properties(Kokkos::mytpl PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${MYTPL_INCLUDE_DIR}" + IMPORTED_LOCATION "${MYTPL_LIBRARY}" +) +```` +If this looks familiar, that's because it is exactly the same code as above for importing the TPL. +Exporting a TPL really just means importing the TPL when Kokkos is loaded by an external project. +We will describe helper functions that simplify this process. + +#### Interface TPLs + +If a TPL is just a library and set of headers, we can make a simple `IMPORTED` target. +However, a TPL is actually completely flexible and need not be limited to just headers and libraries. +TPLs can configure compiler flags, linker flags, or multiple different libraries. +For this, we use a special type of CMake target: `INTERFACE` libraries. +These libraries don't build anything. +They simply populate properties that will configure flags for dependent targets. +We consider the example: +```` +add_library(PTHREAD INTERFACE) +target_compile_options(PTHREAD PUBLIC -pthread) +```` +Kokkos uses the compiler flag `-pthread` to define compiler macros for re-entrant functions rather than treating it simply as a library with header `pthread.h` and library `-lpthread`. +Any property can be configured, e.g. +```` +target_link_libraries(MYTPL ...) +```` +In contrast to imported TPLs which require direct modification of `KokkosConfig.cmake.in`, +we can use CMake's built-in export functions: +```` +INSTALL( + TARGETS MYTPL + EXPORT KokkosTargets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} +) +```` +These interface targets will be automatically populated in the config file. + +#### Linking the TPL +After finishing the import process, it still remains to link the imported target as needed. +For example, +```` +target_link_libraries(kokkoscore PUBLIC Kokkos::HWLOC) +```` +The complexity of which includes, options, and libraries the TPL requires +should be encapsulated in the CMake target. + +#### TPL Helper Functions +##### KOKKOS_IMPORT_TPL +This function can be invoked as, e.g. +```` +KOKKOS_IMPORT_TPL(HWLOC) +```` +This function checks if the TPL was enabled by a `-DKokkos_ENABLE_HWLOC=On` flag. +If so, it calls `find_package(TPLHWLOC)`. +This invokes the file `FindTPLHWLOC.cmake` which should be contained in the `cmake/Modules` folder. +If successful, another function `KOKKOS_EXPORT_CMAKE_TPL` gets invoked. +This automatically adds all the necessary import commands to `KokkosConfig.cmake`. + +##### KOKKOS_FIND_IMPORTED +Inside a `FindTPLX.cmake` file, the simplest way to import a library is to call, e.g. +```` +KOKKOS_FIND_IMPORTED(HWLOC LIBRARY hwloc HEADER hwloc.h) +```` +This finds the location of the library and header and creates an imported target `Kokkos::HWLOC` +that can be linked against. +The library/header find can be guided with `-DHWLOC_ROOT=` or `-DKokkos_HWLOC_DIR=` during CMake configure. +These both specify the install prefix. + +##### KOKKOS_LINK_TPL +This function checks if the TPL has been enabled. +If so, it links a given library against the imported (or interface) TPL target. + +##### KOKKOS_CREATE_IMPORTED_TPL +This helper function is best understood by reading the actual code. +This function takes arguments specifying the properties and creates the actual TPL target. +The most important thing to understand for this function is whether you call this function with the optional `INTERFACE` keyword. +This tells the project to either create the target as an imported target or interface target, as discussed above. + +##### KOKKOS_EXPORT_CMAKE_TPL +Even if the TPL just loads a valid CMake target, we still must "export" it into the config file. +When Kokkos is loaded by a downstream project, this TPL must be loaded. +Calling this function simply appends text recording the location where the TPL was found +and adding a `find_dependency(...)` call that will reload the CMake target. + +### The Great TriBITS Compromise + +TriBITS was a masterpiece of CMake version 2 before the modern CMake idioms of building and using. +TriBITS greatly limited verbosity of CMake files, handled complicated dependency trees between packages, and handled automatically setting up include and linker paths for dependent libraries. + +Kokkos is now used by numerous projects that don't (and won't) depend on TriBITS for their build systems. +Kokkos has to work outside of TriBITS and provide a standard CMake 3+ build system. +At the same time, Kokkos is used by numerous projects that depend on TriBITS and don't (and won't) switch to a standard CMake 3+ build system. + +Instead of calling functions `TRIBITS_X(...)`, the CMake calls wrapper functions `KOKKOS_X(...)`. +If TriBITS is available (as in Trilinos), `KOKKOS_X` will just be a thin wrapper around `TRIBITS_X`. +If TriBITS is not available, Kokkos maps `KOKKOS_X` calls to native CMake that complies with CMake 3 idioms. +For the time being, this seems the most sensible way to handle the competing requirements of a standalone modern CMake and TriBITS build system. + +##### [LICENSE](https://github.com/kokkos/kokkos/blob/devel/LICENSE) + +[![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) + +Under the terms of Contract DE-NA0003525 with NTESS, +the U.S. Government retains certain rights in this software. diff --git a/lib/kokkos/cmake/compile_tests/clang_omp.cpp b/lib/kokkos/cmake/compile_tests/clang_omp.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ce3bbfb262377248628385f56578db9d8c5b6a33 --- /dev/null +++ b/lib/kokkos/cmake/compile_tests/clang_omp.cpp @@ -0,0 +1,9 @@ +#include + +int main(int argc, char** argv) { + int thr = omp_get_num_threads(); + if (thr > 0) + return thr; + else + return 0; +} diff --git a/lib/kokkos/cmake/compile_tests/pthread.cpp b/lib/kokkos/cmake/compile_tests/pthread.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3b13f7ba357f6c05ffad30a61b01620fb33666b0 --- /dev/null +++ b/lib/kokkos/cmake/compile_tests/pthread.cpp @@ -0,0 +1,10 @@ +#include + +void* kokkos_test(void* args) { return args; } + +int main(void) { + pthread_t thread; + pthread_create(&thread, NULL, kokkos_test, NULL); + pthread_join(thread, NULL); + return 0; +} diff --git a/lib/kokkos/cmake/cray.cmake b/lib/kokkos/cmake/cray.cmake new file mode 100644 index 0000000000000000000000000000000000000000..08912f5130f92fec97a4bdb6abb90e860d0b9cda --- /dev/null +++ b/lib/kokkos/cmake/cray.cmake @@ -0,0 +1,9 @@ + + +function(kokkos_set_cray_flags full_standard int_standard) + STRING(TOLOWER ${full_standard} FULL_LC_STANDARD) + STRING(TOLOWER ${int_standard} INT_LC_STANDARD) + SET(KOKKOS_CXX_STANDARD_FLAG "-hstd=c++${FULL_LC_STANDARD}", PARENT_SCOPE) + SET(KOKKOS_CXX_INTERMDIATE_STANDARD_FLAG "-hstd=c++${INT_LC_STANDARD}" PARENT_SCOPE) +endfunction() + diff --git a/lib/kokkos/cmake/deps/CUDA.cmake b/lib/kokkos/cmake/deps/CUDA.cmake index 801c20067b9195db5ba5e6cd6fdd62a426e6e294..4876bca25907eb632a8f12d01352ddaa854fd341 100644 --- a/lib/kokkos/cmake/deps/CUDA.cmake +++ b/lib/kokkos/cmake/deps/CUDA.cmake @@ -73,7 +73,7 @@ IF(NOT _CUDA_FAILURE) GLOBAL_SET(TPL_CUDA_LIBRARY_DIRS) GLOBAL_SET(TPL_CUDA_INCLUDE_DIRS ${CUDA_TOOLKIT_INCLUDE}) GLOBAL_SET(TPL_CUDA_LIBRARIES ${CUDA_CUDART_LIBRARY} ${CUDA_cublas_LIBRARY} ${CUDA_cufft_LIBRARY}) - TIBITS_CREATE_IMPORTED_TPL_LIBRARY(CUSPARSE) + KOKKOS_CREATE_IMPORTED_TPL_LIBRARY(CUSPARSE) ELSE() SET(TPL_ENABLE_CUDA OFF) ENDIF() diff --git a/lib/kokkos/cmake/deps/CUSPARSE.cmake b/lib/kokkos/cmake/deps/CUSPARSE.cmake index 6f26d857c09acf7bb24c2c5449a54f5d507deae8..b2420d1168e55d7ce990ff5380ba638996888a09 100644 --- a/lib/kokkos/cmake/deps/CUSPARSE.cmake +++ b/lib/kokkos/cmake/deps/CUSPARSE.cmake @@ -59,6 +59,6 @@ # GLOBAL_SET(TPL_CUSPARSE_LIBRARY_DIRS) # GLOBAL_SET(TPL_CUSPARSE_INCLUDE_DIRS ${TPL_CUDA_INCLUDE_DIRS}) # GLOBAL_SET(TPL_CUSPARSE_LIBRARIES ${CUDA_cusparse_LIBRARY}) -# TIBITS_CREATE_IMPORTED_TPL_LIBRARY(CUSPARSE) +# KOKKOS_CREATE_IMPORTED_TPL_LIBRARY(CUSPARSE) #ENDIF() diff --git a/lib/kokkos/cmake/deps/HWLOC.cmake b/lib/kokkos/cmake/deps/HWLOC.cmake index 275abd3a5d4ecfb3ce3b207f978959f6f9019061..ed89c8c1e511593663a4565316f596aa36317baa 100644 --- a/lib/kokkos/cmake/deps/HWLOC.cmake +++ b/lib/kokkos/cmake/deps/HWLOC.cmake @@ -64,7 +64,7 @@ # Version: 1.3 # -TRIBITS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( HWLOC +KOKKOS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( HWLOC REQUIRED_HEADERS hwloc.h REQUIRED_LIBS_NAMES "hwloc" ) diff --git a/lib/kokkos/cmake/deps/Pthread.cmake b/lib/kokkos/cmake/deps/Pthread.cmake index 46d0a939cad0e6c5479cb20da1d37ba5ca509b8c..5f835fc3002f752f0aae5a5a54894b3f1d16451b 100644 --- a/lib/kokkos/cmake/deps/Pthread.cmake +++ b/lib/kokkos/cmake/deps/Pthread.cmake @@ -74,9 +74,9 @@ IF(USE_THREADS) SET(TPL_Pthread_INCLUDE_DIRS "") SET(TPL_Pthread_LIBRARIES "${CMAKE_THREAD_LIBS_INIT}") SET(TPL_Pthread_LIBRARY_DIRS "") - TIBITS_CREATE_IMPORTED_TPL_LIBRARY(Pthread) + KOKKOS_CREATE_IMPORTED_TPL_LIBRARY(Pthread) ELSE() - TRIBITS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( Pthread + KOKKOS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( Pthread REQUIRED_HEADERS pthread.h REQUIRED_LIBS_NAMES pthread ) diff --git a/lib/kokkos/cmake/deps/QTHREADS.cmake b/lib/kokkos/cmake/deps/QTHREADS.cmake deleted file mode 100644 index c312f2590bcd29197a0cf3fbd5e0b484579a09c2..0000000000000000000000000000000000000000 --- a/lib/kokkos/cmake/deps/QTHREADS.cmake +++ /dev/null @@ -1,69 +0,0 @@ -# @HEADER -# ************************************************************************ -# -# Trilinos: An Object-Oriented Solver Framework -# Copyright (2001) Sandia Corporation -# -# -# Copyright (2001) Sandia Corporation. Under the terms of Contract -# DE-AC04-94AL85000, there is a non-exclusive license for use of this -# work by or on behalf of the U.S. Government. Export of this program -# may require a license from the United States Government. -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# NOTICE: The United States Government is granted for itself and others -# acting on its behalf a paid-up, nonexclusive, irrevocable worldwide -# license in this data to reproduce, prepare derivative works, and -# perform publicly and display publicly. Beginning five (5) years from -# July 25, 2001, the United States Government is granted for itself and -# others acting on its behalf a paid-up, nonexclusive, irrevocable -# worldwide license in this data to reproduce, prepare derivative works, -# distribute copies to the public, perform publicly and display -# publicly, and to permit others to do so. -# -# NEITHER THE UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT -# OF ENERGY, NOR SANDIA CORPORATION, NOR ANY OF THEIR EMPLOYEES, MAKES -# ANY WARRANTY, EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR -# RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY -# INFORMATION, APPARATUS, PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS -# THAT ITS USE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS. -# -# ************************************************************************ -# @HEADER - - -#----------------------------------------------------------------------------- -# Hardware locality detection and control library. -# -# Acquisition information: -# Date checked: July 2014 -# Checked by: H. Carter Edwards -# Source: https://code.google.com/p/qthreads -# - -TRIBITS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( QTHREADS - REQUIRED_HEADERS qthread.h - REQUIRED_LIBS_NAMES "qthread" - ) diff --git a/lib/kokkos/cmake/fake_tribits.cmake b/lib/kokkos/cmake/fake_tribits.cmake new file mode 100644 index 0000000000000000000000000000000000000000..26948d2cfbbd3bf724703447a81e66ee29ea000e --- /dev/null +++ b/lib/kokkos/cmake/fake_tribits.cmake @@ -0,0 +1,338 @@ +#These are tribits wrappers used by all projects in the Kokkos ecosystem + +INCLUDE(CMakeParseArguments) +INCLUDE(CTest) + +cmake_policy(SET CMP0054 NEW) + +FUNCTION(ASSERT_DEFINED VARS) + FOREACH(VAR ${VARS}) + IF(NOT DEFINED ${VAR}) + MESSAGE(SEND_ERROR "Error, the variable ${VAR} is not defined!") + ENDIF() + ENDFOREACH() +ENDFUNCTION() + +MACRO(KOKKOS_ADD_OPTION_AND_DEFINE USER_OPTION_NAME MACRO_DEFINE_NAME DOCSTRING DEFAULT_VALUE ) +SET( ${USER_OPTION_NAME} "${DEFAULT_VALUE}" CACHE BOOL "${DOCSTRING}" ) +IF(NOT ${MACRO_DEFINE_NAME} STREQUAL "") + IF(${USER_OPTION_NAME}) + GLOBAL_SET(${MACRO_DEFINE_NAME} ON) + ELSE() + GLOBAL_SET(${MACRO_DEFINE_NAME} OFF) + ENDIF() +ENDIF() +ENDMACRO() + +MACRO(GLOBAL_RESET VARNAME) + SET(${VARNAME} "" CACHE INTERNAL "" FORCE) +ENDMACRO() + +MACRO(GLOBAL_OVERWRITE VARNAME VALUE TYPE) + SET(${VARNAME} ${VALUE} CACHE ${TYPE} "" FORCE) +ENDMACRO() + +IF (NOT KOKKOS_HAS_TRILINOS) +MACRO(APPEND_GLOB VAR) + FILE(GLOB LOCAL_TMP_VAR ${ARGN}) + LIST(APPEND ${VAR} ${LOCAL_TMP_VAR}) +ENDMACRO() + +MACRO(GLOBAL_SET VARNAME) + SET(${VARNAME} ${ARGN} CACHE INTERNAL "" FORCE) +ENDMACRO() + +FUNCTION(VERIFY_EMPTY CONTEXT) +if(${ARGN}) +MESSAGE(FATAL_ERROR "Kokkos does not support all of Tribits. Unhandled arguments in ${CONTEXT}:\n${ARGN}") +endif() +ENDFUNCTION() + +MACRO(PREPEND_GLOBAL_SET VARNAME) + ASSERT_DEFINED(${VARNAME}) + GLOBAL_SET(${VARNAME} ${ARGN} ${${VARNAME}}) +ENDMACRO() + +MACRO(PREPEND_TARGET_SET VARNAME TARGET_NAME TYPE) + IF(TYPE STREQUAL "REQUIRED") + SET(REQUIRED TRUE) + ELSE() + SET(REQUIRED FALSE) + ENDIF() + IF(TARGET ${TARGET_NAME}) + PREPEND_GLOBAL_SET(${VARNAME} ${TARGET_NAME}) + ELSE() + IF(REQUIRED) + MESSAGE(FATAL_ERROR "Missing dependency ${TARGET_NAME}") + ENDIF() + ENDIF() +ENDMACRO() +endif() + + +FUNCTION(KOKKOS_CONFIGURE_FILE PACKAGE_NAME_CONFIG_FILE) + if (KOKKOS_HAS_TRILINOS) + TRIBITS_CONFIGURE_FILE(${PACKAGE_NAME_CONFIG_FILE}) + else() + # Configure the file + CONFIGURE_FILE( + ${PACKAGE_SOURCE_DIR}/cmake/${PACKAGE_NAME_CONFIG_FILE}.in + ${CMAKE_CURRENT_BINARY_DIR}/${PACKAGE_NAME_CONFIG_FILE} + ) + endif() +ENDFUNCTION() + +MACRO(ADD_INTERFACE_LIBRARY LIB_NAME) + FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp "") + ADD_LIBRARY(${LIB_NAME} STATIC ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp) + SET_TARGET_PROPERTIES(${LIB_NAME} PROPERTIES INTERFACE TRUE) +ENDMACRO() + +IF(NOT TARGET check) + ADD_CUSTOM_TARGET(check COMMAND ${CMAKE_CTEST_COMMAND} -VV -C ${CMAKE_CFG_INTDIR}) +ENDIF() + +FUNCTION(KOKKOS_ADD_TEST) + if (KOKKOS_HAS_TRILINOS) + CMAKE_PARSE_ARGUMENTS(TEST + "" + "EXE;NAME" + "" + ${ARGN}) + IF(TEST_EXE) + SET(EXE_ROOT ${TEST_EXE}) + ELSE() + SET(EXE_ROOT ${TEST_NAME}) + ENDIF() + + TRIBITS_ADD_TEST( + ${EXE_ROOT} + NAME ${TEST_NAME} + ${ARGN} + COMM serial mpi + NUM_MPI_PROCS 1 + ${TEST_UNPARSED_ARGUMENTS} + ) + else() + CMAKE_PARSE_ARGUMENTS(TEST + "WILL_FAIL" + "FAIL_REGULAR_EXPRESSION;PASS_REGULAR_EXPRESSION;EXE;NAME" + "CATEGORIES;CMD_ARGS" + ${ARGN}) + IF(TEST_EXE) + SET(EXE ${TEST_EXE}) + ELSE() + SET(EXE ${TEST_NAME}) + ENDIF() + IF(WIN32) + ADD_TEST(NAME ${TEST_NAME} WORKING_DIRECTORY ${LIBRARY_OUTPUT_PATH} COMMAND ${EXE}${CMAKE_EXECUTABLE_SUFFIX} ${TEST_CMD_ARGS}) + ELSE() + ADD_TEST(NAME ${TEST_NAME} COMMAND ${EXE} ${TEST_CMD_ARGS}) + ENDIF() + IF(TEST_WILL_FAIL) + SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES WILL_FAIL ${TEST_WILL_FAIL}) + ENDIF() + IF(TEST_FAIL_REGULAR_EXPRESSION) + SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES FAIL_REGULAR_EXPRESSION ${TEST_FAIL_REGULAR_EXPRESSION}) + ENDIF() + IF(TEST_PASS_REGULAR_EXPRESSION) + SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES PASS_REGULAR_EXPRESSION ${TEST_PASS_REGULAR_EXPRESSION}) + ENDIF() + VERIFY_EMPTY(KOKKOS_ADD_TEST ${TEST_UNPARSED_ARGUMENTS}) + endif() +ENDFUNCTION() + +FUNCTION(KOKKOS_ADD_ADVANCED_TEST) + if (KOKKOS_HAS_TRILINOS) + TRIBITS_ADD_ADVANCED_TEST(${ARGN}) + else() + # TODO Write this + endif() +ENDFUNCTION() + +MACRO(KOKKOS_CREATE_IMPORTED_TPL_LIBRARY TPL_NAME) + ADD_INTERFACE_LIBRARY(TPL_LIB_${TPL_NAME}) + TARGET_LINK_LIBRARIES(TPL_LIB_${TPL_NAME} LINK_PUBLIC ${TPL_${TPL_NAME}_LIBRARIES}) + TARGET_INCLUDE_DIRECTORIES(TPL_LIB_${TPL_NAME} INTERFACE ${TPL_${TPL_NAME}_INCLUDE_DIRS}) +ENDMACRO() + +FUNCTION(KOKKOS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES TPL_NAME) + if (KOKKOS_HAS_TRILINOS) + TRIBITS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES(${TPL_NAME} ${ARGN}) + else() + CMAKE_PARSE_ARGUMENTS(PARSE + "" + "" + "REQUIRED_HEADERS;REQUIRED_LIBS_NAMES" + ${ARGN}) + + SET(_${TPL_NAME}_ENABLE_SUCCESS TRUE) + IF (PARSE_REQUIRED_LIBS_NAMES) + FIND_LIBRARY(TPL_${TPL_NAME}_LIBRARIES NAMES ${PARSE_REQUIRED_LIBS_NAMES}) + IF(NOT TPL_${TPL_NAME}_LIBRARIES) + SET(_${TPL_NAME}_ENABLE_SUCCESS FALSE) + ENDIF() + ENDIF() + IF (PARSE_REQUIRED_HEADERS) + FIND_PATH(TPL_${TPL_NAME}_INCLUDE_DIRS NAMES ${PARSE_REQUIRED_HEADERS}) + IF(NOT TPL_${TPL_NAME}_INCLUDE_DIRS) + SET(_${TPL_NAME}_ENABLE_SUCCESS FALSE) + ENDIF() + ENDIF() + IF (_${TPL_NAME}_ENABLE_SUCCESS) + KOKKOS_CREATE_IMPORTED_TPL_LIBRARY(${TPL_NAME}) + ENDIF() + VERIFY_EMPTY(KOKKOS_CREATE_IMPORTED_TPL_LIBRARY ${PARSE_UNPARSED_ARGUMENTS}) + endif() +ENDFUNCTION() + +MACRO(KOKKOS_TARGET_COMPILE_OPTIONS TARGET) +if(KOKKOS_HAS_TRILINOS) + TARGET_COMPILE_OPTIONS(${TARGET} ${ARGN}) +else() + TARGET_COMPILE_OPTIONS(${TARGET} ${ARGN}) +endif() +ENDMACRO() + + +MACRO(KOKKOS_EXCLUDE_AUTOTOOLS_FILES) + if (KOKKOS_HAS_TRILINOS) + TRIBITS_EXCLUDE_AUTOTOOLS_FILES() + else() + #do nothing + endif() +ENDMACRO() + +FUNCTION(KOKKOS_LIB_TYPE LIB RET) +GET_TARGET_PROPERTY(PROP ${LIB} TYPE) +IF (${PROP} STREQUAL "INTERFACE_LIBRARY") + SET(${RET} "INTERFACE" PARENT_SCOPE) +ELSE() + SET(${RET} "PUBLIC" PARENT_SCOPE) +ENDIF() +ENDFUNCTION() + +FUNCTION(KOKKOS_TARGET_INCLUDE_DIRECTORIES TARGET) +IF(KOKKOS_HAS_TRILINOS) + KOKKOS_LIB_TYPE(${TARGET} INCTYPE) + #don't trust tribits to do this correctly - but need to add package name + TARGET_INCLUDE_DIRECTORIES(${TARGET} ${INCTYPE} ${ARGN}) +ELSEIF(TARGET ${TARGET}) + #the target actually exists - this means we are doing separate libs + #or this a test library + KOKKOS_LIB_TYPE(${TARGET} INCTYPE) + TARGET_INCLUDE_DIRECTORIES(${TARGET} ${INCTYPE} ${ARGN}) +ELSE() + GET_PROPERTY(LIBS GLOBAL PROPERTY KOKKOS_LIBRARIES_NAMES) + IF (${TARGET} IN_LIST LIBS) + SET_PROPERTY(GLOBAL APPEND PROPERTY KOKKOS_LIBRARY_INCLUDES ${ARGN}) + ELSE() + MESSAGE(FATAL_ERROR "Trying to set include directories on unknown target ${TARGET}") + ENDIF() +ENDIF() +ENDFUNCTION() + +FUNCTION(KOKKOS_LINK_INTERNAL_LIBRARY TARGET DEPLIB) +IF(KOKKOS_HAS_TRILINOS) + #do nothing +ELSE() + SET(options INTERFACE) + SET(oneValueArgs) + SET(multiValueArgs) + CMAKE_PARSE_ARGUMENTS(PARSE + "INTERFACE" + "" + "" + ${ARGN}) + SET(LINK_TYPE) + IF(PARSE_INTERFACE) + SET(LINK_TYPE INTERFACE) + ELSE() + SET(LINK_TYPE PUBLIC) + ENDIF() + TARGET_LINK_LIBRARIES(${TARGET} ${LINK_TYPE} ${DEPLIB}) + VERIFY_EMPTY(KOKKOS_LINK_INTERNAL_LIBRARY ${PARSE_UNPARSED_ARGUMENTS}) +ENDIF() +ENDFUNCTION() + +FUNCTION(KOKKOS_ADD_TEST_LIBRARY NAME) +IF (KOKKOS_HAS_TRILINOS) + TRIBITS_ADD_LIBRARY(${NAME} ${ARGN} TESTONLY + ADDED_LIB_TARGET_NAME_OUT ${NAME} + ) +ELSE() + SET(oneValueArgs) + SET(multiValueArgs HEADERS SOURCES) + + CMAKE_PARSE_ARGUMENTS(PARSE + "STATIC;SHARED" + "" + "HEADERS;SOURCES" + ${ARGN}) + + IF(PARSE_HEADERS) + LIST(REMOVE_DUPLICATES PARSE_HEADERS) + ENDIF() + IF(PARSE_SOURCES) + LIST(REMOVE_DUPLICATES PARSE_SOURCES) + ENDIF() + ADD_LIBRARY(${NAME} ${PARSE_SOURCES}) + target_link_libraries( + ${NAME} + PUBLIC kokkos + ) +ENDIF() +ENDFUNCTION() + + +FUNCTION(KOKKOS_TARGET_COMPILE_DEFINITIONS) + IF (KOKKOS_HAS_TRILINOS) + TARGET_COMPILE_DEFINITIONS(${TARGET} ${ARGN}) + ELSE() + TARGET_COMPILE_DEFINITIONS(${TARGET} ${ARGN}) + ENDIF() +ENDFUNCTION() + +FUNCTION(KOKKOS_INCLUDE_DIRECTORIES) +IF(KOKKOS_HAS_TRILINOS) + TRIBITS_INCLUDE_DIRECTORIES(${ARGN}) +ELSE() + CMAKE_PARSE_ARGUMENTS( + INC + "REQUIRED_DURING_INSTALLATION_TESTING" + "" + "" + ${ARGN} + ) + INCLUDE_DIRECTORIES(${INC_UNPARSED_ARGUMENTS}) +ENDIF() +ENDFUNCTION() + + +MACRO(KOKKOS_ADD_COMPILE_OPTIONS) +ADD_COMPILE_OPTIONS(${ARGN}) +ENDMACRO() + +MACRO(PRINTALL match) +get_cmake_property(_variableNames VARIABLES) +list (SORT _variableNames) +foreach (_variableName ${_variableNames}) + if("${_variableName}" MATCHES "${match}") + message(STATUS "${_variableName}=${${_variableName}}") + endif() +endforeach() +ENDMACRO() + +MACRO(SET_GLOBAL_REPLACE SUBSTR VARNAME) + STRING(REPLACE ${SUBSTR} ${${VARNAME}} TEMP) + GLOBAL_SET(${VARNAME} ${TEMP}) +ENDMACRO() + +FUNCTION(GLOBAL_APPEND VARNAME) + #We make this a function since we are setting variables + #and want to use scope to avoid overwriting local variables + SET(TEMP ${${VARNAME}}) + LIST(APPEND TEMP ${ARGN}) + GLOBAL_SET(${VARNAME} ${TEMP}) +ENDFUNCTION() + diff --git a/lib/kokkos/cmake/gnu.cmake b/lib/kokkos/cmake/gnu.cmake new file mode 100644 index 0000000000000000000000000000000000000000..aa11fe87b111970ea440a3765c06d0b31b402d15 --- /dev/null +++ b/lib/kokkos/cmake/gnu.cmake @@ -0,0 +1,23 @@ + +FUNCTION(kokkos_set_gnu_flags full_standard int_standard) + STRING(TOLOWER ${full_standard} FULL_LC_STANDARD) + STRING(TOLOWER ${int_standard} INT_LC_STANDARD) + # The following three blocks of code were copied from + # /Modules/Compiler/Intel-CXX.cmake from CMake 3.7.2 and then modified. + IF(CMAKE_CXX_SIMULATE_ID STREQUAL MSVC) + SET(_std -Qstd) + SET(_ext c++) + ELSE() + SET(_std -std) + SET(_ext gnu++) + ENDIF() + + IF (CMAKE_CXX_EXTENSIONS) + SET(KOKKOS_CXX_STANDARD_FLAG "-std=gnu++${FULL_LC_STANDARD}" PARENT_SCOPE) + SET(KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG "-std=gnu++${INT_LC_STANDARD}" PARENT_SCOPE) + ELSE() + SET(KOKKOS_CXX_STANDARD_FLAG "-std=c++${FULL_LC_STANDARD}" PARENT_SCOPE) + SET(KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG "-std=c++${INT_LC_STANDARD}" PARENT_SCOPE) + ENDIF() +ENDFUNCTION() + diff --git a/lib/kokkos/cmake/intel.cmake b/lib/kokkos/cmake/intel.cmake new file mode 100644 index 0000000000000000000000000000000000000000..f36f01d8ca669780c01c2a4145d3e3c78241b2b3 --- /dev/null +++ b/lib/kokkos/cmake/intel.cmake @@ -0,0 +1,30 @@ + +FUNCTION(kokkos_set_intel_flags full_standard int_standard) + STRING(TOLOWER ${full_standard} FULL_LC_STANDARD) + STRING(TOLOWER ${int_standard} INT_LC_STANDARD) + # The following three blocks of code were copied from + # /Modules/Compiler/Intel-CXX.cmake from CMake 3.7.2 and then modified. + IF(CMAKE_CXX_SIMULATE_ID STREQUAL MSVC) + SET(_std -Qstd) + SET(_ext c++) + ELSE() + SET(_std -std) + SET(_ext gnu++) + ENDIF() + + IF(NOT KOKKOS_CXX_STANDARD STREQUAL 11 AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 15.0.2) + #There is no gnu++14 value supported; figure out what to do. + SET(KOKKOS_CXX_STANDARD_FLAG "${_std}=c++${FULL_LC_STANDARD}" PARENT_SCOPE) + SET(KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG "${_std}=c++${INT_LC_STANDARD}" PARENT_SCOPE) + ELSEIF(KOKKOS_CXX_STANDARD STREQUAL 11 AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 13.0) + IF (CMAKE_CXX_EXTENSIONS) + SET(KOKKOS_CXX_STANDARD_FLAG "${_std}=${_ext}c++11" PARENT_SCOPE) + ELSE() + SET(KOKKOS_CXX_STANDARD_FLAG "${_std}=c++11" PARENT_SCOPE) + ENDIF() + ELSE() + MESSAGE(FATAL_ERROR "Intel compiler version too low - need 13.0 for C++11 and 15.0 for C++14") + ENDIF() + +ENDFUNCTION() + diff --git a/lib/kokkos/cmake/kokkos_arch.cmake b/lib/kokkos/cmake/kokkos_arch.cmake new file mode 100644 index 0000000000000000000000000000000000000000..c33247c9550f026d3ac5f611ecb049fe82333b03 --- /dev/null +++ b/lib/kokkos/cmake/kokkos_arch.cmake @@ -0,0 +1,438 @@ + +FUNCTION(KOKKOS_ARCH_OPTION SUFFIX DEV_TYPE DESCRIPTION) + #all optimizations off by default + KOKKOS_OPTION(ARCH_${SUFFIX} OFF BOOL "Optimize for ${DESCRIPTION} (${DEV_TYPE})") + IF (KOKKOS_ARCH_${SUFFIX}) + LIST(APPEND KOKKOS_ENABLED_ARCH_LIST ${SUFFIX}) + SET(KOKKOS_ENABLED_ARCH_LIST ${KOKKOS_ENABLED_ARCH_LIST} PARENT_SCOPE) + ENDIF() + SET(KOKKOS_ARCH_${SUFFIX} ${KOKKOS_ARCH_${SUFFIX}} PARENT_SCOPE) +ENDFUNCTION() + +FUNCTION(ARCH_FLAGS) + SET(COMPILERS NVIDIA PGI XL DEFAULT Cray Intel Clang AppleClang GNU) + CMAKE_PARSE_ARGUMENTS( + PARSE + "LINK_ONLY;COMPILE_ONLY" + "" + "${COMPILERS}" + ${ARGN}) + + SET(COMPILER ${KOKKOS_CXX_COMPILER_ID}) + + SET(FLAGS) + SET(NEW_COMPILE_OPTIONS) + SET(NEW_XCOMPILER_OPTIONS) + SET(NEW_LINK_OPTIONS) + LIST(APPEND NEW_XCOMPILER_OPTIONS ${KOKKOS_XCOMPILER_OPTIONS}) + LIST(APPEND NEW_COMPILE_OPTIONS ${KOKKOS_COMPILE_OPTIONS}) + LIST(APPEND NEW_LINK_OPTIONS ${KOKKOS_LINK_OPTIONS}) + FOREACH(COMP ${COMPILERS}) + IF (COMPILER STREQUAL "${COMP}") + IF (PARSE_${COMPILER}) + IF (NOT "${PARSE_${COMPILER}}" STREQUAL "NO-VALUE-SPECIFIED") + SET(FLAGS ${PARSE_${COMPILER}}) + ENDIF() + ELSEIF(PARSE_DEFAULT) + SET(FLAGS ${PARSE_DEFAULT}) + ENDIF() + ENDIF() + ENDFOREACH() + + IF (NOT LINK_ONLY) + # The funky logic here is for future handling of argument deduplication + # If we naively pass multiple -Xcompiler flags to target_compile_options + # -Xcompiler will get deduplicated and break the build + IF ("-Xcompiler" IN_LIST FLAGS) + LIST(REMOVE_ITEM FLAGS "-Xcompiler") + GLOBAL_APPEND(KOKKOS_XCOMPILER_OPTIONS ${FLAGS}) + ELSE() + GLOBAL_APPEND(KOKKOS_COMPILE_OPTIONS ${FLAGS}) + ENDIF() + ENDIF() + + IF (NOT COMPILE_ONLY) + GLOBAL_APPEND(KOKKOS_LINK_OPTIONS ${FLAGS}) + ENDIF() +ENDFUNCTION() + +# Make sure devices and compiler ID are done +KOKKOS_CFG_DEPENDS(ARCH COMPILER_ID) +KOKKOS_CFG_DEPENDS(ARCH DEVICES) +KOKKOS_CFG_DEPENDS(ARCH OPTIONS) + + +#------------------------------------------------------------------------------- +# List of possible host architectures. +#------------------------------------------------------------------------------- +SET(KOKKOS_ARCH_LIST) + + +KOKKOS_DEPRECATED_LIST(ARCH ARCH) +KOKKOS_ARCH_OPTION(AMDAVX HOST "AMD chip") +KOKKOS_ARCH_OPTION(ARMV80 HOST "ARMv8.0 Compatible CPU") +KOKKOS_ARCH_OPTION(ARMV81 HOST "ARMv8.1 Compatible CPU") +KOKKOS_ARCH_OPTION(ARMV8_THUNDERX HOST "ARMv8 Cavium ThunderX CPU") +KOKKOS_ARCH_OPTION(ARMV8_THUNDERX2 HOST "ARMv8 Cavium ThunderX2 CPU") +KOKKOS_ARCH_OPTION(WSM HOST "Intel Westmere CPU") +KOKKOS_ARCH_OPTION(SNB HOST "Intel Sandy/Ivy Bridge CPUs") +KOKKOS_ARCH_OPTION(HSW HOST "Intel Haswell CPUs") +KOKKOS_ARCH_OPTION(BDW HOST "Intel Broadwell Xeon E-class CPUs") +KOKKOS_ARCH_OPTION(SKX HOST "Intel Sky Lake Xeon E-class HPC CPUs (AVX512)") +KOKKOS_ARCH_OPTION(KNC HOST "Intel Knights Corner Xeon Phi") +KOKKOS_ARCH_OPTION(KNL HOST "Intel Knights Landing Xeon Phi") +KOKKOS_ARCH_OPTION(BGQ HOST "IBM Blue Gene Q") +KOKKOS_ARCH_OPTION(POWER7 HOST "IBM POWER7 CPUs") +KOKKOS_ARCH_OPTION(POWER8 HOST "IBM POWER8 CPUs") +KOKKOS_ARCH_OPTION(POWER9 HOST "IBM POWER9 CPUs") +KOKKOS_ARCH_OPTION(KEPLER30 GPU "NVIDIA Kepler generation CC 3.0") +KOKKOS_ARCH_OPTION(KEPLER32 GPU "NVIDIA Kepler generation CC 3.2") +KOKKOS_ARCH_OPTION(KEPLER35 GPU "NVIDIA Kepler generation CC 3.5") +KOKKOS_ARCH_OPTION(KEPLER37 GPU "NVIDIA Kepler generation CC 3.7") +KOKKOS_ARCH_OPTION(MAXWELL50 GPU "NVIDIA Maxwell generation CC 5.0") +KOKKOS_ARCH_OPTION(MAXWELL52 GPU "NVIDIA Maxwell generation CC 5.2") +KOKKOS_ARCH_OPTION(MAXWELL53 GPU "NVIDIA Maxwell generation CC 5.3") +KOKKOS_ARCH_OPTION(PASCAL60 GPU "NVIDIA Pascal generation CC 6.0") +KOKKOS_ARCH_OPTION(PASCAL61 GPU "NVIDIA Pascal generation CC 6.1") +KOKKOS_ARCH_OPTION(VOLTA70 GPU "NVIDIA Volta generation CC 7.0") +KOKKOS_ARCH_OPTION(VOLTA72 GPU "NVIDIA Volta generation CC 7.2") +KOKKOS_ARCH_OPTION(TURING75 GPU "NVIDIA Turing generation CC 7.5") +KOKKOS_ARCH_OPTION(EPYC HOST "AMD Epyc architecture") + + +IF (KOKKOS_ENABLE_CUDA) + #Regardless of version, make sure we define the general architecture name + IF (KOKKOS_ARCH_KEPLER30 OR KOKKOS_ARCH_KEPLER32 OR KOKKOS_ARCH_KEPLER35 OR KOKKOS_ARCH_KEPLER37) + SET(KOKKOS_ARCH_KEPLER ON) + ENDIF() + + #Regardless of version, make sure we define the general architecture name + IF (KOKKOS_ARCH_MAXWELL50 OR KOKKOS_ARCH_MAXWELL52 OR KOKKOS_ARCH_MAXWELL53) + SET(KOKKOS_ARCH_MAXWELL ON) + ENDIF() + + #Regardless of version, make sure we define the general architecture name + IF (KOKKOS_ARCH_PASCAL60 OR KOKKOS_ARCH_PASCAL61) + SET(KOKKOS_ARCH_PASCAL ON) + ENDIF() + + #Regardless of version, make sure we define the general architecture name + IF (KOKKOS_ARCH_VOLTA70 OR KOKKOS_ARCH_VOLTA72) + SET(KOKKOS_ARCH_VOLTA ON) + ENDIF() +ENDIF() + + + +IF(KOKKOS_ENABLE_COMPILER_WARNINGS) + SET(COMMON_WARNINGS + "-Wall" "-Wshadow" "-pedantic" + "-Wsign-compare" "-Wtype-limits" "-Wuninitialized") + + SET(GNU_WARNINGS "-Wempty-body" "-Wclobbered" "-Wignored-qualifiers" + ${COMMON_WARNINGS}) + + ARCH_FLAGS( + PGI NO-VALUE-SPECIFIED + GNU ${GNU_WARNINGS} + DEFAULT ${COMMON_WARNINGS} + ) +ENDIF() + + +#------------------------------- KOKKOS_CUDA_OPTIONS --------------------------- +GLOBAL_RESET(KOKKOS_CUDA_OPTIONS) +# Construct the Makefile options +IF (KOKKOS_ENABLE_CUDA_LAMBDA) + IF(KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) + GLOBAL_APPEND(KOKKOS_CUDA_OPTIONS "-expt-extended-lambda") + ENDIF() +ENDIF() + +IF (KOKKOS_ENABLE_CUDA_CONSTEXPR) + IF(KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) + GLOBAL_APPEND(KOKKOS_CUDA_OPTIONS "-expt-relaxed-constexpr") + ENDIF() +ENDIF() + +IF (KOKKOS_CXX_COMPILER_ID STREQUAL Clang) + SET(CUDA_ARCH_FLAG "--cuda-gpu-arch") + GLOBAL_APPEND(KOKKOS_CUDA_OPTIONS -x cuda) + IF (KOKKOS_ENABLE_CUDA) + SET(KOKKOS_IMPL_CUDA_CLANG_WORKAROUND ON CACHE BOOL "enable CUDA Clang workarounds" FORCE) + ENDIF() +ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) + SET(CUDA_ARCH_FLAG "-arch") +ENDIF() + +IF (KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) + STRING(TOUPPER "${CMAKE_BUILD_TYPE}" _UPPERCASE_CMAKE_BUILD_TYPE) + IF (KOKKOS_ENABLE_DEBUG OR _UPPERCASE_CMAKE_BUILD_TYPE STREQUAL "DEBUG") + GLOBAL_APPEND(KOKKOS_CUDA_OPTIONS -lineinfo) + ENDIF() + UNSET(_UPPERCASE_CMAKE_BUILD_TYPE) + IF (KOKKOS_CXX_COMPILER_VERSION VERSION_GREATER 9.0 OR KOKKOS_CXX_COMPILER_VERSION VERSION_EQUAL 9.0) + GLOBAL_APPEND(KOKKOS_CUDAFE_OPTIONS --diag_suppress=esa_on_defaulted_function_ignored) + ENDIF() +ENDIF() + +IF(KOKKOS_ENABLE_OPENMP) + IF (KOKKOS_CXX_COMPILER_ID STREQUAL AppleClang) + MESSAGE(FATAL_ERROR "Apple Clang does not support OpenMP. Use native Clang instead") + ENDIF() + ARCH_FLAGS( + Clang -fopenmp=libomp + PGI -mp + NVIDIA -Xcompiler -fopenmp + Cray NO-VALUE-SPECIFIED + XL -qsmp=omp + DEFAULT -fopenmp + ) +ENDIF() + +IF (KOKKOS_ARCH_ARMV80) + ARCH_FLAGS( + Cray NO-VALUE-SPECIFIED + PGI NO-VALUE-SPECIFIED + DEFAULT -march=armv8-a + ) +ENDIF() + +IF (KOKKOS_ARCH_ARMV81) + ARCH_FLAGS( + Cray NO-VALUE-SPECIFIED + PGI NO-VALUE-SPECIFIED + DEFAULT -march=armv8.1-a + ) +ENDIF() + +IF (KOKKOS_ARCH_ARMV8_THUNDERX) + SET(KOKKOS_ARCH_ARMV80 ON) #Not a cache variable + ARCH_FLAGS( + Cray NO-VALUE-SPECIFIED + PGI NO-VALUE-SPECIFIED + DEFAULT -march=armv8-a -mtune=thunderx + ) +ENDIF() + +IF (KOKKOS_ARCH_ARMV8_THUNDERX2) + SET(KOKKOS_ARCH_ARMV81 ON) #Not a cache variable + ARCH_FLAGS( + Cray NO-VALUE-SPECIFIED + PGI NO-VALUE-SPECIFIED + DEFAULT -mcpu=thunderx2t99 -mtune=thunderx2t99 + ) +ENDIF() + +IF (KOKKOS_ARCH_EPYC) + ARCH_FLAGS( + Intel -mavx2 + DEFAULT -march=znver1 -mtune=znver1 + ) + SET(KOKKOS_ARCH_AMD_EPYC ON) + SET(KOKKOS_ARCH_AMD_AVX2 ON) +ENDIF() + +IF (KOKKOS_ARCH_WSM) + ARCH_FLAGS( + Intel -xSSE4.2 + PGI -tp=nehalem + Cray NO-VALUE-SPECIFIED + DEFAULT -msse4.2 + ) + SET(KOKKOS_ARCH_SSE42 ON) +ENDIF() + +IF (KOKKOS_ARCH_SNB OR KOKKOS_ARCH_AMDAVX) + SET(KOKKOS_ARCH_AVX ON) + ARCH_FLAGS( + Intel -mavx + PGI -tp=sandybridge + Cray NO-VALUE-SPECIFIED + DEFAULT -mavx + ) +ENDIF() + +IF (KOKKOS_ARCH_HSW) + SET(KOKKOS_ARCH_AVX2 ON) + ARCH_FLAGS( + Intel -xCORE-AVX2 + PGI -tp=haswell + Cray NO-VALUE-SPECIFIED + DEFAULT -march=core-avx2 -mtune=core-avx2 + ) +ENDIF() + +IF (KOKKOS_ARCH_BDW) + SET(KOKKOS_ARCH_AVX2 ON) + ARCH_FLAGS( + Intel -xCORE-AVX2 + PGI -tp=haswell + Cray NO-VALUE-SPECIFIED + DEFAULT -march=core-avx2 -mtune=core-avx2 -mrtm + ) +ENDIF() + +IF (KOKKOS_ARCH_EPYC) + SET(KOKKOS_ARCH_AMD_AVX2 ON) + ARCH_FLAGS( + Intel -mvax2 + DEFAULT -march=znver1 -mtune=znver1 + ) +ENDIF() + +IF (KOKKOS_ARCH_KNL) + #avx512-mic + SET(KOKKOS_ARCH_AVX512MIC ON) #not a cache variable + ARCH_FLAGS( + Intel -xMIC-AVX512 + PGI NO-VALUE-SPECIFIED + Cray NO-VALUE-SPECIFIED + DEFAULT -march=knl -mtune=knl + ) +ENDIF() + +IF (KOKKOS_ARCH_KNC) + SET(KOKKOS_USE_ISA_KNC ON) + ARCH_FLAGS( + DEFAULT -mmic + ) +ENDIF() + +IF (KOKKOS_ARCH_SKX) + #avx512-xeon + SET(KOKKOS_ARCH_AVX512XEON ON) + ARCH_FLAGS( + Intel -xCORE-AVX512 + PGI NO-VALUE-SPECIFIED + Cray NO-VALUE-SPECIFIED + DEFAULT -march=skylake-avx512 -mtune=skylake-avx512 -mrtm + ) +ENDIF() + +IF (KOKKOS_ARCH_WSM OR KOKKOS_ARCH_SNB OR KOKKOS_ARCH_HSW OR KOKKOS_ARCH_BDW OR KOKKOS_ARCH_KNL OR KOKKOS_ARCH_SKX OR KOKKOS_ARCH_EPYC) + SET(KOKKOS_USE_ISA_X86_64 ON) +ENDIF() + +IF (KOKKOS_ARCH_BDW OR KOKKOS_ARCH_SKX) + SET(KOKKOS_ENABLE_TM ON) #not a cache variable +ENDIF() + +IF (KOKKOS_ARCH_POWER7) + ARCH_FLAGS( + PGI NO-VALUE-SPECIFIED + DEFAULT -mcpu=power7 -mtune=power7 + ) + SET(KOKKOS_USE_ISA_POWERPCBE ON) +ENDIF() + +IF (KOKKOS_ARCH_POWER8) + ARCH_FLAGS( + PGI NO-VALUE-SPECIFIED + NVIDIA NO-VALUE-SPECIFIED + DEFAULT -mcpu=power8 -mtune=power8 + ) +ENDIF() + +IF (KOKKOS_ARCH_POWER9) + ARCH_FLAGS( + PGI NO-VALUE-SPECIFIED + NVIDIA NO-VALUE-SPECIFIED + DEFAULT -mcpu=power9 -mtune=power9 + ) +ENDIF() + +IF (KOKKOS_ARCH_POWER8 OR KOKKOS_ARCH_POWER9) + SET(KOKKOS_USE_ISA_POWERPCLE ON) +ENDIF() + +IF (Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE) + ARCH_FLAGS( + Clang -fcuda-rdc + NVIDIA --relocatable-device-code=true + ) +ENDIF() + + +SET(CUDA_ARCH_ALREADY_SPECIFIED "") +FUNCTION(CHECK_CUDA_ARCH ARCH FLAG) +IF(KOKKOS_ARCH_${ARCH}) + IF(CUDA_ARCH_ALREADY_SPECIFIED) + MESSAGE(FATAL_ERROR "Multiple GPU architectures given! Already have ${CUDA_ARCH_ALREADY_SPECIFIED}, but trying to add ${ARCH}. If you are re-running CMake, try clearing the cache and running again.") + ENDIF() + SET(CUDA_ARCH_ALREADY_SPECIFIED ${ARCH} PARENT_SCOPE) + IF (NOT KOKKOS_ENABLE_CUDA) + MESSAGE(WARNING "Given CUDA arch ${ARCH}, but Kokkos_ENABLE_CUDA is OFF. Option will be ignored.") + UNSET(KOKKOS_ARCH_${ARCH} PARENT_SCOPE) + ELSE() + GLOBAL_APPEND(KOKKOS_CUDA_OPTIONS "${CUDA_ARCH_FLAG}=${FLAG}") + IF(KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE) + GLOBAL_APPEND(KOKKOS_LINK_OPTIONS "${CUDA_ARCH_FLAG}=${FLAG}") + ENDIF() + ENDIF() +ENDIF() +ENDFUNCTION() + + +CHECK_CUDA_ARCH(KEPLER30 sm_30) +CHECK_CUDA_ARCH(KEPLER32 sm_32) +CHECK_CUDA_ARCH(KEPLER35 sm_35) +CHECK_CUDA_ARCH(KEPLER37 sm_37) +CHECK_CUDA_ARCH(MAXWELL50 sm_50) +CHECK_CUDA_ARCH(MAXWELL52 sm_52) +CHECK_CUDA_ARCH(MAXWELL53 sm_53) +CHECK_CUDA_ARCH(PASCAL60 sm_60) +CHECK_CUDA_ARCH(PASCAL61 sm_61) +CHECK_CUDA_ARCH(VOLTA70 sm_70) +CHECK_CUDA_ARCH(VOLTA72 sm_72) +CHECK_CUDA_ARCH(TURING75 sm_75) + +#CMake verbose is kind of pointless +#Let's just always print things +MESSAGE(STATUS "Execution Spaces:") +IF(KOKKOS_ENABLE_CUDA) + MESSAGE(STATUS " Device Parallel: CUDA") +ELSE() + MESSAGE(STATUS " Device Parallel: NONE") +ENDIF() + +FOREACH (_BACKEND OPENMP PTHREAD HPX) + IF(KOKKOS_ENABLE_${_BACKEND}) + IF(_HOST_PARALLEL) + MESSAGE(FATAL_ERROR "Multiple host parallel execution spaces are not allowed! " + "Trying to enable execution space ${_BACKEND}, " + "but execution space ${_HOST_PARALLEL} is already enabled. " + "Remove the CMakeCache.txt file and re-configure.") + ENDIF() + SET(_HOST_PARALLEL ${_BACKEND}) + ENDIF() +ENDFOREACH() + +IF(NOT _HOST_PARALLEL AND NOT KOKKOS_ENABLE_SERIAL) + MESSAGE(FATAL_ERROR "At least one host execution space must be enabled, " + "but no host parallel execution space was requested " + "and Kokkos_ENABLE_SERIAL=OFF.") +ENDIF() + +IF(NOT _HOST_PARALLEL) + SET(_HOST_PARALLEL "NONE") +ENDIF() +MESSAGE(STATUS " Host Parallel: ${_HOST_PARALLEL}") +UNSET(_HOST_PARALLEL) + +IF(KOKKOS_ENABLE_PTHREAD) + SET(KOKKOS_ENABLE_THREADS ON) +ENDIF() + +IF(KOKKOS_ENABLE_SERIAL) + MESSAGE(STATUS " Host Serial: SERIAL") +ELSE() + MESSAGE(STATUS " Host Serial: NONE") +ENDIF() + +MESSAGE(STATUS "") +MESSAGE(STATUS "Architectures:") +FOREACH(Arch ${KOKKOS_ENABLED_ARCH_LIST}) + MESSAGE(STATUS " ${Arch}") +ENDFOREACH() + diff --git a/lib/kokkos/cmake/kokkos_build.cmake b/lib/kokkos/cmake/kokkos_build.cmake deleted file mode 100644 index f9b995baae4f9b319adb0dbe7527f301de8d3a9e..0000000000000000000000000000000000000000 --- a/lib/kokkos/cmake/kokkos_build.cmake +++ /dev/null @@ -1,261 +0,0 @@ -############################ Detect if submodule ############################### -# -# With thanks to StackOverflow: -# http://stackoverflow.com/questions/25199677/how-to-detect-if-current-scope-has-a-parent-in-cmake -# -get_directory_property(HAS_PARENT PARENT_DIRECTORY) -if(HAS_PARENT) - message(STATUS "Submodule build") - SET(KOKKOS_HEADER_DIR "include/kokkos") -else() - message(STATUS "Standalone build") - SET(KOKKOS_HEADER_DIR "include") -endif() - -################################ Handle the actual build ####################### - -SET(INSTALL_LIB_DIR lib CACHE PATH "Installation directory for libraries") -SET(INSTALL_BIN_DIR bin CACHE PATH "Installation directory for executables") -SET(INSTALL_INCLUDE_DIR ${KOKKOS_HEADER_DIR} CACHE PATH - "Installation directory for header files") -IF(WIN32 AND NOT CYGWIN) - SET(DEF_INSTALL_CMAKE_DIR CMake) -ELSE() - SET(DEF_INSTALL_CMAKE_DIR lib/CMake/Kokkos) -ENDIF() - -SET(INSTALL_CMAKE_DIR ${DEF_INSTALL_CMAKE_DIR} CACHE PATH - "Installation directory for CMake files") - -# Make relative paths absolute (needed later on) -FOREACH(p LIB BIN INCLUDE CMAKE) - SET(var INSTALL_${p}_DIR) - IF(NOT IS_ABSOLUTE "${${var}}") - SET(${var} "${CMAKE_INSTALL_PREFIX}/${${var}}") - ENDIF() -ENDFOREACH() - -# set up include-directories -SET (Kokkos_INCLUDE_DIRS - ${Kokkos_SOURCE_DIR}/core/src - ${Kokkos_SOURCE_DIR}/containers/src - ${Kokkos_SOURCE_DIR}/algorithms/src - ${Kokkos_BINARY_DIR} # to find KokkosCore_config.h - ${KOKKOS_INCLUDE_DIRS} -) - -# pass include dirs back to parent scope -if(HAS_PARENT) -SET(Kokkos_INCLUDE_DIRS_RET ${Kokkos_INCLUDE_DIRS} PARENT_SCOPE) -else() -SET(Kokkos_INCLUDE_DIRS_RET ${Kokkos_INCLUDE_DIRS}) -endif() - -INCLUDE_DIRECTORIES(${Kokkos_INCLUDE_DIRS}) - -IF(KOKKOS_SEPARATE_LIBS) - # Sources come from makefile-generated kokkos_generated_settings.cmake file - # Separate libs need to separate the sources - set_kokkos_srcs(KOKKOS_SRC ${KOKKOS_SRC}) - - # kokkoscore - ADD_LIBRARY( - kokkoscore - ${KOKKOS_CORE_SRCS} - ) - - target_compile_options( - kokkoscore - PUBLIC $<$:${KOKKOS_CXX_FLAGS}> - ) - - target_include_directories( - kokkoscore - PUBLIC - ${KOKKOS_TPL_INCLUDE_DIRS} - ) - - foreach(lib IN LISTS KOKKOS_TPL_LIBRARY_NAMES) - if (("${lib}" STREQUAL "cuda") AND (NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")) - set(LIB_cuda "-lcuda") - elseif ("${lib}" STREQUAL "hpx") - find_package(HPX REQUIRED) - if(${HPX_FOUND}) - target_link_libraries(kokkoscore PUBLIC ${HPX_LIBRARIES}) - target_link_libraries(kokkoscontainers PUBLIC ${HPX_LIBRARIES}) - target_link_libraries(kokkosalgorithms PUBLIC ${HPX_LIBRARIES}) - target_include_directories(kokkoscore PUBLIC ${HPX_INCLUDE_DIRS}) - target_include_directories(kokkoscontainers PUBLIC ${HPX_INCLUDE_DIRS}) - target_include_directories(kokkosalgorithms PUBLIC ${HPX_INCLUDE_DIRS}) - else() - message(ERROR "HPX not found. Check the value of HPX_DIR (= ${HPX_DIR}) or CMAKE_PREFIX_PATH (= ${CMAKE_PREFIX_PATH}).") - endif() - else() - find_library(LIB_${lib} ${lib} PATHS ${KOKKOS_TPL_LIBRARY_DIRS}) - endif() - target_link_libraries(kokkoscore PUBLIC ${LIB_${lib}}) - endforeach() - - target_link_libraries(kokkoscore PUBLIC "${KOKKOS_LINK_FLAGS}") - - # Install the kokkoscore library - INSTALL (TARGETS kokkoscore - EXPORT KokkosTargets - ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/lib - LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib - RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin - ) - - # kokkoscontainers - if (DEFINED KOKKOS_CONTAINERS_SRCS) - ADD_LIBRARY( - kokkoscontainers - ${KOKKOS_CONTAINERS_SRCS} - ) - endif() - - TARGET_LINK_LIBRARIES( - kokkoscontainers - kokkoscore - ) - - # Install the kokkocontainers library - INSTALL (TARGETS kokkoscontainers - EXPORT KokkosTargets - ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/lib - LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib - RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) - - # kokkosalgorithms - Build as interface library since no source files. - ADD_LIBRARY( - kokkosalgorithms - INTERFACE - ) - - target_include_directories( - kokkosalgorithms - INTERFACE ${Kokkos_SOURCE_DIR}/algorithms/src - ) - - TARGET_LINK_LIBRARIES( - kokkosalgorithms - INTERFACE kokkoscore - ) - - # Install the kokkoalgorithms library - INSTALL (TARGETS kokkosalgorithms - ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/lib - LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib - RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) - - SET (Kokkos_LIBRARIES_NAMES kokkoscore kokkoscontainers kokkosalgorithms) - -ELSE() - # kokkos - ADD_LIBRARY( - kokkos - ${KOKKOS_CORE_SRCS} - ${KOKKOS_CONTAINERS_SRCS} - ) - - target_compile_options( - kokkos - PUBLIC $<$:${KOKKOS_CXX_FLAGS}> - ) - - target_include_directories( - kokkos - PUBLIC - ${KOKKOS_TPL_INCLUDE_DIRS} - ) - - foreach(lib IN LISTS KOKKOS_TPL_LIBRARY_NAMES) - if (("${lib}" STREQUAL "cuda") AND (NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")) - set(LIB_cuda "-lcuda") - elseif ("${lib}" STREQUAL "hpx") - find_package(HPX REQUIRED) - if(${HPX_FOUND}) - target_link_libraries(kokkos PUBLIC ${HPX_LIBRARIES}) - target_include_directories(kokkos PUBLIC ${HPX_INCLUDE_DIRS}) - else() - message(ERROR "HPX not found. Check the value of HPX_DIR (= ${HPX_DIR}) or CMAKE_PREFIX_PATH (= ${CMAKE_PREFIX_PATH}).") - endif() - else() - find_library(LIB_${lib} ${lib} PATHS ${KOKKOS_TPL_LIBRARY_DIRS}) - endif() - target_link_libraries(kokkos PUBLIC ${LIB_${lib}}) - endforeach() - - target_link_libraries(kokkos PUBLIC "${KOKKOS_LINK_FLAGS}") - - # Install the kokkos library - INSTALL (TARGETS kokkos - EXPORT KokkosTargets - ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/lib - LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib - RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) - - - SET (Kokkos_LIBRARIES_NAMES kokkos) - -endif() # KOKKOS_SEPARATE_LIBS - -# Install the kokkos headers -INSTALL (DIRECTORY - EXPORT KokkosTargets - ${Kokkos_SOURCE_DIR}/core/src/ - DESTINATION ${KOKKOS_HEADER_DIR} - FILES_MATCHING PATTERN "*.hpp" -) -INSTALL (DIRECTORY - EXPORT KokkosTargets - ${Kokkos_SOURCE_DIR}/containers/src/ - DESTINATION ${KOKKOS_HEADER_DIR} - FILES_MATCHING PATTERN "*.hpp" -) -INSTALL (DIRECTORY - EXPORT KokkosTargets - ${Kokkos_SOURCE_DIR}/algorithms/src/ - DESTINATION ${KOKKOS_HEADER_DIR} - FILES_MATCHING PATTERN "*.hpp" -) - -INSTALL (FILES - ${Kokkos_BINARY_DIR}/KokkosCore_config.h - DESTINATION ${KOKKOS_HEADER_DIR} -) - -# Add all targets to the build-tree export set -export(TARGETS ${Kokkos_LIBRARIES_NAMES} - FILE "${Kokkos_BINARY_DIR}/KokkosTargets.cmake") - -# Export the package for use from the build-tree -# (this registers the build-tree with a global CMake-registry) -export(PACKAGE Kokkos) - -# Create the KokkosConfig.cmake and KokkosConfigVersion files -file(RELATIVE_PATH REL_INCLUDE_DIR "${INSTALL_CMAKE_DIR}" - "${INSTALL_INCLUDE_DIR}") -# ... for the build tree -set(CONF_INCLUDE_DIRS "${Kokkos_SOURCE_DIR}" "${Kokkos_BINARY_DIR}") -configure_file(${Kokkos_SOURCE_DIR}/cmake/KokkosConfig.cmake.in - "${Kokkos_BINARY_DIR}/KokkosConfig.cmake" @ONLY) -# ... for the install tree -set(CONF_INCLUDE_DIRS "\${Kokkos_CMAKE_DIR}/${REL_INCLUDE_DIR}") -configure_file(${Kokkos_SOURCE_DIR}/cmake/KokkosConfig.cmake.in - "${Kokkos_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/KokkosConfig.cmake" @ONLY) - -# Install the KokkosConfig.cmake and KokkosConfigVersion.cmake -install(FILES - "${Kokkos_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/KokkosConfig.cmake" - DESTINATION "${INSTALL_CMAKE_DIR}") - -#This seems not to do anything? -#message(STATUS "KokkosTargets: " ${KokkosTargets}) -# Install the export set for use with the install-tree -INSTALL(EXPORT KokkosTargets DESTINATION - "${INSTALL_CMAKE_DIR}") - -# build and install pkgconfig file -CONFIGURE_FILE(core/src/kokkos.pc.in kokkos.pc @ONLY) -INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/kokkos.pc DESTINATION lib/pkgconfig) diff --git a/lib/kokkos/cmake/kokkos_compiler_id.cmake b/lib/kokkos/cmake/kokkos_compiler_id.cmake new file mode 100644 index 0000000000000000000000000000000000000000..d239c3b32e5d2d853a6a5ad9705a49aec895c5a5 --- /dev/null +++ b/lib/kokkos/cmake/kokkos_compiler_id.cmake @@ -0,0 +1,80 @@ +KOKKOS_CFG_DEPENDS(COMPILER_ID NONE) + +SET(KOKKOS_CXX_COMPILER ${CMAKE_CXX_COMPILER}) +SET(KOKKOS_CXX_COMPILER_ID ${CMAKE_CXX_COMPILER_ID}) +SET(KOKKOS_CXX_COMPILER_VERSION ${CMAKE_CXX_COMPILER_VERSION}) + +# Check if the compiler is nvcc (which really means nvcc_wrapper). +EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} --version + COMMAND grep nvcc + COMMAND wc -l + OUTPUT_VARIABLE INTERNAL_HAVE_COMPILER_NVCC + OUTPUT_STRIP_TRAILING_WHITESPACE) + + +STRING(REGEX REPLACE "^ +" "" + INTERNAL_HAVE_COMPILER_NVCC ${INTERNAL_HAVE_COMPILER_NVCC}) + + +IF(INTERNAL_HAVE_COMPILER_NVCC) + # SET the compiler id to nvcc. We use the value used by CMake 3.8. + SET(KOKKOS_CXX_COMPILER_ID NVIDIA CACHE STRING INTERNAL FORCE) + + # SET nvcc's compiler version. + EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} --version + COMMAND grep release + OUTPUT_VARIABLE INTERNAL_CXX_COMPILER_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE) + + STRING(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]+$" + TEMP_CXX_COMPILER_VERSION ${INTERNAL_CXX_COMPILER_VERSION}) + SET(KOKKOS_CXX_COMPILER_VERSION ${TEMP_CXX_COMPILER_VERSION} CACHE STRING INTERNAL FORCE) +ENDIF() + +IF(KOKKOS_CXX_COMPILER_ID STREQUAL Cray) + + # SET nvcc's compiler version. + EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} --version + OUTPUT_VARIABLE INTERNAL_CXX_COMPILER_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE) + + STRING(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]+$" + TEMP_CXX_COMPILER_VERSION ${INTERNAL_CXX_COMPILER_VERSION}) + SET(KOKKOS_CXX_COMPILER_VERSION ${TEMP_CXX_COMPILER_VERSION} CACHE STRING INTERNAL FORCE) +ENDIF() + +# Enforce the minimum compilers supported by Kokkos. +SET(KOKKOS_MESSAGE_TEXT "Compiler not supported by Kokkos. Required compiler versions:") +SET(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n Clang 3.5.2 or higher") +SET(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n GCC 4.8.4 or higher") +SET(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n Intel 15.0.2 or higher") +SET(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n NVCC 9.0.69 or higher") +SET(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n PGI 17.1 or higher\n") + +IF(KOKKOS_CXX_COMPILER_ID STREQUAL Clang) + IF(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS 3.5.2) + MESSAGE(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") + ENDIF() +ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL GNU) + IF(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS 4.8.4) + MESSAGE(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") + ENDIF() +ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL Intel) + IF(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS 15.0.2) + MESSAGE(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") + ENDIF() +ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) + IF(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS 9.0.69) + MESSAGE(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") + ENDIF() + SET(CMAKE_CXX_EXTENSIONS OFF CACHE BOOL "Kokkos turns off CXX extensions" FORCE) +ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL PGI) + IF(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS 17.1) + MESSAGE(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") + ENDIF() +ENDIF() + +STRING(REPLACE "." ";" VERSION_LIST ${KOKKOS_CXX_COMPILER_VERSION}) +LIST(GET VERSION_LIST 0 KOKKOS_COMPILER_VERSION_MAJOR) +LIST(GET VERSION_LIST 1 KOKKOS_COMPILER_VERSION_MINOR) +LIST(GET VERSION_LIST 2 KOKKOS_COMPILER_VERSION_PATCH) diff --git a/lib/kokkos/cmake/kokkos_corner_cases.cmake b/lib/kokkos/cmake/kokkos_corner_cases.cmake new file mode 100644 index 0000000000000000000000000000000000000000..c03c385faf69a533fd0f91107fd07668fbb9cdab --- /dev/null +++ b/lib/kokkos/cmake/kokkos_corner_cases.cmake @@ -0,0 +1,35 @@ +IF(KOKKOS_CXX_COMPILER_ID STREQUAL Clang AND KOKKOS_ENABLE_OPENMP) + # The clang "version" doesn't actually tell you what runtimes and tools + # were built into Clang. We should therefore make sure that libomp + # was actually built into Clang. Otherwise the user will get nonsensical + # errors when they try to build. + + #Try compile is the height of CMake nonsense + #I can't just give it compiler and link flags + #I have to hackily pretend that compiler flags are compiler definitions + #and that linker flags are libraries + #also - this is easier to use than CMakeCheckCXXSourceCompiles + TRY_COMPILE(CLANG_HAS_OMP + ${KOKKOS_TOP_BUILD_DIR}/corner_cases + ${KOKKOS_SOURCE_DIR}/cmake/compile_tests/clang_omp.cpp + COMPILE_DEFINITIONS -fopenmp=libomp + LINK_LIBRARIES -fopenmp=libomp + ) + IF (NOT CLANG_HAS_OMP) + UNSET(CLANG_HAS_OMP CACHE) #make sure CMake always re-runs this + MESSAGE(FATAL_ERROR "Clang failed OpenMP check. You have requested -DKokkos_ENABLE_OPENMP=ON, but the Clang compiler does not appear to have been built with OpenMP support") + ENDIF() + UNSET(CLANG_HAS_OMP CACHE) #make sure CMake always re-runs this +ENDIF() + + +IF (KOKKOS_CXX_STANDARD STREQUAL 17) + IF (KOKKOS_CXX_COMPILER_ID STREQUAL GNU AND KOKKOS_CXX_COMPILER_VERSION VERSION_LESS 7) + MESSAGE(FATAL_ERROR "You have requested c++17 support for GCC ${KOKKOS_CXX_COMPILER_VERSION}. Although CMake has allowed this and GCC accepts -std=c++1z/c++17, GCC <= 6 does not properly support *this capture. Please reduce the C++ standard to 14 or upgrade the compiler if you do need 17 support") + ENDIF() + + IF (KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) + MESSAGE(FATAL_ERROR "You have requested c++17 support for NVCC. Please reduce the C++ standard to 14. No versions of NVCC currently support 17.") + ENDIF() +ENDIF() + diff --git a/lib/kokkos/cmake/kokkos_enable_devices.cmake b/lib/kokkos/cmake/kokkos_enable_devices.cmake new file mode 100644 index 0000000000000000000000000000000000000000..ff098766736dd80efb121fb691c8466e5f99f404 --- /dev/null +++ b/lib/kokkos/cmake/kokkos_enable_devices.cmake @@ -0,0 +1,61 @@ + +FUNCTION(KOKKOS_DEVICE_OPTION SUFFIX DEFAULT DEV_TYPE DOCSTRING) + KOKKOS_OPTION(ENABLE_${SUFFIX} ${DEFAULT} BOOL ${DOCSTRING}) + STRING(TOUPPER ${SUFFIX} UC_NAME) + IF (KOKKOS_ENABLE_${UC_NAME}) + LIST(APPEND KOKKOS_ENABLED_DEVICES ${SUFFIX}) + #I hate that CMake makes me do this + SET(KOKKOS_ENABLED_DEVICES ${KOKKOS_ENABLED_DEVICES} PARENT_SCOPE) + ENDIF() + SET(KOKKOS_ENABLE_${UC_NAME} ${KOKKOS_ENABLE_${UC_NAME}} PARENT_SCOPE) + IF (KOKKOS_ENABLE_${UC_NAME} AND DEV_TYPE STREQUAL "HOST") + SET(KOKKOS_HAS_HOST ON PARENT_SCOPE) + ENDIF() +ENDFUNCTION() + +KOKKOS_CFG_DEPENDS(DEVICES NONE) + +# Put a check in just in case people are using this option +KOKKOS_DEPRECATED_LIST(DEVICES ENABLE) + + +KOKKOS_DEVICE_OPTION(PTHREAD OFF HOST "Whether to build Pthread backend") +IF (KOKKOS_ENABLE_PTHREAD) + #patch the naming here + SET(KOKKOS_ENABLE_THREADS ON) +ENDIF() + +IF(Trilinos_ENABLE_Kokkos AND Trilinos_ENABLE_OpenMP) + SET(OMP_DEFAULT ON) +ELSE() + SET(OMP_DEFAULT OFF) +ENDIF() +KOKKOS_DEVICE_OPTION(OPENMP ${OMP_DEFAULT} HOST "Whether to build OpenMP backend") + +IF(Trilinos_ENABLE_Kokkos AND TPL_ENABLE_CUDA) + SET(CUDA_DEFAULT ON) +ELSE() + SET(CUDA_DEFAULT OFF) +ENDIF() +KOKKOS_DEVICE_OPTION(CUDA ${CUDA_DEFAULT} DEVICE "Whether to build CUDA backend") + +IF (KOKKOS_ENABLE_CUDA) + GLOBAL_SET(KOKKOS_DONT_ALLOW_EXTENSIONS "CUDA enabled") +ENDIF() + +# We want this to default to OFF for cache reasons, but if no +# host space is given, then activate serial +IF (KOKKOS_HAS_TRILINOS) + #However, Trilinos always wants Serial ON + SET(SERIAL_DEFAULT ON) +ELSEIF (KOKKOS_HAS_HOST) + SET(SERIAL_DEFAULT OFF) +ELSE() + SET(SERIAL_DEFAULT ON) + IF (NOT DEFINED Kokkos_ENABLE_SERIAL) + MESSAGE(STATUS "SERIAL backend is being turned on to ensure there is at least one Host space. To change this, you must enable another host execution space and configure with -DKokkos_ENABLE_SERIAL=OFF or change CMakeCache.txt") + ENDIF() +ENDIF() +KOKKOS_DEVICE_OPTION(SERIAL ${SERIAL_DEFAULT} HOST "Whether to build serial backend") + +KOKKOS_DEVICE_OPTION(HPX OFF HOST "Whether to build HPX backend (experimental)") diff --git a/lib/kokkos/cmake/kokkos_enable_options.cmake b/lib/kokkos/cmake/kokkos_enable_options.cmake new file mode 100644 index 0000000000000000000000000000000000000000..c0e49482b66c51de0f897bf9730c7db35436e787 --- /dev/null +++ b/lib/kokkos/cmake/kokkos_enable_options.cmake @@ -0,0 +1,92 @@ +########################## NOTES ############################################### +# List the options for configuring kokkos using CMake method of doing it. +# These options then get mapped onto KOKKOS_SETTINGS environment variable by +# kokkos_settings.cmake. It is separate to allow other packages to override +# these variables (e.g., TriBITS). + +########################## AVAILABLE OPTIONS ################################### +# Use lists for documentation, verification, and programming convenience + + +FUNCTION(KOKKOS_ENABLE_OPTION SUFFIX DEFAULT DOCSTRING) + KOKKOS_OPTION(ENABLE_${SUFFIX} ${DEFAULT} BOOL ${DOCSTRING}) + STRING(TOUPPER ${SUFFIX} UC_NAME) + IF (KOKKOS_ENABLE_${UC_NAME}) + LIST(APPEND KOKKOS_ENABLED_OPTIONS ${UC_NAME}) + #I hate that CMake makes me do this + SET(KOKKOS_ENABLED_OPTIONS ${KOKKOS_ENABLED_OPTIONS} PARENT_SCOPE) + ENDIF() + SET(KOKKOS_ENABLE_${UC_NAME} ${KOKKOS_ENABLE_${UC_NAME}} PARENT_SCOPE) +ENDFUNCTION() + +# Certain defaults will depend on knowing the enabled devices +KOKKOS_CFG_DEPENDS(OPTIONS DEVICES) + +# Put a check in just in case people are using this option +KOKKOS_DEPRECATED_LIST(OPTIONS ENABLE) + +KOKKOS_ENABLE_OPTION(CUDA_RELOCATABLE_DEVICE_CODE OFF "Whether to enable relocatable device code (RDC) for CUDA") +KOKKOS_ENABLE_OPTION(CUDA_UVM OFF "Whether to use unified memory (UM) for CUDA by default") +KOKKOS_ENABLE_OPTION(CUDA_LDG_INTRINSIC OFF "Whether to use CUDA LDG intrinsics") +KOKKOS_ENABLE_OPTION(HPX_ASYNC_DISPATCH OFF "Whether HPX supports asynchronous dispatch") +KOKKOS_ENABLE_OPTION(TESTS OFF "Whether to build the unit tests") +STRING(TOUPPER "${CMAKE_BUILD_TYPE}" UPPERCASE_CMAKE_BUILD_TYPE) +IF(UPPERCASE_CMAKE_BUILD_TYPE STREQUAL "DEBUG") + KOKKOS_ENABLE_OPTION(DEBUG ON "Whether to activate extra debug features - may increase compile times") + KOKKOS_ENABLE_OPTION(DEBUG_DUALVIEW_MODIFY_CHECK ON "Debug check on dual views") +ELSE() + KOKKOS_ENABLE_OPTION(DEBUG OFF "Whether to activate extra debug features - may increase compile times") + KOKKOS_ENABLE_OPTION(DEBUG_DUALVIEW_MODIFY_CHECK OFF "Debug check on dual views") +ENDIF() +UNSET(_UPPERCASE_CMAKE_BUILD_TYPE) +KOKKOS_ENABLE_OPTION(LARGE_MEM_TESTS OFF "Whether to perform extra large memory tests") +KOKKOS_ENABLE_OPTION(DEBUG_BOUNDS_CHECK OFF "Whether to use bounds checking - will increase runtime") +KOKKOS_ENABLE_OPTION(COMPILER_WARNINGS OFF "Whether to print all compiler warnings") +KOKKOS_ENABLE_OPTION(PROFILING ON "Whether to create bindings for profiling tools") +KOKKOS_ENABLE_OPTION(PROFILING_LOAD_PRINT OFF "Whether to print information about which profiling tools got loaded") +KOKKOS_ENABLE_OPTION(AGGRESSIVE_VECTORIZATION OFF "Whether to aggressively vectorize loops") +KOKKOS_ENABLE_OPTION(DEPRECATED_CODE OFF "Whether to enable deprecated code") + +IF (KOKKOS_ENABLE_CUDA) + SET(KOKKOS_COMPILER_CUDA_VERSION "${KOKKOS_COMPILER_VERSION_MAJOR}${KOKKOS_COMPILER_VERSION_MINOR}") +ENDIF() + +IF (Trilinos_ENABLE_Kokkos AND TPL_ENABLE_CUDA AND DEFINED KOKKOS_COMPILER_CUDA_VERSION AND KOKKOS_COMPILER_CUDA_VERSION GREATER 70) + SET(LAMBDA_DEFAULT ON) +ELSE() + SET(LAMBDA_DEFAULT OFF) +ENDIF() +KOKKOS_ENABLE_OPTION(CUDA_LAMBDA ${LAMBDA_DEFAULT} "Whether to activate experimental lambda features") +IF (Trilinos_ENABLE_Kokkos) + SET(COMPLEX_ALIGN_DEFAULT OFF) +ELSE() + SET(COMPLEX_ALIGN_DEFAULT ON) +ENDIF() +KOKKOS_ENABLE_OPTION(COMPLEX_ALIGN ${COMPLEX_ALIGN_DEFAULT} "Whether to align Kokkos::complex to 2*alignof(RealType)") + +KOKKOS_ENABLE_OPTION(CUDA_CONSTEXPR OFF "Whether to activate experimental relaxed constexpr functions") + +FUNCTION(check_device_specific_options) + CMAKE_PARSE_ARGUMENTS(SOME "" "DEVICE" "OPTIONS" ${ARGN}) + IF(NOT KOKKOS_ENABLE_${SOME_DEVICE}) + FOREACH(OPTION ${SOME_OPTIONS}) + IF(CMAKE_VERSION VERSION_GREATER_EQUAL 3.14) + IF(NOT DEFINED CACHE{Kokkos_ENABLE_${OPTION}} OR NOT DEFINED CACHE{Kokkos_ENABLE_${SOME_DEVICE}}) + MESSAGE(FATAL_ERROR "Internal logic error: option '${OPTION}' or device '${SOME_DEVICE}' not recognized.") + ENDIF() + ENDIF() + IF(KOKKOS_ENABLE_${OPTION}) + MESSAGE(WARNING "Kokkos_ENABLE_${OPTION} is ON but ${SOME_DEVICE} backend is not enabled. Option will be ignored.") + UNSET(KOKKOS_ENABLE_${OPTION} PARENT_SCOPE) + ENDIF() + ENDFOREACH() + ENDIF() +ENDFUNCTION() + +CHECK_DEVICE_SPECIFIC_OPTIONS(DEVICE CUDA OPTIONS CUDA_UVM CUDA_RELOCATABLE_DEVICE_CODE CUDA_LAMBDA CUDA_CONSTEXPR CUDA_LDG_INTRINSIC) +CHECK_DEVICE_SPECIFIC_OPTIONS(DEVICE HPX OPTIONS HPX_ASYNC_DISPATCH) + +# Needed due to change from deprecated name to new header define name +IF (KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION) + SET(KOKKOS_OPT_RANGE_AGGRESSIVE_VECTORIZATION ON) +ENDIF() diff --git a/lib/kokkos/cmake/kokkos_functions.cmake b/lib/kokkos/cmake/kokkos_functions.cmake index 616618753b11544367ead73b725884cbf9affee8..3644c48ddde7c861e29727805323cbb21ac37d33 100644 --- a/lib/kokkos/cmake/kokkos_functions.cmake +++ b/lib/kokkos/cmake/kokkos_functions.cmake @@ -1,345 +1,700 @@ ################################### FUNCTIONS ################################## # List of functions -# set_kokkos_cxx_compiler -# set_kokkos_cxx_standard -# set_kokkos_srcs - -#------------------------------------------------------------------------------- -# function(set_kokkos_cxx_compiler) -# Sets the following compiler variables that are analogous to the CMAKE_* -# versions. We add the ability to detect NVCC (really nvcc_wrapper). -# KOKKOS_CXX_COMPILER -# KOKKOS_CXX_COMPILER_ID -# KOKKOS_CXX_COMPILER_VERSION -# -# Inputs: -# KOKKOS_ENABLE_CUDA -# CMAKE_CXX_COMPILER -# CMAKE_CXX_COMPILER_ID -# CMAKE_CXX_COMPILER_VERSION -# -# Also verifies the compiler version meets the minimum required by Kokkos. -function(set_kokkos_cxx_compiler) - # Since CMake doesn't recognize the nvcc compiler until 3.8, we use our own - # version of the CMake variables and detect nvcc ourselves. Initially set to - # the CMake variable values. - set(INTERNAL_CXX_COMPILER ${CMAKE_CXX_COMPILER}) - set(INTERNAL_CXX_COMPILER_ID ${CMAKE_CXX_COMPILER_ID}) - set(INTERNAL_CXX_COMPILER_VERSION ${CMAKE_CXX_COMPILER_VERSION}) - - # Check if the compiler is nvcc (which really means nvcc_wrapper). - execute_process(COMMAND ${INTERNAL_CXX_COMPILER} --version - COMMAND grep nvcc - COMMAND wc -l - OUTPUT_VARIABLE INTERNAL_HAVE_COMPILER_NVCC - OUTPUT_STRIP_TRAILING_WHITESPACE) - - string(REGEX REPLACE "^ +" "" - INTERNAL_HAVE_COMPILER_NVCC ${INTERNAL_HAVE_COMPILER_NVCC}) - - if(INTERNAL_HAVE_COMPILER_NVCC) - # Set the compiler id to nvcc. We use the value used by CMake 3.8. - set(INTERNAL_CXX_COMPILER_ID NVIDIA) - - # Set nvcc's compiler version. - execute_process(COMMAND ${INTERNAL_CXX_COMPILER} --version - COMMAND grep release - OUTPUT_VARIABLE INTERNAL_CXX_COMPILER_VERSION - OUTPUT_STRIP_TRAILING_WHITESPACE) - - string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]+$" - INTERNAL_CXX_COMPILER_VERSION ${INTERNAL_CXX_COMPILER_VERSION}) - endif() - - # Enforce the minimum compilers supported by Kokkos. - set(KOKKOS_MESSAGE_TEXT "Compiler not supported by Kokkos. Required compiler versions:") - set(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n Clang 3.5.2 or higher") - set(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n GCC 4.8.4 or higher") - set(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n Intel 15.0.2 or higher") - set(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n NVCC 7.0.28 or higher") - set(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n PGI 17.1 or higher\n") - - if(INTERNAL_CXX_COMPILER_ID STREQUAL Clang) - if(INTERNAL_CXX_COMPILER_VERSION VERSION_LESS 3.5.2) - message(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") - endif() - elseif(INTERNAL_CXX_COMPILER_ID STREQUAL GNU) - if(INTERNAL_CXX_COMPILER_VERSION VERSION_LESS 4.8.4) - message(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") - endif() - elseif(INTERNAL_CXX_COMPILER_ID STREQUAL Intel) - if(INTERNAL_CXX_COMPILER_VERSION VERSION_LESS 15.0.2) - message(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") - endif() - elseif(INTERNAL_CXX_COMPILER_ID STREQUAL NVIDIA) - if(INTERNAL_CXX_COMPILER_VERSION VERSION_LESS 7.0.28) - message(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") - endif() - elseif(INTERNAL_CXX_COMPILER_ID STREQUAL PGI) - if(INTERNAL_CXX_COMPILER_VERSION VERSION_LESS 17.1) - message(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") - endif() - endif() - - # Enforce that extensions are turned off for nvcc_wrapper. - if(INTERNAL_CXX_COMPILER_ID STREQUAL NVIDIA) - if(DEFINED CMAKE_CXX_EXTENSIONS AND CMAKE_CXX_EXTENSIONS STREQUAL ON) - message(FATAL_ERROR "NVCC doesn't support C++ extensions. Set CMAKE_CXX_EXTENSIONS to OFF in your CMakeLists.txt.") - endif() - endif() - - if(KOKKOS_ENABLE_CUDA) - # Enforce that the compiler can compile CUDA code. - if(INTERNAL_CXX_COMPILER_ID STREQUAL Clang) - if(INTERNAL_CXX_COMPILER_VERSION VERSION_LESS 4.0.0) - message(FATAL_ERROR "Compiling CUDA code directly with Clang requires version 4.0.0 or higher.") - endif() - elseif(NOT INTERNAL_CXX_COMPILER_ID STREQUAL NVIDIA) - message(FATAL_ERROR "Invalid compiler for CUDA. The compiler must be nvcc_wrapper or Clang, but compiler ID was ${INTERNAL_CXX_COMPILER_ID}") - endif() - endif() - - set(KOKKOS_CXX_COMPILER ${INTERNAL_CXX_COMPILER} PARENT_SCOPE) - set(KOKKOS_CXX_COMPILER_ID ${INTERNAL_CXX_COMPILER_ID} PARENT_SCOPE) - set(KOKKOS_CXX_COMPILER_VERSION ${INTERNAL_CXX_COMPILER_VERSION} PARENT_SCOPE) -endfunction() - -#------------------------------------------------------------------------------- -# function(set_kokkos_cxx_standard) -# Transitively enforces that the appropriate CXX standard compile flags (C++11 -# or above) are added to targets that use the Kokkos library. Compile features -# are used if possible. Otherwise, the appropriate flags are added to -# KOKKOS_CXX_FLAGS. Values set by the user to CMAKE_CXX_STANDARD and -# CMAKE_CXX_EXTENSIONS are honored. -# -# Outputs: -# KOKKOS_CXX11_FEATURES -# KOKKOS_CXX_FLAGS -# -# Inputs: -# KOKKOS_CXX_COMPILER -# KOKKOS_CXX_COMPILER_ID -# KOKKOS_CXX_COMPILER_VERSION -# -function(set_kokkos_cxx_standard) - # The following table lists the versions of CMake that supports CXX_STANDARD - # and the CXX compile features for different compilers. The versions are - # based on CMake documentation, looking at CMake code, and verifying by - # testing with specific CMake versions. - # - # COMPILER CXX_STANDARD Compile Features - # --------------------------------------------------------------- - # Clang 3.1 3.1 - # GNU 3.1 3.2 - # AppleClang 3.2 3.2 - # Intel 3.6 3.6 - # Cray No No - # PGI No No - # XL No No - # - # For compiling CUDA code using nvcc_wrapper, we will use the host compiler's - # flags for turning on C++11. Since for compiler ID and versioning purposes - # CMake recognizes the host compiler when calling nvcc_wrapper, this just - # works. Both NVCC and nvcc_wrapper only recognize '-std=c++11' which means - # that we can only use host compilers for CUDA builds that use those flags. - # It also means that extensions (gnu++11) can't be turned on for CUDA builds. - - # Check if we can use compile features. - if(NOT KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) - if(CMAKE_CXX_COMPILER_ID STREQUAL Clang) - if(NOT CMAKE_VERSION VERSION_LESS 3.1) - set(INTERNAL_USE_COMPILE_FEATURES ON) - endif() - elseif(CMAKE_CXX_COMPILER_ID STREQUAL AppleClang OR CMAKE_CXX_COMPILER_ID STREQUAL GNU) - if(NOT CMAKE_VERSION VERSION_LESS 3.2) - set(INTERNAL_USE_COMPILE_FEATURES ON) - endif() - elseif(CMAKE_CXX_COMPILER_ID STREQUAL Intel) - if(NOT CMAKE_VERSION VERSION_LESS 3.6) - set(INTERNAL_USE_COMPILE_FEATURES ON) - endif() - endif() - endif() - - if(INTERNAL_USE_COMPILE_FEATURES) - # Use the compile features aspect of CMake to transitively cause C++ flags - # to populate to user code. - - # I'm using a hack by requiring features that I know force the lowest version - # of the compilers we want to support. Clang 3.3 and later support all of - # the C++11 standard. With CMake 3.8 and higher, we could switch to using - # cxx_std_11. - set(KOKKOS_CXX11_FEATURES - cxx_nonstatic_member_init # Forces GCC 4.7 or later and Intel 14.0 or later. - PARENT_SCOPE - ) - else() - # CXX compile features are not yet implemented for this combination of - # compiler and version of CMake. - - if(CMAKE_CXX_COMPILER_ID STREQUAL AppleClang) - # Versions of CMAKE before 3.2 don't support CXX_STANDARD or C++ compile - # features for the AppleClang compiler. Set compiler flags transitively - # here such that they trickle down to a call to target_compile_options(). - - # The following two blocks of code were copied from - # /Modules/Compiler/AppleClang-CXX.cmake from CMake 3.7.2 and then - # modified. - if(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.0) - set(INTERNAL_CXX11_STANDARD_COMPILE_OPTION "-std=c++11") - set(INTERNAL_CXX11_EXTENSION_COMPILE_OPTION "-std=gnu++11") - endif() - - if(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 6.1) - set(INTERNAL_CXX14_STANDARD_COMPILE_OPTION "-std=c++14") - set(INTERNAL_CXX14_EXTENSION_COMPILE_OPTION "-std=gnu++14") - elseif(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.1) - # AppleClang 5.0 knows this flag, but does not set a __cplusplus macro - # greater than 201103L. - set(INTERNAL_CXX14_STANDARD_COMPILE_OPTION "-std=c++1y") - set(INTERNAL_CXX14_EXTENSION_COMPILE_OPTION "-std=gnu++1y") - endif() - elseif(CMAKE_CXX_COMPILER_ID STREQUAL Intel) - # Versions of CMAKE before 3.6 don't support CXX_STANDARD or C++ compile - # features for the Intel compiler. Set compiler flags transitively here - # such that they trickle down to a call to target_compile_options(). - - # The following three blocks of code were copied from - # /Modules/Compiler/Intel-CXX.cmake from CMake 3.7.2 and then modified. - if("x${CMAKE_CXX_SIMULATE_ID}" STREQUAL "xMSVC") - set(_std -Qstd) - set(_ext c++) - else() - set(_std -std) - set(_ext gnu++) - endif() - - if(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 15.0.2) - set(INTERNAL_CXX14_STANDARD_COMPILE_OPTION "${_std}=c++14") - # TODO: There is no gnu++14 value supported; figure out what to do. - set(INTERNAL_CXX14_EXTENSION_COMPILE_OPTION "${_std}=c++14") - elseif(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 15.0.0) - set(INTERNAL_CXX14_STANDARD_COMPILE_OPTION "${_std}=c++1y") - # TODO: There is no gnu++14 value supported; figure out what to do. - set(INTERNAL_CXX14_EXTENSION_COMPILE_OPTION "${_std}=c++1y") - endif() - - if(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 13.0) - set(INTERNAL_CXX11_STANDARD_COMPILE_OPTION "${_std}=c++11") - set(INTERNAL_CXX11_EXTENSION_COMPILE_OPTION "${_std}=${_ext}11") - elseif(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 12.1) - set(INTERNAL_CXX11_STANDARD_COMPILE_OPTION "${_std}=c++0x") - set(INTERNAL_CXX11_EXTENSION_COMPILE_OPTION "${_std}=${_ext}0x") - endif() - elseif(CMAKE_CXX_COMPILER_ID STREQUAL Cray) - # CMAKE doesn't support CXX_STANDARD or C++ compile features for the Cray - # compiler. Set compiler options transitively here such that they trickle - # down to a call to target_compile_options(). - set(INTERNAL_CXX11_STANDARD_COMPILE_OPTION "-hstd=c++11") - set(INTERNAL_CXX11_EXTENSION_COMPILE_OPTION "-hstd=c++11") - set(INTERNAL_CXX14_STANDARD_COMPILE_OPTION "-hstd=c++11") - set(INTERNAL_CXX14_EXTENSION_COMPILE_OPTION "-hstd=c++11") - elseif(CMAKE_CXX_COMPILER_ID STREQUAL PGI) - # CMAKE doesn't support CXX_STANDARD or C++ compile features for the PGI - # compiler. Set compiler options transitively here such that they trickle - # down to a call to target_compile_options(). - set(INTERNAL_CXX11_STANDARD_COMPILE_OPTION "--c++11") - set(INTERNAL_CXX11_EXTENSION_COMPILE_OPTION "--c++11") - set(INTERNAL_CXX14_STANDARD_COMPILE_OPTION "--c++11") - set(INTERNAL_CXX14_EXTENSION_COMPILE_OPTION "--c++11") - elseif(CMAKE_CXX_COMPILER_ID STREQUAL XL) - # CMAKE doesn't support CXX_STANDARD or C++ compile features for the XL - # compiler. Set compiler options transitively here such that they trickle - # down to a call to target_compile_options(). - set(INTERNAL_CXX11_STANDARD_COMPILE_OPTION "-std=c++11") - set(INTERNAL_CXX11_EXTENSION_COMPILE_OPTION "-std=c++11") - set(INTERNAL_CXX14_STANDARD_COMPILE_OPTION "-std=c++11") - set(INTERNAL_CXX14_EXTENSION_COMPILE_OPTION "-std=c++11") - else() - # Assume GNU. CMAKE_CXX_STANDARD is handled correctly by CMake 3.1 and - # above for this compiler. If the user explicitly requests a C++ - # standard, CMake takes care of it. If not, transitively require C++11. - if(NOT CMAKE_CXX_STANDARD) - set(INTERNAL_CXX11_STANDARD_COMPILE_OPTION ${CMAKE_CXX11_STANDARD_COMPILE_OPTION}) - set(INTERNAL_CXX11_EXTENSION_COMPILE_OPTION ${CMAKE_CXX11_EXTENSION_COMPILE_OPTION}) - endif() - endif() - - # Set the C++ standard info for Kokkos respecting user set values for - # CMAKE_CXX_STANDARD and CMAKE_CXX_EXTENSIONS. - # Only use cxx extension if explicitly requested - if(CMAKE_CXX_STANDARD EQUAL 14) - if(DEFINED CMAKE_CXX_EXTENSIONS AND CMAKE_CXX_EXTENSIONS STREQUAL ON) - set(INTERNAL_CXX_FLAGS ${INTERNAL_CXX14_EXTENSION_COMPILE_OPTION}) - else() - set(INTERNAL_CXX_FLAGS ${INTERNAL_CXX14_STANDARD_COMPILE_OPTION}) - endif() - elseif(CMAKE_CXX_STANDARD EQUAL 11) - if(DEFINED CMAKE_CXX_EXTENSIONS AND CMAKE_CXX_EXTENSIONS STREQUAL ON) - set(INTERNAL_CXX_FLAGS ${INTERNAL_CXX11_EXTENSION_COMPILE_OPTION}) - else() - set(INTERNAL_CXX_FLAGS ${INTERNAL_CXX11_STANDARD_COMPILE_OPTION}) - endif() - else() - # The user didn't explicitly request a standard, transitively require - # C++11 respecting CMAKE_CXX_EXTENSIONS. - if(DEFINED CMAKE_CXX_EXTENSIONS AND CMAKE_CXX_EXTENSIONS STREQUAL ON) - set(INTERNAL_CXX_FLAGS ${INTERNAL_CXX11_EXTENSION_COMPILE_OPTION}) - else() - set(INTERNAL_CXX_FLAGS ${INTERNAL_CXX11_STANDARD_COMPILE_OPTION}) - endif() - endif() - - set(KOKKOS_CXX_FLAGS ${INTERNAL_CXX_FLAGS} PARENT_SCOPE) - endif() -endfunction() - - -#------------------------------------------------------------------------------- -# function(set_kokkos_sources) -# Takes a list of sources for kokkos (e.g., KOKKOS_SRC from Makefile.kokkos and -# put it into kokkos_generated_settings.cmake) and sorts the files into the subpackages or -# separate_libraries. This is core and containers (algorithms is pure header -# files). -# -# Inputs: -# KOKKOS_SRC +# kokkos_option + +# Validate options are given with correct case and define an internal +# upper-case version for use within + # -# Outputs: -# KOKKOS_CORE_SRCS -# KOKKOS_CONTAINERS_SRCS -# -function(set_kokkos_srcs) - set(opts ) # no-value args - set(oneValArgs ) - set(multValArgs KOKKOS_SRC) # e.g., lists - cmake_parse_arguments(IN "${opts}" "${oneValArgs}" "${multValArgs}" ${ARGN}) - - foreach(sfile ${IN_KOKKOS_SRC}) - string(REPLACE "${CMAKE_CURRENT_SOURCE_DIR}/" "" stripfile "${sfile}") - string(REPLACE "/" ";" striplist "${stripfile}") - list(GET striplist 0 firstdir) - if(${firstdir} STREQUAL "core") - list(APPEND KOKKOS_CORE_SRCS ${sfile}) - else() - list(APPEND KOKKOS_CONTAINERS_SRCS ${sfile}) - endif() - endforeach() - set(KOKKOS_CORE_SRCS ${KOKKOS_CORE_SRCS} PARENT_SCOPE) - set(KOKKOS_CONTAINERS_SRCS ${KOKKOS_CONTAINERS_SRCS} PARENT_SCOPE) - return() -endfunction() - -# Setting a default value if it is not already set -macro(set_kokkos_default_default VARIABLE DEFAULT) - IF( "${KOKKOS_INTERNAL_ENABLE_${VARIABLE}_DEFAULT}" STREQUAL "" ) - IF( "${KOKKOS_ENABLE_${VARIABLE}}" STREQUAL "" ) - set(KOKKOS_INTERNAL_ENABLE_${VARIABLE}_DEFAULT ${DEFAULT}) - # MESSAGE(WARNING "Set: KOKKOS_INTERNAL_ENABLE_${VARIABLE}_DEFAULT to ${KOKKOS_INTERNAL_ENABLE_${VARIABLE}_DEFAULT}") +# +# @FUNCTION: kokkos_deprecated_list +# +# Function that checks if a deprecated list option like Kokkos_ARCH was given. +# This prints an error and prevents configure from completing. +# It attempts to print a helpful message about updating the options for the new CMake. +# Kokkos_${SUFFIX} is the name of the option (like Kokkos_ARCH) being checked. +# Kokkos_${PREFIX}_X is the name of new option to be defined from a list X,Y,Z,... +FUNCTION(kokkos_deprecated_list SUFFIX PREFIX) + SET(CAMEL_NAME Kokkos_${SUFFIX}) + STRING(TOUPPER ${CAMEL_NAME} UC_NAME) + + #I don't love doing it this way but better to be safe + FOREACH(opt ${KOKKOS_GIVEN_VARIABLES}) + STRING(TOUPPER ${opt} OPT_UC) + IF ("${OPT_UC}" STREQUAL "${UC_NAME}") + STRING(REPLACE "," ";" optlist "${${opt}}") + SET(ERROR_MSG "Given deprecated option list ${opt}. This must now be given as separate -D options, which assuming you spelled options correctly would be:") + FOREACH(entry ${optlist}) + STRING(TOUPPER ${entry} ENTRY_UC) + STRING(APPEND ERROR_MSG "\n -DKokkos_${PREFIX}_${ENTRY_UC}=ON") + ENDFOREACH() + STRING(APPEND ERROR_MSG "\nRemove CMakeCache.txt and re-run. For a list of valid options, refer to BUILD.md or even look at CMakeCache.txt (before deleting it).") + IF (KOKKOS_HAS_TRILINOS) + MESSAGE(WARNING ${ERROR_MSG}) + FOREACH(entry ${optlist}) + STRING(TOUPPER ${entry} ENTRY_UC) + SET(${CAMEL_NAME}_${ENTRY_UC} ON CACHE BOOL "Deprecated Trilinos translation") + ENDFOREACH() + UNSET(${opt} CACHE) + ELSE() + MESSAGE(SEND_ERROR ${ERROR_MSG}) + ENDIF() + ENDIF() + ENDFOREACH() +ENDFUNCTION() + +FUNCTION(kokkos_option CAMEL_SUFFIX DEFAULT TYPE DOCSTRING) + SET(CAMEL_NAME Kokkos_${CAMEL_SUFFIX}) + STRING(TOUPPER ${CAMEL_NAME} UC_NAME) + + # Make sure this appears in the cache with the appropriate DOCSTRING + SET(${CAMEL_NAME} ${DEFAULT} CACHE ${TYPE} ${DOCSTRING}) + + #I don't love doing it this way because it's N^2 in number options, but cest la vie + FOREACH(opt ${KOKKOS_GIVEN_VARIABLES}) + STRING(TOUPPER ${opt} OPT_UC) + IF ("${OPT_UC}" STREQUAL "${UC_NAME}") + IF (NOT "${opt}" STREQUAL "${CAMEL_NAME}") + IF (KOKKOS_HAS_TRILINOS) + #Allow this for now if Trilinos... we need to bootstrap our way to integration + MESSAGE(WARNING "Deprecated option ${opt} found - please change spelling to ${CAMEL_NAME}") + SET(${CAMEL_NAME} "${${opt}}" CACHE ${TYPE} ${DOCSTRING} FORCE) + UNSET(${opt} CACHE) + ELSE() + MESSAGE(FATAL_ERROR "Matching option found for ${CAMEL_NAME} with the wrong case ${opt}. Please delete your CMakeCache.txt and change option to -D${CAMEL_NAME}=${${opt}}. This is now enforced to avoid hard-to-debug CMake cache inconsistencies.") + ENDIF() + ENDIF() + ENDIF() + ENDFOREACH() + + #okay, great, we passed the validation test - use the default + IF (DEFINED ${CAMEL_NAME}) + SET(${UC_NAME} ${${CAMEL_NAME}} PARENT_SCOPE) + ELSE() + SET(${UC_NAME} ${DEFAULT} PARENT_SCOPE) + ENDIF() + +ENDFUNCTION() + +FUNCTION(kokkos_append_config_line LINE) + GLOBAL_APPEND(KOKKOS_TPL_EXPORTS "${LINE}") +ENDFUNCTION() + +MACRO(kokkos_export_cmake_tpl NAME) + #CMake TPLs are located with a call to find_package + #find_package locates XConfig.cmake files through + #X_DIR or X_ROOT variables set prior to calling find_package + + #If Kokkos was configured to find the TPL through a _DIR variable + #make sure thar DIR variable is available to downstream packages + IF (DEFINED ${NAME}_DIR) + #The downstream project may override the TPL location that Kokkos used + #Check if the downstream project chose its own TPL location + #If not, make the Kokkos found location available + KOKKOS_APPEND_CONFIG_LINE("IF(NOT DEFINED ${NAME}_DIR)") + KOKKOS_APPEND_CONFIG_LINE(" SET(${NAME}_DIR ${${NAME}_DIR})") + KOKKOS_APPEND_CONFIG_LINE("ENDIF()") + ENDIF() + + IF (DEFINED ${NAME}_ROOT) + #The downstream project may override the TPL location that Kokkos used + #Check if the downstream project chose its own TPL location + #If not, make the Kokkos found location available + KOKKOS_APPEND_CONFIG_LINE("IF(NOT DEFINED ${NAME}_ROOT)") + KOKKOS_APPEND_CONFIG_LINE(" SET(${NAME}_ROOT ${${NAME}_ROOT})") + KOKKOS_APPEND_CONFIG_LINE("ENDIF()") + ENDIF() + KOKKOS_APPEND_CONFIG_LINE("FIND_DEPENDENCY(${NAME})") +ENDMACRO() + +MACRO(kokkos_export_imported_tpl NAME) + IF (NOT KOKKOS_HAS_TRILINOS) + GET_TARGET_PROPERTY(LIB_TYPE ${NAME} TYPE) + IF (${LIB_TYPE} STREQUAL "INTERFACE_LIBRARY") + # This is not an imported target + # This an interface library that we created + INSTALL( + TARGETS ${NAME} + EXPORT KokkosTargets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + ) ELSE() - set(KOKKOS_INTERNAL_ENABLE_${VARIABLE}_DEFAULT ${KOKKOS_ENABLE_${VARIABLE}}) - # MESSAGE(WARNING "Set: KOKKOS_INTERNAL_ENABLE_${VARIABLE}_DEFAULT to ${KOKKOS_INTERNAL_ENABLE_${VARIABLE}_DEFAULT}") + #make sure this also gets "exported" in the config file + KOKKOS_APPEND_CONFIG_LINE("IF(NOT TARGET ${NAME})") + KOKKOS_APPEND_CONFIG_LINE("ADD_LIBRARY(${NAME} UNKNOWN IMPORTED)") + KOKKOS_APPEND_CONFIG_LINE("SET_TARGET_PROPERTIES(${NAME} PROPERTIES") + + GET_TARGET_PROPERTY(TPL_LIBRARY ${NAME} IMPORTED_LOCATION) + IF(TPL_LIBRARY) + KOKKOS_APPEND_CONFIG_LINE("IMPORTED_LOCATION ${TPL_LIBRARY}") + ENDIF() + + GET_TARGET_PROPERTY(TPL_INCLUDES ${NAME} INTERFACE_INCLUDE_DIRECTORIES) + IF(TPL_INCLUDES) + KOKKOS_APPEND_CONFIG_LINE("INTERFACE_INCLUDE_DIRECTORIES ${TPL_INCLUDES}") + ENDIF() + + GET_TARGET_PROPERTY(TPL_COMPILE_OPTIONS ${NAME} INTERFACE_COMPILE_OPTIONS) + IF(TPL_COMPILE_OPTIONS) + KOKKOS_APPEND_CONFIG_LINE("INTERFACE_COMPILE_OPTIONS ${TPL_COMPILE_OPTIONS}") + ENDIF() + + SET(TPL_LINK_OPTIONS) + IF(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.13.0") + GET_TARGET_PROPERTY(TPL_LINK_OPTIONS ${NAME} INTERFACE_LINK_OPTIONS) + ENDIF() + IF(TPL_LINK_OPTIONS) + KOKKOS_APPEND_CONFIG_LINE("INTERFACE_LINK_OPTIONS ${TPL_LINK_OPTIONS}") + ENDIF() + + GET_TARGET_PROPERTY(TPL_LINK_LIBRARIES ${NAME} INTERFACE_LINK_LIBRARIES) + IF(TPL_LINK_LIBRARIES) + KOKKOS_APPEND_CONFIG_LINE("INTERFACE_LINK_LIBRARIES ${TPL_LINK_LIBRARIES}") + ENDIF() + KOKKOS_APPEND_CONFIG_LINE(")") + KOKKOS_APPEND_CONFIG_LINE("ENDIF()") ENDIF() ENDIF() - UNSET(KOKKOS_ENABLE_${VARIABLE} CACHE) -endmacro() +ENDMACRO() + + +# +# @MACRO: KOKKOS_IMPORT_TPL() +# +# Function that checks if a third-party library (TPL) has been enabled and calls `find_package` +# to create an imported target encapsulating all the flags and libraries +# needed to use the TPL +# +# Usage:: +# +# KOKKOS_IMPORT_TPL( +# +# NO_EXPORT +# INTERFACE +# +# ``NO_EXPORT`` +# +# If specified, this TPL will not be added to KokkosConfig.cmake as an export +# +# ``INTERFACE`` +# +# If specified, this TPL will build an INTERFACE library rather than an +# IMPORTED target +MACRO(kokkos_import_tpl NAME) + CMAKE_PARSE_ARGUMENTS(TPL + "NO_EXPORT;INTERFACE" + "" + "" + ${ARGN}) + IF (TPL_INTERFACE) + SET(TPL_IMPORTED_NAME ${NAME}) + ELSE() + SET(TPL_IMPORTED_NAME Kokkos::${NAME}) + ENDIF() + + # Even though this policy gets set in the top-level CMakeLists.txt, + # I have still been getting errors about ROOT variables being ignored + # I'm not sure if this is a scope issue - but make sure + # the policy is set before we do any find_package calls + IF(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.12.0") + CMAKE_POLICY(SET CMP0074 NEW) + ENDIF() + + IF (KOKKOS_ENABLE_${NAME}) + #Tack on a TPL here to make sure we avoid using anyone else's find + FIND_PACKAGE(TPL${NAME} REQUIRED MODULE) + IF(NOT TARGET ${TPL_IMPORTED_NAME}) + MESSAGE(FATAL_ERROR "Find module succeeded for ${NAME}, but did not produce valid target ${TPL_IMPORTED_NAME}") + ENDIF() + IF(NOT TPL_NO_EXPORT) + KOKKOS_EXPORT_IMPORTED_TPL(${TPL_IMPORTED_NAME}) + ENDIF() + LIST(APPEND KOKKOS_ENABLED_TPLS ${NAME}) + ENDIF() +ENDMACRO(kokkos_import_tpl) + +MACRO(kokkos_import_cmake_tpl MODULE_NAME) + kokkos_import_tpl(${MODULE_NAME} ${ARGN} NO_EXPORT) + CMAKE_PARSE_ARGUMENTS(TPL + "NO_EXPORT" + "OPTION_NAME" + "" + ${ARGN}) + + IF (NOT TPL_OPTION_NAME) + SET(TPL_OPTION_NAME ${MODULE_NAME}) + ENDIF() + + IF (NOT TPL_NO_EXPORT) + KOKKOS_EXPORT_CMAKE_TPL(${MODULE_NAME}) + ENDIF() +ENDMACRO() + +# +# @MACRO: KOKKOS_CREATE_IMPORTED_TPL() +# +# Function that creates an imported target encapsulating all the flags +# and libraries needed to use the TPL +# +# Usage:: +# +# KOKKOS_CREATE_IMPORTED_TPL( +# +# INTERFACE +# LIBRARY +# LINK_LIBRARIES ... +# COMPILE_OPTIONS ... +# LINK_OPTIONS ... +# +# ``INTERFACE`` +# +# If specified, this TPL will build an INTERFACE library rather than an +# IMPORTED target +# +# ``LIBRARY `` +# +# If specified, this gives the IMPORTED_LOCATION of the library. +# +# ``LINK_LIBRARIES ...`` +# +# If specified, this gives a list of dependent libraries that also +# need to be linked against. Each entry can be a library path or +# the name of a valid CMake target. +# +# ``INCLUDES ...`` +# +# If specified, this gives a list of directories that must be added +# to the include path for using this library. +# +# ``COMPILE_OPTIONS ...`` +# +# If specified, this gives a list of compiler flags that must be used +# for using this library. +# +# ``LINK_OPTIONS ...`` +# +# If specified, this gives a list of linker flags that must be used +# for using this library. +MACRO(kokkos_create_imported_tpl NAME) + CMAKE_PARSE_ARGUMENTS(TPL + "INTERFACE" + "LIBRARY" + "LINK_LIBRARIES;INCLUDES;COMPILE_OPTIONS;LINK_OPTIONS" + ${ARGN}) + + + IF (KOKKOS_HAS_TRILINOS) + #TODO: we need to set a bunch of cache variables here + ELSEIF (TPL_INTERFACE) + ADD_LIBRARY(${NAME} INTERFACE) + #Give this an importy-looking name + ADD_LIBRARY(Kokkos::${NAME} ALIAS ${NAME}) + IF (TPL_LIBRARY) + MESSAGE(SEND_ERROR "TPL Interface library ${NAME} should not have an IMPORTED_LOCATION") + ENDIF() + #Things have to go in quoted in case we have multiple list entries + IF(TPL_LINK_LIBRARIES) + TARGET_LINK_LIBRARIES(${NAME} INTERFACE ${TPL_LINK_LIBRARIES}) + ENDIF() + IF(TPL_INCLUDES) + TARGET_INCLUDE_DIRECTORIES(${NAME} INTERFACE ${TPL_INCLUDES}) + ENDIF() + IF(TPL_COMPILE_OPTIONS) + TARGET_COMPILE_OPTIONS(${NAME} INTERFACE ${TPL_COMPILE_OPTIONS}) + ENDIF() + IF(TPL_LINK_OPTIONS) + TARGET_LINK_LIBRARIES(${NAME} INTERFACE ${TPL_LINK_OPTIONS}) + ENDIF() + ELSE() + ADD_LIBRARY(${NAME} UNKNOWN IMPORTED) + IF(TPL_LIBRARY) + SET_TARGET_PROPERTIES(${NAME} PROPERTIES + IMPORTED_LOCATION ${TPL_LIBRARY}) + ENDIF() + #Things have to go in quoted in case we have multiple list entries + IF(TPL_LINK_LIBRARIES) + SET_TARGET_PROPERTIES(${NAME} PROPERTIES + INTERFACE_LINK_LIBRARIES "${TPL_LINK_LIBRARIES}") + ENDIF() + IF(TPL_INCLUDES) + SET_TARGET_PROPERTIES(${NAME} PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${TPL_INCLUDES}") + ENDIF() + IF(TPL_COMPILE_OPTIONS) + SET_TARGET_PROPERTIES(${NAME} PROPERTIES + INTERFACE_COMPILE_OPTIONS "${TPL_COMPILE_OPTIONS}") + ENDIF() + IF(TPL_LINK_OPTIONS) + SET_TARGET_PROPERTIES(${NAME} PROPERTIES + INTERFACE_LINK_LIBRARIES "${TPL_LINK_OPTIONS}") + ENDIF() + ENDIF() +ENDMACRO() + +# +# @MACRO: KOKKOS_FIND_HEADER +# +# Function that finds a particular header. This searches custom paths +# or default system paths depending on options. In constrast to CMake +# default, custom paths are prioritized over system paths. The searched +# order is: +# 1. _ROOT variable +# 2. Kokkos__DIR variable +# 3. Locations in the PATHS option +# 4. Default system paths, if allowed. +# +# Default system paths are allowed if none of options (1)-(3) are specified +# or if default paths are specifically allowed via ALLOW_SYSTEM_PATH_FALLBACK +# +# Usage:: +# +# KOKKOS_FIND_HEADER( +# +#
+# +# [ALLOW_SYSTEM_PATH_FALLBACK] +# [PATHS path1 [path2 ...]] +# ) +# +# ```` +# +# The variable to define with the success or failure of the find +# +# ``
`` +# +# The name of the header to find +# +# ```` +# +# The name of the TPL the header corresponds to +# +# ``[ALLOW_SYSTEM_PATH_FALLBACK]`` +# +# If custom paths are given and the header is not found +# should we be allowed to search default system paths +# or error out if not found in given paths +# +# ``[PATHS path1 [path2 ...]]`` +# +# Custom paths to search for the header +# +MACRO(kokkos_find_header VAR_NAME HEADER TPL_NAME) + CMAKE_PARSE_ARGUMENTS(TPL + "ALLOW_SYSTEM_PATH_FALLBACK" + "" + "PATHS" + ${ARGN}) + + SET(${HEADER}_FOUND FALSE) + SET(HAVE_CUSTOM_PATHS FALSE) + IF(NOT ${HEADER}_FOUND AND DEFINED ${TPL_NAME}_ROOT) + #ONLY look in the root directory + FIND_PATH(${VAR_NAME} ${HEADER} PATHS ${${TPL_NAME}_ROOT}/include NO_DEFAULT_PATH) + SET(HAVE_CUSTOM_PATHS TRUE) + ENDIF() + + IF(NOT ${HEADER}_FOUND AND DEFINED KOKKOS_${TPL_NAME}_DIR) + #ONLY look in the root directory + FIND_PATH(${VAR_NAME} ${HEADER} PATHS ${KOKKOS_${TPL_NAME}_DIR}/include NO_DEFAULT_PATH) + SET(HAVE_CUSTOM_PATHS TRUE) + ENDIF() + + IF (NOT ${HEADER}_FOUND AND TPL_PATHS) + #we got custom paths + #ONLY look in these paths and nowhere else + FIND_PATH(${VAR_NAME} ${HEADER} PATHS ${TPL_PATHS} NO_DEFAULT_PATH) + SET(HAVE_CUSTOM_PATHS TRUE) + ENDIF() + + IF (NOT HAVE_CUSTOM_PATHS OR TPL_ALLOW_SYSTEM_PATH_FALLBACK) + #Now go ahead and look in system paths + IF (NOT ${HEADER}_FOUND) + FIND_PATH(${VAR_NAME} ${HEADER}) + ENDIF() + ENDIF() +ENDMACRO() + +# +# @MACRO: KOKKOS_FIND_LIBRARY +# +# Function that find a particular library. This searches custom paths +# or default system paths depending on options. In constrast to CMake +# default, custom paths are prioritized over system paths. The search +# order is: +# 1. _ROOT variable +# 2. Kokkos__DIR variable +# 3. Locations in the PATHS option +# 4. Default system paths, if allowed. +# +# Default system paths are allowed if none of options (1)-(3) are specified +# or if default paths are specifically allowed via ALLOW_SYSTEM_PATH_FALLBACK +# +# Usage:: +# +# KOKKOS_FIND_LIBRARY( +# +#
+# +# [ALLOW_SYSTEM_PATH_FALLBACK] +# [PATHS path1 [path2 ...]] +# ) +# +# ```` +# +# The variable to define with the success or failure of the find +# +# ```` +# +# The name of the library to find (NOT prefixed with -l) +# +# ```` +# +# The name of the TPL the library corresponds to +# +# ``ALLOW_SYSTEM_PATH_FALLBACK`` +# +# If custom paths are given and the library is not found +# should we be allowed to search default system paths +# or error out if not found in given paths +# +# ``PATHS`` +# +# Custom paths to search for the library +# +MACRO(kokkos_find_library VAR_NAME LIB TPL_NAME) + CMAKE_PARSE_ARGUMENTS(TPL + "ALLOW_SYSTEM_PATH_FALLBACK" + "" + "PATHS" + ${ARGN}) + + SET(${LIB}_FOUND FALSE) + SET(HAVE_CUSTOM_PATHS FALSE) + IF(NOT ${LIB}_FOUND AND DEFINED ${TPL_NAME}_ROOT) + FIND_LIBRARY(${VAR_NAME} ${LIB} PATHS ${${TPL_NAME}_ROOT}/lib ${${TPL_NAME}_ROOT}/lib64 NO_DEFAULT_PATH) + SET(HAVE_CUSTOM_PATHS TRUE) + ENDIF() + + IF(NOT ${LIB}_FOUND AND DEFINED KOKKOS_${TPL_NAME}_DIR) + #we got root paths, only look in these paths and nowhere else + FIND_LIBRARY(${VAR_NAME} ${LIB} PATHS ${KOKKOS_${TPL_NAME}_DIR}/lib ${KOKKOS_${TPL_NAME}_DIR}/lib64 NO_DEFAULT_PATH) + SET(HAVE_CUSTOM_PATHS TRUE) + ENDIF() + + IF (NOT ${LIB}_FOUND AND TPL_PATHS) + #we got custom paths, only look in these paths and nowhere else + FIND_LIBRARY(${VAR_NAME} ${LIB} PATHS ${TPL_PATHS} NO_DEFAULT_PATH) + SET(HAVE_CUSTOM_PATHS TRUE) + ENDIF() + + + IF (NOT HAVE_CUSTOM_PATHS OR TPL_ALLOW_SYSTEM_PATH_FALLBACK) + IF (NOT ${LIB}_FOUND) + #Now go ahead and look in system paths + FIND_LIBRARY(${VAR_NAME} ${LIB}) + ENDIF() + ENDIF() +ENDMACRO() + +# +# @MACRO: KOKKOS_FIND_IMPORTED +# +# Function that finds all libraries and headers needed for the tpl +# and creates an imported target encapsulating all the flags and libraries +# +# Usage:: +# +# KOKKOS_FIND_IMPORTED( +# +# INTERFACE +# ALLOW_SYSTEM_PATH_FALLBACK +# LIBRARY +# LINK_LIBRARIES ... +# COMPILE_OPTIONS ... +# LINK_OPTIONS ... +# +# ``INTERFACE`` +# +# If specified, this TPL will build an INTERFACE library rather than an +# IMPORTED target +# +# ``ALLOW_SYSTEM_PATH_FALLBACK" +# +# If custom paths are given and the library is not found +# should we be allowed to search default system paths +# or error out if not found in given paths. +# +# ``LIBRARY `` +# +# If specified, this gives the name of the library to look for +# +# ``MODULE_NAME `` +# +# If specified, the name of the enclosing module passed to +# FIND_PACKAGE(). Defaults to TPL${NAME} if not +# given. +# +# ``IMPORTED_NAME `` +# +# If specified, this gives the name of the target to build. +# Defaults to Kokkos:: +# +# ``LIBRARY_PATHS ...`` +# +# If specified, this gives a list of paths to search for the library +# If not given, _ROOT/lib and _ROOT/lib64 will be searched. +# +# ``HEADER_PATHS ...`` +# +# If specified, this gives a list of paths to search for the headers +# If not given, _ROOT/include and _ROOT/include will be searched. +# +# ``HEADERS ...`` +# +# If specified, this gives a list of headers to find for the package +# +# ``LIBRARIES ...`` +# +# If specified, this gives a list of libraries to find for the package +# +MACRO(kokkos_find_imported NAME) + CMAKE_PARSE_ARGUMENTS(TPL + "INTERFACE;ALLOW_SYSTEM_PATH_FALLBACK" + "HEADER;LIBRARY;IMPORTED_NAME;MODULE_NAME" + "HEADER_PATHS;LIBRARY_PATHS;HEADERS;LIBRARIES" + ${ARGN}) + + IF(NOT TPL_MODULE_NAME) + SET(TPL_MODULE_NAME TPL${NAME}) + ENDIF() + + IF (TPL_ALLOW_SYSTEM_PATH_FALLBACK) + SET(ALLOW_PATH_FALLBACK_OPT ALLOW_SYSTEM_PATH_FALLBACK) + ELSE() + SET(ALLOW_PATH_FALLBACK_OPT) + ENDIF() + + IF (NOT TPL_IMPORTED_NAME) + IF (TPL_INTERFACE) + SET(TPL_IMPORTED_NAME ${NAME}) + ELSE() + SET(TPL_IMPORTED_NAME Kokkos::${NAME}) + ENDIF() + ENDIF() + + SET(${NAME}_INCLUDE_DIRS) + IF (TPL_HEADER) + KOKKOS_FIND_HEADER(${NAME}_INCLUDE_DIRS ${TPL_HEADER} ${NAME} ${ALLOW_PATH_FALLBACK_OPT} PATHS ${TPL_HEADER_PATHS}) + ENDIF() + + FOREACH(HEADER ${TPL_HEADERS}) + KOKKOS_FIND_HEADER(HEADER_FIND_TEMP ${HEADER} ${NAME} ${ALLOW_PATH_FALLBACK_OPT} PATHS ${TPL_HEADER_PATHS}) + IF(HEADER_FIND_TEMP) + LIST(APPEND ${NAME}_INCLUDE_DIRS ${HEADER_FIND_TEMP}) + ENDIF() + ENDFOREACH() + + SET(${NAME}_LIBRARY) + IF(TPL_LIBRARY) + KOKKOS_FIND_LIBRARY(${NAME}_LIBRARY ${TPL_LIBRARY} ${NAME} ${ALLOW_PATH_FALLBACK_OPT} PATHS ${TPL_LIBRARY_PATHS}) + ENDIF() + + SET(${NAME}_FOUND_LIBRARIES) + FOREACH(LIB ${TPL_LIBRARIES}) + KOKKOS_FIND_LIBRARY(${LIB}_LOCATION ${LIB} ${NAME} ${ALLOW_PATH_FALLBACK_OPT} PATHS ${TPL_LIBRARY_PATHS}) + IF(${LIB}_LOCATION) + LIST(APPEND ${NAME}_FOUND_LIBRARIES ${${LIB}_LOCATION}) + ELSE() + SET(${NAME}_FOUND_LIBRARIES ${${LIB}_LOCATION}) + BREAK() + ENDIF() + ENDFOREACH() + + INCLUDE(FindPackageHandleStandardArgs) + #Collect all the variables we need to be valid for + #find_package to have succeeded + SET(TPL_VARS_NEEDED) + IF (TPL_LIBRARY) + LIST(APPEND TPL_VARS_NEEDED ${NAME}_LIBRARY) + ENDIF() + IF(TPL_HEADER) + LIST(APPEND TPL_VARS_NEEDED ${NAME}_INCLUDE_DIRS) + ENDIF() + IF(TPL_LIBRARIES) + LIST(APPEND TPL_VARS_NEEDED ${NAME}_FOUND_LIBRARIES) + ENDIF() + FIND_PACKAGE_HANDLE_STANDARD_ARGS(${TPL_MODULE_NAME} REQUIRED_VARS ${TPL_VARS_NEEDED}) + + MARK_AS_ADVANCED(${NAME}_INCLUDE_DIRS ${NAME}_FOUND_LIBRARIES ${NAME}_LIBRARY) + + IF (${TPL_MODULE_NAME}_FOUND) + SET(IMPORT_TYPE) + IF (TPL_INTERFACE) + SET(IMPORT_TYPE "INTERFACE") + ENDIF() + KOKKOS_CREATE_IMPORTED_TPL(${TPL_IMPORTED_NAME} + ${IMPORT_TYPE} + INCLUDES "${${NAME}_INCLUDE_DIRS}" + LIBRARY "${${NAME}_LIBRARY}" + LINK_LIBRARIES "${${NAME}_FOUND_LIBRARIES}") + ENDIF() +ENDMACRO(kokkos_find_imported) + +# +# @MACRO: KOKKOS_LINK_TPL() +# +# Function that checks if a third-party library (TPL) has been enabled and +# calls target_link_libraries on the given target +# +# Usage:: +# +# KOKKOS_LINK_TPL( +# +# PUBLIC +# PRIVATE +# INTERFACE +# IMPORTED_NAME +# +# +# Checks if Kokkos_ENABLE_=ON and if so links the library +# +# ``PUBLIC/PRIVATE/INTERFACE`` +# +# Specifies the linkage mode. One of these arguments should be given. +# This will then invoke target_link_libraries( PUBLIC/PRIVATE/INTERFACE ) +# +# ``IMPORTED_NAME `` +# +# If specified, this gives the exact name of the target to link against +# target_link_libraries( ) +# +FUNCTION(kokkos_link_tpl TARGET) + CMAKE_PARSE_ARGUMENTS(TPL + "PUBLIC;PRIVATE;INTERFACE" + "IMPORTED_NAME" + "" + ${ARGN}) + #the name of the TPL + SET(TPL ${TPL_UNPARSED_ARGUMENTS}) + IF (KOKKOS_HAS_TRILINOS) + #Do nothing, they will have already been linked + ELSE() + IF (NOT TPL_IMPORTED_NAME) + SET(TPL_IMPORTED_NAME Kokkos::${TPL}) + ENDIF() + IF (KOKKOS_ENABLE_${TPL}) + IF (TPL_PUBLIC) + TARGET_LINK_LIBRARIES(${TARGET} PUBLIC ${TPL_IMPORTED_NAME}) + ELSEIF (TPL_PRIVATE) + TARGET_LINK_LIBRARIES(${TARGET} PRIVATE ${TPL_IMPORTED_NAME}) + ELSEIF (TPL_INTERFACE) + TARGET_LINK_LIBRARIES(${TARGET} INTERFACE ${TPL_IMPORTED_NAME}) + ELSE() + TARGET_LINK_LIBRARIES(${TARGET} ${TPL_IMPORTED_NAME}) + ENDIF() + ENDIF() + ENDIF() +ENDFUNCTION() + diff --git a/lib/kokkos/cmake/kokkos_install.cmake b/lib/kokkos/cmake/kokkos_install.cmake new file mode 100644 index 0000000000000000000000000000000000000000..1e4a5a2aad7e2dfd1eb7fd7d804fd175fef9d564 --- /dev/null +++ b/lib/kokkos/cmake/kokkos_install.cmake @@ -0,0 +1,42 @@ +IF (NOT KOKKOS_HAS_TRILINOS) + INCLUDE(GNUInstallDirs) + + #Set all the variables needed for KokkosConfig.cmake + GET_PROPERTY(KOKKOS_PROP_LIBS GLOBAL PROPERTY KOKKOS_LIBRARIES_NAMES) + SET(KOKKOS_LIBRARIES ${KOKKOS_PROP_LIBS}) + + INCLUDE(CMakePackageConfigHelpers) + CONFIGURE_PACKAGE_CONFIG_FILE( + cmake/KokkosConfig.cmake.in + "${Kokkos_BINARY_DIR}/KokkosConfig.cmake" + INSTALL_DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/cmake) + + INCLUDE(CMakePackageConfigHelpers) + CONFIGURE_PACKAGE_CONFIG_FILE( + cmake/KokkosConfigCommon.cmake.in + "${Kokkos_BINARY_DIR}/KokkosConfigCommon.cmake" + INSTALL_DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/cmake) + + WRITE_BASIC_PACKAGE_VERSION_FILE("${Kokkos_BINARY_DIR}/KokkosConfigVersion.cmake" + VERSION "${Kokkos_VERSION}" + COMPATIBILITY SameMajorVersion) + + # Install the KokkosConfig*.cmake files + install(FILES + "${Kokkos_BINARY_DIR}/KokkosConfig.cmake" + "${Kokkos_BINARY_DIR}/KokkosConfigCommon.cmake" + "${Kokkos_BINARY_DIR}/KokkosConfigVersion.cmake" + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Kokkos) + install(EXPORT KokkosTargets NAMESPACE Kokkos:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Kokkos) +ELSE() + CONFIGURE_FILE(cmake/KokkosConfigCommon.cmake.in ${Kokkos_BINARY_DIR}/KokkosConfigCommon.cmake @ONLY) + file(READ ${Kokkos_BINARY_DIR}/KokkosConfigCommon.cmake KOKKOS_CONFIG_COMMON) + file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/KokkosConfig_install.cmake" ${KOKKOS_CONFIG_COMMON}) +ENDIF() + +# build and install pkgconfig file +CONFIGURE_FILE(core/src/kokkos.pc.in kokkos.pc @ONLY) +INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/kokkos.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) + +INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/KokkosCore_config.h DESTINATION ${KOKKOS_HEADER_DIR}) + diff --git a/lib/kokkos/cmake/kokkos_options.cmake b/lib/kokkos/cmake/kokkos_options.cmake deleted file mode 100644 index 239301925cfebb79a8e60bd504afcf349b58d6ea..0000000000000000000000000000000000000000 --- a/lib/kokkos/cmake/kokkos_options.cmake +++ /dev/null @@ -1,419 +0,0 @@ -########################## NOTES ############################################### -# List the options for configuring kokkos using CMake method of doing it. -# These options then get mapped onto KOKKOS_SETTINGS environment variable by -# kokkos_settings.cmake. It is separate to allow other packages to override -# these variables (e.g., TriBITS). - -########################## AVAILABLE OPTIONS ################################### -# Use lists for documentation, verification, and programming convenience - -# All CMake options of the type KOKKOS_ENABLE_* -set(KOKKOS_INTERNAL_ENABLE_OPTIONS_LIST) -list(APPEND KOKKOS_INTERNAL_ENABLE_OPTIONS_LIST - Serial - OpenMP - Pthread - Qthread - HPX - Cuda - ROCm - HWLOC - MEMKIND - LIBRT - Cuda_Lambda - Cuda_Relocatable_Device_Code - Cuda_UVM - Cuda_LDG_Intrinsic - HPX_ASYNC_DISPATCH - Debug - Debug_DualView_Modify_Check - Debug_Bounds_Check - Compiler_Warnings - Profiling - Profiling_Load_Print - Aggressive_Vectorization - Deprecated_Code - Explicit_Instantiation - ) - -#------------------------------------------------------------------------------- -#------------------------------- Recognize CamelCase Options --------------------------- -#------------------------------------------------------------------------------- - -foreach(opt ${KOKKOS_INTERNAL_ENABLE_OPTIONS_LIST}) - string(TOUPPER ${opt} OPT ) - IF(DEFINED Kokkos_ENABLE_${opt}) - IF(DEFINED KOKKOS_ENABLE_${OPT}) - IF(NOT ("${KOKKOS_ENABLE_${OPT}}" STREQUAL "${Kokkos_ENABLE_${opt}}")) - IF(DEFINED KOKKOS_ENABLE_${OPT}_INTERNAL) - MESSAGE(WARNING "Defined both Kokkos_ENABLE_${opt}=[${Kokkos_ENABLE_${opt}}] and KOKKOS_ENABLE_${OPT}=[${KOKKOS_ENABLE_${OPT}}] and they differ! Could be caused by old CMakeCache Variable. Run CMake again and warning should disappear. If not you are truly setting both variables.") - IF(NOT ("${Kokkos_ENABLE_${opt}}" STREQUAL "${KOKKOS_ENABLE_${OPT}_INTERNAL}")) - UNSET(KOKKOS_ENABLE_${OPT} CACHE) - SET(KOKKOS_ENABLE_${OPT} ${Kokkos_ENABLE_${opt}}) - MESSAGE(WARNING "SET BOTH VARIABLES KOKKOS_ENABLE_${OPT}: ${KOKKOS_ENABLE_${OPT}}") - ELSE() - SET(Kokkos_ENABLE_${opt} ${KOKKOS_ENABLE_${OPT}}) - ENDIF() - ELSE() - MESSAGE(FATAL_ERROR "Defined both Kokkos_ENABLE_${opt}=[${Kokkos_ENABLE_${opt}}] and KOKKOS_ENABLE_${OPT}=[${KOKKOS_ENABLE_${OPT}}] and they differ!") - ENDIF() - ENDIF() - ELSE() - SET(KOKKOS_INTERNAL_ENABLE_${OPT}_DEFAULT ${Kokkos_ENABLE_${opt}}) - ENDIF() - ENDIF() -endforeach() - -IF(DEFINED Kokkos_ARCH) - MESSAGE(FATAL_ERROR "Defined Kokkos_ARCH, use KOKKOS_ARCH instead!") -ENDIF() -IF(DEFINED Kokkos_Arch) - MESSAGE(FATAL_ERROR "Defined Kokkos_Arch, use KOKKOS_ARCH instead!") -ENDIF() - -#------------------------------------------------------------------------------- -# List of possible host architectures. -#------------------------------------------------------------------------------- -set(KOKKOS_ARCH_LIST) -list(APPEND KOKKOS_ARCH_LIST - None # No architecture optimization - AMDAVX # (HOST) AMD chip - EPYC # (HOST) AMD EPYC Zen-Core CPU - ARMv80 # (HOST) ARMv8.0 Compatible CPU - ARMv81 # (HOST) ARMv8.1 Compatible CPU - ARMv8-ThunderX # (HOST) ARMv8 Cavium ThunderX CPU - ARMv8-TX2 # (HOST) ARMv8 Cavium ThunderX2 CPU - WSM # (HOST) Intel Westmere CPU - SNB # (HOST) Intel Sandy/Ivy Bridge CPUs - HSW # (HOST) Intel Haswell CPUs - BDW # (HOST) Intel Broadwell Xeon E-class CPUs - SKX # (HOST) Intel Sky Lake Xeon E-class HPC CPUs (AVX512) - KNC # (HOST) Intel Knights Corner Xeon Phi - KNL # (HOST) Intel Knights Landing Xeon Phi - BGQ # (HOST) IBM Blue Gene Q - Power7 # (HOST) IBM POWER7 CPUs - Power8 # (HOST) IBM POWER8 CPUs - Power9 # (HOST) IBM POWER9 CPUs - Kepler # (GPU) NVIDIA Kepler default (generation CC 3.5) - Kepler30 # (GPU) NVIDIA Kepler generation CC 3.0 - Kepler32 # (GPU) NVIDIA Kepler generation CC 3.2 - Kepler35 # (GPU) NVIDIA Kepler generation CC 3.5 - Kepler37 # (GPU) NVIDIA Kepler generation CC 3.7 - Maxwell # (GPU) NVIDIA Maxwell default (generation CC 5.0) - Maxwell50 # (GPU) NVIDIA Maxwell generation CC 5.0 - Maxwell52 # (GPU) NVIDIA Maxwell generation CC 5.2 - Maxwell53 # (GPU) NVIDIA Maxwell generation CC 5.3 - Pascal60 # (GPU) NVIDIA Pascal generation CC 6.0 - Pascal61 # (GPU) NVIDIA Pascal generation CC 6.1 - Volta70 # (GPU) NVIDIA Volta generation CC 7.0 - Volta72 # (GPU) NVIDIA Volta generation CC 7.2 - Turing75 # (GPU) NVIDIA Turing generation CC 7.5 - ) - -# List of possible device architectures. -# The case and spelling here needs to match Makefile.kokkos -set(KOKKOS_DEVICES_LIST) -# Options: Cuda,ROCm,OpenMP,Pthread,Qthreads,Serial -list(APPEND KOKKOS_DEVICES_LIST - Cuda # NVIDIA GPU -- see below - OpenMP # OpenMP - Pthread # pthread - Qthreads # qthreads - HPX # HPX - Serial # serial - ROCm # Relocatable device code - ) - -# List of possible TPLs for Kokkos -# From Makefile.kokkos: Options: hwloc,librt,experimental_memkind -set(KOKKOS_USE_TPLS_LIST) -if(APPLE) -list(APPEND KOKKOS_USE_TPLS_LIST - HWLOC # hwloc - MEMKIND # experimental_memkind - ) -else() -list(APPEND KOKKOS_USE_TPLS_LIST - HWLOC # hwloc - LIBRT # librt - MEMKIND # experimental_memkind - ) -endif() -# Map of cmake variables to Makefile variables -set(KOKKOS_INTERNAL_HWLOC hwloc) -set(KOKKOS_INTERNAL_LIBRT librt) -set(KOKKOS_INTERNAL_MEMKIND experimental_memkind) - -# List of possible Advanced options -set(KOKKOS_OPTIONS_LIST) -list(APPEND KOKKOS_OPTIONS_LIST - AGGRESSIVE_VECTORIZATION - DISABLE_PROFILING - DISABLE_DUALVIEW_MODIFY_CHECK - ENABLE_PROFILE_LOAD_PRINT - ) -# Map of cmake variables to Makefile variables -set(KOKKOS_INTERNAL_LDG_INTRINSIC use_ldg) -set(KOKKOS_INTERNAL_UVM librt) -set(KOKKOS_INTERNAL_RELOCATABLE_DEVICE_CODE rdc) - - -#------------------------------------------------------------------------------- -# List of possible Options for CUDA -#------------------------------------------------------------------------------- -# From Makefile.kokkos: Options: use_ldg,force_uvm,rdc -set(KOKKOS_CUDA_OPTIONS_LIST) -list(APPEND KOKKOS_CUDA_OPTIONS_LIST - LDG_INTRINSIC # use_ldg - UVM # force_uvm - RELOCATABLE_DEVICE_CODE # rdc - LAMBDA # enable_lambda - ) - -# Map of cmake variables to Makefile variables -set(KOKKOS_INTERNAL_LDG_INTRINSIC use_ldg) -set(KOKKOS_INTERNAL_UVM force_uvm) -set(KOKKOS_INTERNAL_RELOCATABLE_DEVICE_CODE rdc) -set(KOKKOS_INTERNAL_LAMBDA enable_lambda) - - -#------------------------------------------------------------------------------- -# List of possible Options for HPX -#------------------------------------------------------------------------------- -# From Makefile.kokkos: Options: enable_async_dispatch -set(KOKKOS_HPX_OPTIONS_LIST) -list(APPEND KOKKOS_HPX_OPTIONS_LIST - ASYNC_DISPATCH # enable_async_dispatch - ) - -# Map of cmake variables to Makefile variables -set(KOKKOS_INTERNAL_ENABLE_ASYNC_DISPATCH enable_async_dispatch) - - -#------------------------------------------------------------------------------- -#------------------------------- Create doc strings ---------------------------- -#------------------------------------------------------------------------------- - -set(tmpr "\n ") -string(REPLACE ";" ${tmpr} KOKKOS_INTERNAL_ARCH_DOCSTR "${KOKKOS_ARCH_LIST}") -set(KOKKOS_INTERNAL_ARCH_DOCSTR "${tmpr}${KOKKOS_INTERNAL_ARCH_DOCSTR}") -# This would be useful, but we use Foo_ENABLE mechanisms -#string(REPLACE ";" ${tmpr} KOKKOS_INTERNAL_DEVICES_DOCSTR "${KOKKOS_DEVICES_LIST}") -#string(REPLACE ";" ${tmpr} KOKKOS_INTERNAL_USE_TPLS_DOCSTR "${KOKKOS_USE_TPLS_LIST}") -#string(REPLACE ";" ${tmpr} KOKKOS_INTERNAL_CUDA_OPTIONS_DOCSTR "${KOKKOS_CUDA_OPTIONS_LIST}") - -#------------------------------------------------------------------------------- -#------------------------------- GENERAL OPTIONS ------------------------------- -#------------------------------------------------------------------------------- - -# Setting this variable to a value other than "None" can improve host -# performance by turning on architecture specific code. -# NOT SET is used to determine if the option is passed in. It is reset to -# default "None" down below. -set(KOKKOS_ARCH "NOT_SET" CACHE STRING - "Optimize for specific host architecture. Options are: ${KOKKOS_INTERNAL_ARCH_DOCSTR}") - -# Whether to build separate libraries or now -set(KOKKOS_SEPARATE_LIBS OFF CACHE BOOL "OFF = kokkos. ON = kokkoscore, kokkoscontainers, and kokkosalgorithms.") - -# Qthreads options. -set(KOKKOS_QTHREADS_DIR "" CACHE PATH "Location of Qthreads library.") - -# HPX options. -set(KOKKOS_HPX_DIR "" CACHE PATH "Location of HPX library.") - -# Whether to build separate libraries or now -set(KOKKOS_SEPARATE_TESTS OFF CACHE BOOL "Provide unit test targets with finer granularity.") - -#------------------------------------------------------------------------------- -#------------------------------- KOKKOS_DEVICES -------------------------------- -#------------------------------------------------------------------------------- -# Figure out default settings -IF(Trilinos_ENABLE_Kokkos) - set_kokkos_default_default(SERIAL ON) - set_kokkos_default_default(PTHREAD OFF) - IF(TPL_ENABLE_QTHREAD) - set_kokkos_default_default(QTHREADS ${TPL_ENABLE_QTHREAD}) - ELSE() - set_kokkos_default_default(QTHREADS OFF) - ENDIF() - IF(TPL_ENABLE_HPX) - set_kokkos_default_default(HPX ON) - ELSE() - set_kokkos_default_default(HPX OFF) - ENDIF() - IF(Trilinos_ENABLE_OpenMP) - set_kokkos_default_default(OPENMP ${Trilinos_ENABLE_OpenMP}) - ELSE() - set_kokkos_default_default(OPENMP OFF) - ENDIF() - IF(TPL_ENABLE_CUDA) - set_kokkos_default_default(CUDA ${TPL_ENABLE_CUDA}) - ELSE() - set_kokkos_default_default(CUDA OFF) - ENDIF() - set_kokkos_default_default(ROCM OFF) -ELSE() - set_kokkos_default_default(SERIAL ON) - set_kokkos_default_default(OPENMP OFF) - set_kokkos_default_default(PTHREAD OFF) - set_kokkos_default_default(QTHREAD OFF) - set_kokkos_default_default(HPX OFF) - set_kokkos_default_default(CUDA OFF) - set_kokkos_default_default(ROCM OFF) -ENDIF() - -# Set which Kokkos backend to use. -# These are the actual options that define the settings. -set(KOKKOS_ENABLE_SERIAL ${KOKKOS_INTERNAL_ENABLE_SERIAL_DEFAULT} CACHE BOOL "Whether to enable the Kokkos::Serial device. This device executes \"parallel\" kernels sequentially on a single CPU thread. It is enabled by default. If you disable this device, please enable at least one other CPU device, such as Kokkos::OpenMP or Kokkos::Threads.") -set(KOKKOS_ENABLE_OPENMP ${KOKKOS_INTERNAL_ENABLE_OPENMP_DEFAULT} CACHE BOOL "Enable OpenMP support in Kokkos." FORCE) -set(KOKKOS_ENABLE_PTHREAD ${KOKKOS_INTERNAL_ENABLE_PTHREAD_DEFAULT} CACHE BOOL "Enable Pthread support in Kokkos.") -set(KOKKOS_ENABLE_QTHREADS ${KOKKOS_INTERNAL_ENABLE_QTHREADS_DEFAULT} CACHE BOOL "Enable Qthreads support in Kokkos.") -set(KOKKOS_ENABLE_HPX ${KOKKOS_INTERNAL_ENABLE_HPX_DEFAULT} CACHE BOOL "Enable HPX support in Kokkos.") -set(KOKKOS_ENABLE_CUDA ${KOKKOS_INTERNAL_ENABLE_CUDA_DEFAULT} CACHE BOOL "Enable CUDA support in Kokkos.") -set(KOKKOS_ENABLE_ROCM ${KOKKOS_INTERNAL_ENABLE_ROCM_DEFAULT} CACHE BOOL "Enable ROCm support in Kokkos.") - - - -#------------------------------------------------------------------------------- -#------------------------------- KOKKOS DEBUG and PROFILING -------------------- -#------------------------------------------------------------------------------- - -# Debug related options enable compiler warnings - -set_kokkos_default_default(DEBUG OFF) -set(KOKKOS_ENABLE_DEBUG ${KOKKOS_INTERNAL_ENABLE_DEBUG_DEFAULT} CACHE BOOL "Enable Kokkos Debug.") - -# From Makefile.kokkos: Advanced Options: -#compiler_warnings, aggressive_vectorization, disable_profiling, disable_dualview_modify_check, enable_profile_load_print -set_kokkos_default_default(COMPILER_WARNINGS OFF) -set(KOKKOS_ENABLE_COMPILER_WARNINGS ${KOKKOS_INTERNAL_ENABLE_COMPILER_WARNINGS_DEFAULT} CACHE BOOL "Enable compiler warnings.") - -set_kokkos_default_default(DEBUG_DUALVIEW_MODIFY_CHECK OFF) -set(KOKKOS_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK ${KOKKOS_INTERNAL_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK_DEFAULT} CACHE BOOL "Enable dualview modify check.") - -# Enable aggressive vectorization. -set_kokkos_default_default(AGGRESSIVE_VECTORIZATION OFF) -set(KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION ${KOKKOS_INTERNAL_ENABLE_AGGRESSIVE_VECTORIZATION_DEFAULT} CACHE BOOL "Enable aggressive vectorization.") - -# Enable profiling. -set_kokkos_default_default(PROFILING ON) -set(KOKKOS_ENABLE_PROFILING ${KOKKOS_INTERNAL_ENABLE_PROFILING_DEFAULT} CACHE BOOL "Enable profiling.") - -set_kokkos_default_default(PROFILING_LOAD_PRINT OFF) -set(KOKKOS_ENABLE_PROFILING_LOAD_PRINT ${KOKKOS_INTERNAL_ENABLE_PROFILING_LOAD_PRINT_DEFAULT} CACHE BOOL "Enable profile load print.") - -set_kokkos_default_default(DEPRECATED_CODE ON) -set(KOKKOS_ENABLE_DEPRECATED_CODE ${KOKKOS_INTERNAL_ENABLE_DEPRECATED_CODE_DEFAULT} CACHE BOOL "Enable deprecated code.") - -set_kokkos_default_default(EXPLICIT_INSTANTIATION OFF) -set(KOKKOS_ENABLE_EXPLICIT_INSTANTIATION ${KOKKOS_INTERNAL_ENABLE_EXPLICIT_INSTANTIATION_DEFAULT} CACHE BOOL "Enable explicit template instantiation.") - -#------------------------------------------------------------------------------- -#------------------------------- KOKKOS_USE_TPLS ------------------------------- -#------------------------------------------------------------------------------- -# Enable hwloc library. -# Figure out default: -IF(Trilinos_ENABLE_Kokkos AND TPL_ENABLE_HWLOC) - set_kokkos_default_default(HWLOC ON) -ELSE() - set_kokkos_default_default(HWLOC OFF) -ENDIF() -set(KOKKOS_ENABLE_HWLOC ${KOKKOS_INTERNAL_ENABLE_HWLOC_DEFAULT} CACHE BOOL "Enable hwloc for better process placement.") -set(KOKKOS_HWLOC_DIR "" CACHE PATH "Location of hwloc library. (kokkos tpl)") - -# Enable memkind library. -set_kokkos_default_default(MEMKIND OFF) -set(KOKKOS_ENABLE_MEMKIND ${KOKKOS_INTERNAL_ENABLE_MEMKIND_DEFAULT} CACHE BOOL "Enable memkind. (kokkos tpl)") -set(KOKKOS_MEMKIND_DIR "" CACHE PATH "Location of memkind library. (kokkos tpl)") - -# Enable rt library. -IF(Trilinos_ENABLE_Kokkos) - IF(DEFINED TPL_ENABLE_LIBRT) - set_kokkos_default_default(LIBRT ${TPL_ENABLE_LIBRT}) - ELSE() - set_kokkos_default_default(LIBRT OFF) - ENDIF() -ELSE() - set_kokkos_default_default(LIBRT ON) -ENDIF() -set(KOKKOS_ENABLE_LIBRT ${KOKKOS_INTERNAL_ENABLE_LIBRT_DEFAULT} CACHE BOOL "Enable librt for more precise timer. (kokkos tpl)") - - -#------------------------------------------------------------------------------- -#------------------------------- KOKKOS_CUDA_OPTIONS --------------------------- -#------------------------------------------------------------------------------- - -# CUDA options. -# Set Defaults -set_kokkos_default_default(CUDA_LDG_INTRINSIC_DEFAULT OFF) -set_kokkos_default_default(CUDA_UVM_DEFAULT OFF) -set_kokkos_default_default(CUDA_RELOCATABLE_DEVICE_CODE OFF) -IF(Trilinos_ENABLE_Kokkos) - IF(KOKKOS_ENABLE_CUDA) - find_package(CUDA) - ENDIF() - IF (DEFINED CUDA_VERSION) - IF (CUDA_VERSION VERSION_GREATER "7.0") - set_kokkos_default_default(CUDA_LAMBDA ON) - ELSE() - set_kokkos_default_default(CUDA_LAMBDA OFF) - ENDIF() - ENDIF() -ELSE() - set_kokkos_default_default(CUDA_LAMBDA OFF) -ENDIF() - -# Set actual options -set(KOKKOS_CUDA_DIR "" CACHE PATH "Location of CUDA library. Defaults to where nvcc installed.") -set(KOKKOS_ENABLE_CUDA_LDG_INTRINSIC ${KOKKOS_INTERNAL_ENABLE_CUDA_LDG_INTRINSIC_DEFAULT} CACHE BOOL "Enable CUDA LDG. (cuda option)") -set(KOKKOS_ENABLE_CUDA_UVM ${KOKKOS_INTERNAL_ENABLE_CUDA_UVM_DEFAULT} CACHE BOOL "Enable CUDA unified virtual memory.") -set(KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE ${KOKKOS_INTERNAL_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE_DEFAULT} CACHE BOOL "Enable relocatable device code for CUDA. (cuda option)") -set(KOKKOS_ENABLE_CUDA_LAMBDA ${KOKKOS_INTERNAL_ENABLE_CUDA_LAMBDA_DEFAULT} CACHE BOOL "Enable lambdas for CUDA. (cuda option)") - - -#------------------------------------------------------------------------------- -#------------------------------- KOKKOS_HPX_OPTIONS ---------------------------- -#------------------------------------------------------------------------------- - -# HPX options. -# Set Defaults -set_kokkos_default_default(HPX_ASYNC_DISPATCH OFF) - -# Set actual options -set(KOKKOS_ENABLE_HPX_ASYNC_DISPATCH ${KOKKOS_INTERNAL_ENABLE_HPX_ASYNC_DISPATCH_DEFAULT} CACHE BOOL "Enable HPX async dispatch.") - - -#------------------------------------------------------------------------------- -#----------------------- HOST ARCH AND LEGACY TRIBITS -------------------------- -#------------------------------------------------------------------------------- - -# This defines the previous legacy TriBITS builds. -set(KOKKOS_LEGACY_TRIBITS False) -IF ("${KOKKOS_ARCH}" STREQUAL "NOT_SET") - set(KOKKOS_ARCH "None") - IF(KOKKOS_HAS_TRILINOS) - set(KOKKOS_LEGACY_TRIBITS True) - ENDIF() -ENDIF() -IF (KOKKOS_HAS_TRILINOS) - IF (KOKKOS_LEGACY_TRIBITS) - message(STATUS "Using the legacy tribits build because KOKKOS_ARCH not set") - ELSE() - message(STATUS "NOT using the legacy tribits build because KOKKOS_ARCH *is* set") - ENDIF() -ENDIF() - -#------------------------------------------------------------------------------- -#----------------------- Set CamelCase Options if they are not yet set --------- -#------------------------------------------------------------------------------- - -foreach(opt ${KOKKOS_INTERNAL_ENABLE_OPTIONS_LIST}) - string(TOUPPER ${opt} OPT ) - UNSET(KOKKOS_ENABLE_${OPT}_INTERNAL CACHE) - SET(KOKKOS_ENABLE_${OPT}_INTERNAL ${KOKKOS_ENABLE_${OPT}} CACHE BOOL INTERNAL) - IF(DEFINED KOKKOS_ENABLE_${OPT}) - UNSET(Kokkos_ENABLE_${opt} CACHE) - SET(Kokkos_ENABLE_${opt} ${KOKKOS_ENABLE_${OPT}} CACHE BOOL "CamelCase Compatibility setting for KOKKOS_ENABLE_${OPT}") - ENDIF() -endforeach() diff --git a/lib/kokkos/cmake/kokkos_pick_cxx_std.cmake b/lib/kokkos/cmake/kokkos_pick_cxx_std.cmake new file mode 100644 index 0000000000000000000000000000000000000000..cf14948f43f26b089078f994817a2c3e1230744f --- /dev/null +++ b/lib/kokkos/cmake/kokkos_pick_cxx_std.cmake @@ -0,0 +1,46 @@ +# From CMake 3.10 documentation + +#This can run at any time +KOKKOS_OPTION(CXX_STANDARD "" STRING "The C++ standard for Kokkos to use: 11, 14, 17, or 20. If empty, this will default to CMAKE_CXX_STANDARD. If both CMAKE_CXX_STANDARD and Kokkos_CXX_STANDARD are empty, this will default to 11") + +# Set CXX standard flags +SET(KOKKOS_ENABLE_CXX11 OFF) +SET(KOKKOS_ENABLE_CXX14 OFF) +SET(KOKKOS_ENABLE_CXX17 OFF) +SET(KOKKOS_ENABLE_CXX20 OFF) +IF (KOKKOS_CXX_STANDARD) + IF (${KOKKOS_CXX_STANDARD} STREQUAL "c++98") + MESSAGE(FATAL_ERROR "Kokkos no longer supports C++98 - minimum C++11") + ELSEIF (${KOKKOS_CXX_STANDARD} STREQUAL "c++11") + MESSAGE(WARNING "Deprecated Kokkos C++ standard set as 'c++11'. Use '11' instead.") + SET(KOKKOS_CXX_STANDARD "11") + ELSEIF(${KOKKOS_CXX_STANDARD} STREQUAL "c++14") + MESSAGE(WARNING "Deprecated Kokkos C++ standard set as 'c++14'. Use '14' instead.") + SET(KOKKOS_CXX_STANDARD "14") + ELSEIF(${KOKKOS_CXX_STANDARD} STREQUAL "c++17") + MESSAGE(WARNING "Deprecated Kokkos C++ standard set as 'c++17'. Use '17' instead.") + SET(KOKKOS_CXX_STANDARD "17") + ELSEIF(${KOKKOS_CXX_STANDARD} STREQUAL "c++1y") + MESSAGE(WARNING "Deprecated Kokkos C++ standard set as 'c++1y'. Use '1Y' instead.") + SET(KOKKOS_CXX_STANDARD "1Y") + ELSEIF(${KOKKOS_CXX_STANDARD} STREQUAL "c++1z") + MESSAGE(WARNING "Deprecated Kokkos C++ standard set as 'c++1z'. Use '1Z' instead.") + SET(KOKKOS_CXX_STANDARD "1Z") + ELSEIF(${KOKKOS_CXX_STANDARD} STREQUAL "c++2a") + MESSAGE(WARNING "Deprecated Kokkos C++ standard set as 'c++2a'. Use '2A' instead.") + SET(KOKKOS_CXX_STANDARD "2A") + ENDIF() +ENDIF() + +IF (NOT KOKKOS_CXX_STANDARD AND NOT CMAKE_CXX_STANDARD) + MESSAGE(STATUS "Setting default Kokkos CXX standard to 11") + SET(KOKKOS_CXX_STANDARD "11") +ELSEIF(NOT KOKKOS_CXX_STANDARD) + MESSAGE(STATUS "Setting default Kokkos CXX standard to ${CMAKE_CXX_STANDARD}") + SET(KOKKOS_CXX_STANDARD ${CMAKE_CXX_STANDARD}) +ENDIF() + + + + + diff --git a/lib/kokkos/cmake/kokkos_settings.cmake b/lib/kokkos/cmake/kokkos_settings.cmake deleted file mode 100644 index 2c622d0de91f87f59f78a598820b80bc294d0b19..0000000000000000000000000000000000000000 --- a/lib/kokkos/cmake/kokkos_settings.cmake +++ /dev/null @@ -1,259 +0,0 @@ -########################## NOTES ############################################### -# This files goal is to take CMake options found in kokkos_options.cmake but -# possibly set from elsewhere -# (see: trilinos/cmake/ProjectCOmpilerPostConfig.cmake) -# using CMake idioms and map them onto the KOKKOS_SETTINGS variables that gets -# passed to the kokkos makefile configuration: -# make -f ${CMAKE_SOURCE_DIR}/core/src/Makefile ${KOKKOS_SETTINGS} build-makefile-cmake-kokkos -# that generates KokkosCore_config.h and kokkos_generated_settings.cmake -# To understand how to form KOKKOS_SETTINGS, see -# /Makefile.kokkos - -#------------------------------------------------------------------------------- -#------------------------------- GENERAL OPTIONS ------------------------------- -#------------------------------------------------------------------------------- - -# Ensure that KOKKOS_ARCH is in the ARCH_LIST -if (KOKKOS_ARCH MATCHES ",") - message("-- Detected a comma in: KOKKOS_ARCH=`${KOKKOS_ARCH}`") - message("-- Although we prefer KOKKOS_ARCH to be semicolon-delimited, we do allow") - message("-- comma-delimited values for compatibility with scripts (see github.com/trilinos/Trilinos/issues/2330)") - string(REPLACE "," ";" KOKKOS_ARCH "${KOKKOS_ARCH}") - message("-- Commas were changed to semicolons, now KOKKOS_ARCH=`${KOKKOS_ARCH}`") -endif() -foreach(arch ${KOKKOS_ARCH}) - list(FIND KOKKOS_ARCH_LIST ${arch} indx) - if (indx EQUAL -1) - message(FATAL_ERROR "`${arch}` is not an accepted value in KOKKOS_ARCH=`${KOKKOS_ARCH}`." - " Please pick from these choices: ${KOKKOS_INTERNAL_ARCH_DOCSTR}") - endif () -endforeach() - -# KOKKOS_SETTINGS uses KOKKOS_ARCH -string(REPLACE ";" "," KOKKOS_GMAKE_ARCH "${KOKKOS_ARCH}") - -# From Makefile.kokkos: Options: yes,no -if(${KOKKOS_ENABLE_DEBUG}) - set(KOKKOS_GMAKE_DEBUG yes) -else() - set(KOKKOS_GMAKE_DEBUG no) -endif() - -#------------------------------- KOKKOS_DEVICES -------------------------------- -# Can have multiple devices -set(KOKKOS_DEVICESl) -foreach(devopt ${KOKKOS_DEVICES_LIST}) - string(TOUPPER ${devopt} devoptuc) - if (${KOKKOS_ENABLE_${devoptuc}}) - list(APPEND KOKKOS_DEVICESl ${devopt}) - endif () -endforeach() -# List needs to be comma-delmitted -string(REPLACE ";" "," KOKKOS_GMAKE_DEVICES "${KOKKOS_DEVICESl}") - -#------------------------------- KOKKOS_OPTIONS -------------------------------- -# From Makefile.kokkos: Options: aggressive_vectorization,disable_profiling,disable_deprecated_code -#compiler_warnings, aggressive_vectorization, disable_profiling, disable_dualview_modify_check, enable_profile_load_print - -set(KOKKOS_OPTIONSl) -if(${KOKKOS_ENABLE_COMPILER_WARNINGS}) - list(APPEND KOKKOS_OPTIONSl compiler_warnings) -endif() -if(${KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION}) - list(APPEND KOKKOS_OPTIONSl aggressive_vectorization) -endif() -if(NOT ${KOKKOS_ENABLE_PROFILING}) - list(APPEND KOKKOS_OPTIONSl disable_profiling) -endif() -if(NOT ${KOKKOS_ENABLE_DEPRECATED_CODE}) - list(APPEND KOKKOS_OPTIONSl disable_deprecated_code) -endif() -if(NOT ${KOKKOS_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK}) - list(APPEND KOKKOS_OPTIONSl disable_dualview_modify_check) -endif() -if(${KOKKOS_ENABLE_PROFILING_LOAD_PRINT}) - list(APPEND KOKKOS_OPTIONSl enable_profile_load_print) -endif() -if(${KOKKOS_ENABLE_EXPLICIT_INSTANTIATION}) - list(APPEND KOKKOS_OPTIONSl enable_eti) -endif() -# List needs to be comma-delimitted -string(REPLACE ";" "," KOKKOS_GMAKE_OPTIONS "${KOKKOS_OPTIONSl}") - - -#------------------------------- KOKKOS_USE_TPLS ------------------------------- -# Construct the Makefile options -set(KOKKOS_USE_TPLSl) -foreach(tplopt ${KOKKOS_USE_TPLS_LIST}) - if (${KOKKOS_ENABLE_${tplopt}}) - list(APPEND KOKKOS_USE_TPLSl ${KOKKOS_INTERNAL_${tplopt}}) - endif () -endforeach() -# List needs to be comma-delimitted -string(REPLACE ";" "," KOKKOS_GMAKE_USE_TPLS "${KOKKOS_USE_TPLSl}") - - -#------------------------------- KOKKOS_CUDA_OPTIONS --------------------------- -# Construct the Makefile options -set(KOKKOS_CUDA_OPTIONSl) -foreach(cudaopt ${KOKKOS_CUDA_OPTIONS_LIST}) - if (${KOKKOS_ENABLE_CUDA_${cudaopt}}) - list(APPEND KOKKOS_CUDA_OPTIONSl ${KOKKOS_INTERNAL_${cudaopt}}) - endif () -endforeach() -# List needs to be comma-delmitted -string(REPLACE ";" "," KOKKOS_GMAKE_CUDA_OPTIONS "${KOKKOS_CUDA_OPTIONSl}") - -#------------------------------- PATH VARIABLES -------------------------------- -# Want makefile to use same executables specified which means modifying -# the path so the $(shell ...) commands in the makefile see the right exec -# Also, the Makefile's use FOO_PATH naming scheme for -I/-L construction -#TODO: Makefile.kokkos allows this to be overwritten? ROCM_HCC_PATH - -set(KOKKOS_INTERNAL_PATHS) -set(addpathl) -foreach(kvar IN LISTS KOKKOS_USE_TPLS_LIST ITEMS CUDA QTHREADS) - if(${KOKKOS_ENABLE_${kvar}}) - if(DEFINED KOKKOS_${kvar}_DIR) - set(KOKKOS_INTERNAL_PATHS ${KOKKOS_INTERNAL_PATHS} "${kvar}_PATH=${KOKKOS_${kvar}_DIR}") - if(IS_DIRECTORY ${KOKKOS_${kvar}_DIR}/bin) - list(APPEND addpathl ${KOKKOS_${kvar}_DIR}/bin) - endif() - endif() - endif() -endforeach() -# Path env is : delimitted -string(REPLACE ";" ":" KOKKOS_INTERNAL_ADDTOPATH "${addpathl}") - - -######################### SET KOKKOS_SETTINGS ################################## -# Set the KOKKOS_SETTINGS String -- this is the primary communication with the -# makefile configuration. See Makefile.kokkos - -set(KOKKOS_SETTINGS KOKKOS_CMAKE=yes) -set(KOKKOS_SETTINGS ${KOKKOS_SETTINGS} KOKKOS_SRC_PATH=${KOKKOS_SRC_PATH}) -set(KOKKOS_SETTINGS ${KOKKOS_SETTINGS} KOKKOS_PATH=${KOKKOS_PATH}) -set(KOKKOS_SETTINGS ${KOKKOS_SETTINGS} KOKKOS_INSTALL_PATH=${CMAKE_INSTALL_PREFIX}) - -# Form of KOKKOS_foo=$KOKKOS_foo -foreach(kvar ARCH;DEVICES;DEBUG;OPTIONS;CUDA_OPTIONS;USE_TPLS) - if(DEFINED KOKKOS_GMAKE_${kvar}) - if (NOT "${KOKKOS_GMAKE_${kvar}}" STREQUAL "") - set(KOKKOS_SETTINGS ${KOKKOS_SETTINGS} KOKKOS_${kvar}=${KOKKOS_GMAKE_${kvar}}) - endif() - endif() -endforeach() - -# Form of VAR=VAL -#TODO: Makefile supports MPICH_CXX, OMPI_CXX as well -foreach(ovar CXX;CXXFLAGS;LDFLAGS) - if(DEFINED ${ovar}) - if (NOT "${${ovar}}" STREQUAL "") - set(KOKKOS_SETTINGS ${KOKKOS_SETTINGS} ${ovar}=${${ovar}}) - endif() - endif() -endforeach() - -# Finally, do the paths -if (NOT "${KOKKOS_INTERNAL_PATHS}" STREQUAL "") - set(KOKKOS_SETTINGS ${KOKKOS_SETTINGS} ${KOKKOS_INTERNAL_PATHS}) -endif() -if (NOT "${KOKKOS_INTERNAL_ADDTOPATH}" STREQUAL "") - set(KOKKOS_SETTINGS ${KOKKOS_SETTINGS} "PATH=${KOKKOS_INTERNAL_ADDTOPATH}:$ENV{PATH}") -endif() - -if (CMAKE_CXX_STANDARD) - if (CMAKE_CXX_STANDARD STREQUAL "98") - message(FATAL_ERROR "Kokkos requires C++11 or newer!") - endif() - set(KOKKOS_CXX_STANDARD "c++${CMAKE_CXX_STANDARD}") - if (CMAKE_CXX_EXTENSIONS) - if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(KOKKOS_CXX_STANDARD "gnu++${CMAKE_CXX_STANDARD}") - endif() - endif() - set(KOKKOS_SETTINGS ${KOKKOS_SETTINGS} "KOKKOS_CXX_STANDARD=\"${KOKKOS_CXX_STANDARD}\"") -endif() - -# Final form that gets passed to make -set(KOKKOS_SETTINGS env ${KOKKOS_SETTINGS}) - - -############################ PRINT CONFIGURE STATUS ############################ - -if(KOKKOS_CMAKE_VERBOSE) - message(STATUS "") - message(STATUS "****************** Kokkos Settings ******************") - message(STATUS "Execution Spaces") - - if(KOKKOS_ENABLE_CUDA) - message(STATUS " Device Parallel: Cuda") - else() - message(STATUS " Device Parallel: None") - endif() - - if(KOKKOS_ENABLE_OPENMP) - message(STATUS " Host Parallel: OpenMP") - elseif(KOKKOS_ENABLE_PTHREAD) - message(STATUS " Host Parallel: Pthread") - elseif(KOKKOS_ENABLE_QTHREADS) - message(STATUS " Host Parallel: Qthreads") - elseif(KOKKOS_ENABLE_HPX) - message(STATUS " Host Parallel: HPX") - else() - message(STATUS " Host Parallel: None") - endif() - - if(KOKKOS_ENABLE_SERIAL) - message(STATUS " Host Serial: Serial") - else() - message(STATUS " Host Serial: None") - endif() - - message(STATUS "") - message(STATUS "Architectures:") - message(STATUS " ${KOKKOS_GMAKE_ARCH}") - - message(STATUS "") - message(STATUS "Enabled options") - - if(KOKKOS_SEPARATE_LIBS) - message(STATUS " KOKKOS_SEPARATE_LIBS") - endif() - - foreach(opt IN LISTS KOKKOS_INTERNAL_ENABLE_OPTIONS_LIST) - string(TOUPPER ${opt} OPT) - if (KOKKOS_ENABLE_${OPT}) - message(STATUS " KOKKOS_ENABLE_${OPT}") - endif() - endforeach() - - if(KOKKOS_ENABLE_CUDA) - if(KOKKOS_CUDA_DIR) - message(STATUS " KOKKOS_CUDA_DIR: ${KOKKOS_CUDA_DIR}") - endif() - endif() - - if(KOKKOS_QTHREADS_DIR) - message(STATUS " KOKKOS_QTHREADS_DIR: ${KOKKOS_QTHREADS_DIR}") - endif() - - if(KOKKOS_HWLOC_DIR) - message(STATUS " KOKKOS_HWLOC_DIR: ${KOKKOS_HWLOC_DIR}") - endif() - - if(KOKKOS_MEMKIND_DIR) - message(STATUS " KOKKOS_MEMKIND_DIR: ${KOKKOS_MEMKIND_DIR}") - endif() - - if(KOKKOS_HPX_DIR) - message(STATUS " KOKKOS_HPX_DIR: ${KOKKOS_HPX_DIR}") - endif() - - message(STATUS "") - message(STATUS "Final kokkos settings variable:") - message(STATUS " ${KOKKOS_SETTINGS}") - - message(STATUS "*****************************************************") - message(STATUS "") -endif() diff --git a/lib/kokkos/cmake/kokkos_test_cxx_std.cmake b/lib/kokkos/cmake/kokkos_test_cxx_std.cmake new file mode 100644 index 0000000000000000000000000000000000000000..c264517abe114bc306a19ca3459f584548506dd5 --- /dev/null +++ b/lib/kokkos/cmake/kokkos_test_cxx_std.cmake @@ -0,0 +1,144 @@ +KOKKOS_CFG_DEPENDS(CXX_STD COMPILER_ID) + +FUNCTION(kokkos_set_cxx_standard_feature standard) + SET(EXTENSION_NAME CMAKE_CXX${standard}_EXTENSION_COMPILE_OPTION) + SET(STANDARD_NAME CMAKE_CXX${standard}_STANDARD_COMPILE_OPTION) + SET(FEATURE_NAME cxx_std_${standard}) + #CMake's way of telling us that the standard (or extension) + #flags are supported is the extension/standard variables + IF (NOT DEFINED CMAKE_CXX_EXTENSIONS) + IF(KOKKOS_DONT_ALLOW_EXTENSIONS) + GLOBAL_SET(KOKKOS_USE_CXX_EXTENSIONS OFF) + ELSE() + GLOBAL_SET(KOKKOS_USE_CXX_EXTENSIONS ON) + ENDIF() + ELSEIF(CMAKE_CXX_EXTENSIONS) + IF(KOKKOS_DONT_ALLOW_EXTENSIONS) + MESSAGE(FATAL_ERROR "The chosen configuration does not support CXX extensions flags: ${KOKKOS_DONT_ALLOW_EXTENSIONS}. Must set CMAKE_CXX_EXTENSIONS=OFF to continue") + ELSE() + GLOBAL_SET(KOKKOS_USE_CXX_EXTENSIONS ON) + ENDIF() + ELSE() + #For trilinos, we need to make sure downstream projects + GLOBAL_SET(KOKKOS_USE_CXX_EXTENSIONS OFF) + ENDIF() + + IF (KOKKOS_USE_CXX_EXTENSIONS AND ${EXTENSION_NAME}) + MESSAGE(STATUS "Using ${${EXTENSION_NAME}} for C++${standard} extensions as feature") + GLOBAL_SET(KOKKOS_CXX_STANDARD_FEATURE ${FEATURE_NAME}) + ELSEIF(NOT KOKKOS_USE_CXX_EXTENSIONS AND ${STANDARD_NAME}) + MESSAGE(STATUS "Using ${${STANDARD_NAME}} for C++${standard} standard as feature") + GLOBAL_SET(KOKKOS_CXX_STANDARD_FEATURE ${FEATURE_NAME}) + ELSE() + #nope, we can't do anything here + MESSAGE(WARNING "C++${standard} is not supported as a compiler feature. We will choose custom flags for now, but this behavior has been deprecated. Please open an issue at https://github.com/kokkos/kokkos/issues reporting that ${KOKKOS_CXX_COMPILER_ID} ${KOKKOS_CXX_COMPILER_VERSION} failed for ${KOKKOS_CXX_STANDARD}, preferrably including your CMake command.") + GLOBAL_SET(KOKKOS_CXX_STANDARD_FEATURE "") + ENDIF() + + IF(NOT ${FEATURE_NAME} IN_LIST CMAKE_CXX_COMPILE_FEATURES) + MESSAGE(FATAL_ERROR "Compiler ${KOKKOS_CXX_COMPILER_ID} should support ${FEATURE_NAME}, but CMake reports feature not supported") + ENDIF() +ENDFUNCTION() + + +IF (KOKKOS_CXX_STANDARD AND CMAKE_CXX_STANDARD) + #make sure these are consistent + IF (NOT KOKKOS_CXX_STANDARD STREQUAL CMAKE_CXX_STANDARD) + MESSAGE(WARNING "Specified both CMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD} and KOKKOS_CXX_STANDARD=${KOKKOS_CXX_STANDARD}, but they don't match") + SET(CMAKE_CXX_STANDARD ${KOKKOS_CXX_STANDARD} CACHE STRING "C++ standard" FORCE) + ENDIF() +ENDIF() + + +IF (KOKKOS_CXX_STANDARD STREQUAL "11" ) + kokkos_set_cxx_standard_feature(11) + SET(KOKKOS_ENABLE_CXX11 ON) + SET(KOKKOS_CXX_INTERMEDIATE_STANDARD "11") +ELSEIF(KOKKOS_CXX_STANDARD STREQUAL "14") + kokkos_set_cxx_standard_feature(14) + SET(KOKKOS_CXX_INTERMEDIATE_STANDARD "1Y") + SET(KOKKOS_ENABLE_CXX14 ON) +ELSEIF(KOKKOS_CXX_STANDARD STREQUAL "17") + kokkos_set_cxx_standard_feature(17) + SET(KOKKOS_CXX_INTERMEDIATE_STANDARD "1Z") + SET(KOKKOS_ENABLE_CXX17 ON) +ELSEIF(KOKKOS_CXX_STANDARD STREQUAL "20") + kokkos_set_cxx_standard_feature(20) + SET(KOKKOS_CXX_INTERMEDIATE_STANDARD "2A") + SET(KOKKOS_ENABLE_CXX20 ON) +ELSEIF(KOKKOS_CXX_STANDARD STREQUAL "98") + MESSAGE(FATAL_ERROR "Kokkos requires C++11 or newer!") +ELSE() + MESSAGE(FATAL_ERROR "Unknown C++ standard ${KOKKOS_CXX_STANDARD} - must be 11, 14, 17, or 20") +ENDIF() + + + +# Enforce that extensions are turned off for nvcc_wrapper. +# For compiling CUDA code using nvcc_wrapper, we will use the host compiler's +# flags for turning on C++11. Since for compiler ID and versioning purposes +# CMake recognizes the host compiler when calling nvcc_wrapper, this just +# works. Both NVCC and nvcc_wrapper only recognize '-std=c++11' which means +# that we can only use host compilers for CUDA builds that use those flags. +# It also means that extensions (gnu++11) can't be turned on for CUDA builds. + +IF(KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) + IF(NOT DEFINED CMAKE_CXX_EXTENSIONS) + SET(CMAKE_CXX_EXTENSIONS OFF) + ELSEIF(CMAKE_CXX_EXTENSIONS) + MESSAGE(FATAL_ERROR "NVCC doesn't support C++ extensions. Set -DCMAKE_CXX_EXTENSIONS=OFF") + ENDIF() +ENDIF() + +IF(KOKKOS_ENABLE_CUDA) + # ENFORCE that the compiler can compile CUDA code. + IF(KOKKOS_CXX_COMPILER_ID STREQUAL Clang) + IF(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS 4.0.0) + MESSAGE(FATAL_ERROR "Compiling CUDA code directly with Clang requires version 4.0.0 or higher.") + ENDIF() + IF(NOT DEFINED CMAKE_CXX_EXTENSIONS) + SET(CMAKE_CXX_EXTENSIONS OFF) + ELSEIF(CMAKE_CXX_EXTENSIONS) + MESSAGE(FATAL_ERROR "Compiling CUDA code with clang doesn't support C++ extensions. Set -DCMAKE_CXX_EXTENSIONS=OFF") + ENDIF() + ELSEIF(NOT KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) + MESSAGE(FATAL_ERROR "Invalid compiler for CUDA. The compiler must be nvcc_wrapper or Clang, but compiler ID was ${KOKKOS_CXX_COMPILER_ID}") + ENDIF() +ENDIF() + +IF (NOT KOKKOS_CXX_STANDARD_FEATURE) + #we need to pick the C++ flags ourselves + UNSET(CMAKE_CXX_STANDARD) + UNSET(CMAKE_CXX_STANDARD CACHE) + IF(KOKKOS_CXX_COMPILER_ID STREQUAL Cray) + INCLUDE(${KOKKOS_SRC_PATH}/cmake/cray.cmake) + kokkos_set_cray_flags(${KOKKOS_CXX_STANDARD} ${KOKKOS_CXX_INTERMEDIATE_STANDARD}) + ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL PGI) + INCLUDE(${KOKKOS_SRC_PATH}/cmake/pgi.cmake) + kokkos_set_pgi_flags(${KOKKOS_CXX_STANDARD} ${KOKKOS_CXX_INTERMEDIATE_STANDARD}) + ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL Intel) + INCLUDE(${KOKKOS_SRC_PATH}/cmake/intel.cmake) + kokkos_set_intel_flags(${KOKKOS_CXX_STANDARD} ${KOKKOS_CXX_INTERMEDIATE_STANDARD}) + ELSE() + INCLUDE(${KOKKOS_SRC_PATH}/cmake/gnu.cmake) + kokkos_set_gnu_flags(${KOKKOS_CXX_STANDARD} ${KOKKOS_CXX_INTERMEDIATE_STANDARD}) + ENDIF() + #check that the compiler accepts the C++ standard flag + INCLUDE(CheckCXXCompilerFlag) + IF (DEFINED CXX_STD_FLAGS_ACCEPTED) + UNSET(CXX_STD_FLAGS_ACCEPTED CACHE) + ENDIF() + CHECK_CXX_COMPILER_FLAG(${KOKKOS_CXX_STANDARD_FLAG} CXX_STD_FLAGS_ACCEPTED) + IF (NOT CXX_STD_FLAGS_ACCEPTED) + CHECK_CXX_COMPILER_FLAG(${KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG} CXX_INT_STD_FLAGS_ACCEPTED) + IF (NOT CXX_INT_STD_FLAGS_ACCEPTED) + MESSAGE(FATAL_ERROR "${KOKKOS_CXX_COMPILER_ID} did not accept ${KOKKOS_CXX_STANDARD_FLAG} or ${KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG}. You likely need to reduce the level of the C++ standard from ${KOKKOS_CXX_STANDARD}") + ENDIF() + SET(KOKKOS_CXX_STANDARD_FLAG ${KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG}) + ENDIF() + MESSAGE(STATUS "Compiler features not supported, but ${KOKKOS_CXX_COMPILER_ID} accepts ${KOKKOS_CXX_STANDARD_FLAG}") +ENDIF() + + + + diff --git a/lib/kokkos/cmake/kokkos_tpls.cmake b/lib/kokkos/cmake/kokkos_tpls.cmake new file mode 100644 index 0000000000000000000000000000000000000000..181a497d52a0e5f579e4a5ffbe05c1c1591edc29 --- /dev/null +++ b/lib/kokkos/cmake/kokkos_tpls.cmake @@ -0,0 +1,47 @@ +KOKKOS_CFG_DEPENDS(TPLS OPTIONS) +KOKKOS_CFG_DEPENDS(TPLS DEVICES) + +FUNCTION(KOKKOS_TPL_OPTION PKG DEFAULT) + KOKKOS_ENABLE_OPTION(${PKG} ${DEFAULT} "Whether to enable the ${PKG} library") + KOKKOS_OPTION(${PKG}_DIR "" PATH "Location of ${PKG} library") + SET(KOKKOS_ENABLE_${PKG} ${KOKKOS_ENABLE_${PKG}} PARENT_SCOPE) + SET(KOKKOS_${PKG}_DIR ${KOKKOS_${PKG}_DIR} PARENT_SCOPE) +ENDFUNCTION() + +KOKKOS_TPL_OPTION(HWLOC Off) +KOKKOS_TPL_OPTION(LIBNUMA Off) +KOKKOS_TPL_OPTION(MEMKIND Off) +KOKKOS_TPL_OPTION(CUDA Off) +KOKKOS_TPL_OPTION(LIBRT Off) +KOKKOS_TPL_OPTION(LIBDL On) + +IF(Trilinos_ENABLE_Kokkos AND TPL_ENABLE_HPX) +SET(HPX_DEFAULT ON) +ELSE() +SET(HPX_DEFAULT OFF) +ENDIF() +KOKKOS_TPL_OPTION(HPX ${HPX_DEFAULT}) + +IF(Trilinos_ENABLE_Kokkos AND TPL_ENABLE_PTHREAD) +SET(PTHREAD_DEFAULT ON) +ELSE() +SET(PTHREAD_DEFAULT OFF) +ENDIF() +KOKKOS_TPL_OPTION(PTHREAD ${PTHREAD_DEFAULT}) + + +#Make sure we use our local FindKokkosCuda.cmake +KOKKOS_IMPORT_TPL(HPX INTERFACE) +KOKKOS_IMPORT_TPL(CUDA INTERFACE) +KOKKOS_IMPORT_TPL(HWLOC) +KOKKOS_IMPORT_TPL(LIBNUMA) +KOKKOS_IMPORT_TPL(LIBRT) +KOKKOS_IMPORT_TPL(LIBDL) +KOKKOS_IMPORT_TPL(MEMKIND) +KOKKOS_IMPORT_TPL(PTHREAD INTERFACE) + +#Convert list to newlines (which CMake doesn't always like in cache variables) +STRING(REPLACE ";" "\n" KOKKOS_TPL_EXPORT_TEMP "${KOKKOS_TPL_EXPORTS}") +#Convert to a regular variable +UNSET(KOKKOS_TPL_EXPORTS CACHE) +SET(KOKKOS_TPL_EXPORTS ${KOKKOS_TPL_EXPORT_TEMP}) diff --git a/lib/kokkos/cmake/kokkos_tribits.cmake b/lib/kokkos/cmake/kokkos_tribits.cmake new file mode 100644 index 0000000000000000000000000000000000000000..d2317d2446ffdbf2626fff6117941af7510db865 --- /dev/null +++ b/lib/kokkos/cmake/kokkos_tribits.cmake @@ -0,0 +1,392 @@ +#These are tribits wrappers only ever called by Kokkos itself + +INCLUDE(CMakeParseArguments) +INCLUDE(CTest) +INCLUDE(GNUInstallDirs) + +MESSAGE(STATUS "The project name is: ${PROJECT_NAME}") + +#Leave this here for now - but only do for tribits +#This breaks the standalone CMake +IF (KOKKOS_HAS_TRILINOS) + IF(NOT DEFINED ${PROJECT_NAME}_ENABLE_OpenMP) + SET(${PROJECT_NAME}_ENABLE_OpenMP OFF) + ENDIF() + + IF(NOT DEFINED ${PROJECT_NAME}_ENABLE_HPX) + SET(${PROJECT_NAME}_ENABLE_HPX OFF) + ENDIF() + + IF(NOT DEFINED ${PROJECT_NAME}_ENABLE_DEBUG) + SET(${PROJECT_NAME}_ENABLE_DEBUG OFF) + ENDIF() + + IF(NOT DEFINED ${PROJECT_NAME}_ENABLE_CXX11) + SET(${PROJECT_NAME}_ENABLE_CXX11 ON) + ENDIF() + + IF(NOT DEFINED ${PROJECT_NAME}_ENABLE_TESTS) + SET(${PROJECT_NAME}_ENABLE_TESTS OFF) + ENDIF() + + IF(NOT DEFINED TPL_ENABLE_Pthread) + SET(TPL_ENABLE_Pthread OFF) + ENDIF() +ENDIF() + +MACRO(KOKKOS_SUBPACKAGE NAME) + if (KOKKOS_HAS_TRILINOS) + TRIBITS_SUBPACKAGE(${NAME}) + else() + SET(PACKAGE_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + SET(PARENT_PACKAGE_NAME ${PACKAGE_NAME}) + SET(PACKAGE_NAME ${PACKAGE_NAME}${NAME}) + STRING(TOUPPER ${PACKAGE_NAME} PACKAGE_NAME_UC) + SET(${PACKAGE_NAME}_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + endif() +ENDMACRO() + +MACRO(KOKKOS_SUBPACKAGE_POSTPROCESS) + if (KOKKOS_HAS_TRILINOS) + TRIBITS_SUBPACKAGE_POSTPROCESS() + endif() +ENDMACRO() + +MACRO(KOKKOS_PACKAGE_DECL) + + if (KOKKOS_HAS_TRILINOS) + TRIBITS_PACKAGE_DECL(Kokkos) + else() + SET(PACKAGE_NAME Kokkos) + SET(${PACKAGE_NAME}_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + STRING(TOUPPER ${PACKAGE_NAME} PACKAGE_NAME_UC) + endif() + + #SET(TRIBITS_DEPS_DIR "${CMAKE_SOURCE_DIR}/cmake/deps") + #FILE(GLOB TPLS_FILES "${TRIBITS_DEPS_DIR}/*.cmake") + #FOREACH(TPL_FILE ${TPLS_FILES}) + # TRIBITS_PROCESS_TPL_DEP_FILE(${TPL_FILE}) + #ENDFOREACH() + +ENDMACRO() + + +MACRO(KOKKOS_PROCESS_SUBPACKAGES) + if (KOKKOS_HAS_TRILINOS) + TRIBITS_PROCESS_SUBPACKAGES() + else() + ADD_SUBDIRECTORY(core) + ADD_SUBDIRECTORY(containers) + ADD_SUBDIRECTORY(algorithms) + ADD_SUBDIRECTORY(example) + endif() +ENDMACRO() + +MACRO(KOKKOS_PACKAGE_DEF) + if (KOKKOS_HAS_TRILINOS) + TRIBITS_PACKAGE_DEF() + else() + #do nothing + endif() +ENDMACRO() + +MACRO(KOKKOS_INTERNAL_ADD_LIBRARY_INSTALL LIBRARY_NAME) + KOKKOS_LIB_TYPE(${LIBRARY_NAME} INCTYPE) + TARGET_INCLUDE_DIRECTORIES(${LIBRARY_NAME} ${INCTYPE} $) + + INSTALL( + TARGETS ${LIBRARY_NAME} + EXPORT ${PROJECT_NAME} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + COMPONENT ${PACKAGE_NAME} + ) + + INSTALL( + TARGETS ${LIBRARY_NAME} + EXPORT KokkosTargets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + ) + + VERIFY_EMPTY(KOKKOS_ADD_LIBRARY ${PARSE_UNPARSED_ARGUMENTS}) +ENDMACRO() + +FUNCTION(KOKKOS_ADD_EXECUTABLE EXE_NAME) + if (KOKKOS_HAS_TRILINOS) + TRIBITS_ADD_EXECUTABLE(${EXE_NAME} ${ARGN}) + else() + CMAKE_PARSE_ARGUMENTS(PARSE + "TESTONLY" + "" + "SOURCES;TESTONLYLIBS" + ${ARGN}) + + ADD_EXECUTABLE(${EXE_NAME} ${PARSE_SOURCES}) + IF (PARSE_TESTONLYLIBS) + TARGET_LINK_LIBRARIES(${EXE_NAME} ${PARSE_TESTONLYLIBS}) + ENDIF() + VERIFY_EMPTY(KOKKOS_ADD_EXECUTABLE ${PARSE_UNPARSED_ARGUMENTS}) + endif() +ENDFUNCTION() + +IF(NOT TARGET check) + ADD_CUSTOM_TARGET(check COMMAND ${CMAKE_CTEST_COMMAND} -VV -C ${CMAKE_CFG_INTDIR}) +ENDIF() + + +FUNCTION(KOKKOS_ADD_EXECUTABLE_AND_TEST ROOT_NAME) +IF (KOKKOS_HAS_TRILINOS) + TRIBITS_ADD_EXECUTABLE_AND_TEST( + ${ROOT_NAME} + TESTONLYLIBS kokkos_gtest + ${ARGN} + NUM_MPI_PROCS 1 + COMM serial mpi + FAIL_REGULAR_EXPRESSION " FAILED " + ) +ELSE() + CMAKE_PARSE_ARGUMENTS(PARSE + "" + "" + "SOURCES;CATEGORIES" + ${ARGN}) + VERIFY_EMPTY(KOKKOS_ADD_EXECUTABLE_AND_TEST ${PARSE_UNPARSED_ARGUMENTS}) + SET(EXE_NAME ${PACKAGE_NAME}_${ROOT_NAME}) + KOKKOS_ADD_TEST_EXECUTABLE(${EXE_NAME} + SOURCES ${PARSE_SOURCES} + ) + KOKKOS_ADD_TEST(NAME ${ROOT_NAME} + EXE ${EXE_NAME} + FAIL_REGULAR_EXPRESSION " FAILED " + ) +ENDIF() +ENDFUNCTION() + +MACRO(KOKKOS_SETUP_BUILD_ENVIRONMENT) + INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_compiler_id.cmake) + INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_enable_devices.cmake) + INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_enable_options.cmake) + INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_test_cxx_std.cmake) + INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_arch.cmake) + IF (NOT KOKKOS_HAS_TRILINOS) + SET(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${Kokkos_SOURCE_DIR}/cmake/Modules/") + INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_tpls.cmake) + ENDIF() + INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_corner_cases.cmake) +ENDMACRO() + +MACRO(KOKKOS_ADD_TEST_EXECUTABLE EXE_NAME) + CMAKE_PARSE_ARGUMENTS(PARSE + "" + "" + "SOURCES" + ${ARGN}) + KOKKOS_ADD_EXECUTABLE(${EXE_NAME} + SOURCES ${PARSE_SOURCES} + ${PARSE_UNPARSED_ARGUMENTS} + TESTONLYLIBS kokkos_gtest + ) + IF (NOT KOKKOS_HAS_TRILINOS) + ADD_DEPENDENCIES(check ${EXE_NAME}) + ENDIF() +ENDMACRO() + +MACRO(KOKKOS_PACKAGE_POSTPROCESS) + if (KOKKOS_HAS_TRILINOS) + TRIBITS_PACKAGE_POSTPROCESS() + endif() +ENDMACRO() + +FUNCTION(KOKKOS_SET_LIBRARY_PROPERTIES LIBRARY_NAME) + CMAKE_PARSE_ARGUMENTS(PARSE + "PLAIN_STYLE" + "" + "" + ${ARGN}) + + IF(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.13") + #great, this works the "right" way + TARGET_LINK_OPTIONS( + ${LIBRARY_NAME} PUBLIC ${KOKKOS_LINK_OPTIONS} + ) + ELSE() + IF (PARSE_PLAIN_STYLE) + TARGET_LINK_LIBRARIES( + ${LIBRARY_NAME} ${KOKKOS_LINK_OPTIONS} + ) + ELSE() + #well, have to do it the wrong way for now + TARGET_LINK_LIBRARIES( + ${LIBRARY_NAME} PUBLIC ${KOKKOS_LINK_OPTIONS} + ) + ENDIF() + ENDIF() + + TARGET_COMPILE_OPTIONS( + ${LIBRARY_NAME} PUBLIC + $<$:${KOKKOS_COMPILE_OPTIONS}> + ) + + IF (KOKKOS_ENABLE_CUDA) + TARGET_COMPILE_OPTIONS( + ${LIBRARY_NAME} + PUBLIC $<$:${KOKKOS_CUDA_OPTIONS}> + ) + SET(NODEDUP_CUDAFE_OPTIONS) + FOREACH(OPT ${KOKKOS_CUDAFE_OPTIONS}) + LIST(APPEND NODEDUP_CUDAFE_OPTIONS -Xcudafe ${OPT}) + ENDFOREACH() + TARGET_COMPILE_OPTIONS( + ${LIBRARY_NAME} + PUBLIC $<$:${NODEDUP_CUDAFE_OPTIONS}> + ) + ENDIF() + + LIST(LENGTH KOKKOS_XCOMPILER_OPTIONS XOPT_LENGTH) + IF (XOPT_LENGTH GREATER 1) + MESSAGE(FATAL_ERROR "CMake deduplication does not allow multiple -Xcompiler flags (${KOKKOS_XCOMPILER_OPTIONS}): will require Kokkos to upgrade to minimum 3.12") + ENDIF() + IF(KOKKOS_XCOMPILER_OPTIONS) + SET(NODEDUP_XCOMPILER_OPTIONS) + FOREACH(OPT ${KOKKOS_XCOMPILER_OPTIONS}) + #I have to do this for now because we can't guarantee 3.12 support + #I really should do this with the shell option + LIST(APPEND NODEDUP_XCOMPILER_OPTIONS -Xcompiler) + LIST(APPEND NODEDUP_XCOMPILER_OPTIONS ${OPT}) + ENDFOREACH() + TARGET_COMPILE_OPTIONS( + ${LIBRARY_NAME} + PUBLIC $<$:${NODEDUP_XCOMPILER_OPTIONS}> + ) + ENDIF() + + IF (KOKKOS_CXX_STANDARD_FEATURE) + #GREAT! I can do this the right way + TARGET_COMPILE_FEATURES(${LIBRARY_NAME} PUBLIC ${KOKKOS_CXX_STANDARD_FEATURE}) + IF (NOT KOKKOS_USE_CXX_EXTENSIONS) + SET_TARGET_PROPERTIES(${LIBRARY_NAME} PROPERTIES CXX_EXTENSIONS OFF) + ENDIF() + ELSE() + #OH, well, no choice but the wrong way + TARGET_COMPILE_OPTIONS(${LIBRARY_NAME} PUBLIC ${KOKKOS_CXX_STANDARD_FLAG}) + ENDIF() +ENDFUNCTION() + +FUNCTION(KOKKOS_INTERNAL_ADD_LIBRARY LIBRARY_NAME) + CMAKE_PARSE_ARGUMENTS(PARSE + "STATIC;SHARED" + "" + "HEADERS;SOURCES" + ${ARGN}) + + IF(PARSE_HEADERS) + LIST(REMOVE_DUPLICATES PARSE_HEADERS) + ENDIF() + IF(PARSE_SOURCES) + LIST(REMOVE_DUPLICATES PARSE_SOURCES) + ENDIF() + + ADD_LIBRARY( + ${LIBRARY_NAME} + ${PARSE_HEADERS} + ${PARSE_SOURCES} + ) + + KOKKOS_INTERNAL_ADD_LIBRARY_INSTALL(${LIBRARY_NAME}) + + INSTALL( + FILES ${PARSE_HEADERS} + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + COMPONENT ${PACKAGE_NAME} + ) + + #In case we are building in-tree, add an alias name + #that matches the install Kokkos:: name + ADD_LIBRARY(Kokkos::${LIBRARY_NAME} ALIAS ${LIBRARY_NAME}) +ENDFUNCTION() + +FUNCTION(KOKKOS_ADD_LIBRARY LIBRARY_NAME) + IF (KOKKOS_HAS_TRILINOS) + TRIBITS_ADD_LIBRARY(${LIBRARY_NAME} ${ARGN}) + #Stolen from Tribits - it can add prefixes + SET(TRIBITS_LIBRARY_NAME_PREFIX "${${PROJECT_NAME}_LIBRARY_NAME_PREFIX}") + SET(TRIBITS_LIBRARY_NAME ${TRIBITS_LIBRARY_NAME_PREFIX}${LIBRARY_NAME}) + #Tribits has way too much techinical debt and baggage to even + #allow PUBLIC target_compile_options to be used. It forces C++ flags on projects + #as a giant blob of space-separated strings. We end up with duplicated + #flags between the flags implicitly forced on Kokkos-dependent and those Kokkos + #has in its public INTERFACE_COMPILE_OPTIONS. + #These do NOT get de-deduplicated because Tribits + #creates flags as a giant monolithic space-separated string + #Do not set any transitive properties and keep everything working as before + #KOKKOS_SET_LIBRARY_PROPERTIES(${TRIBITS_LIBRARY_NAME} PLAIN_STYLE) + ELSE() + KOKKOS_INTERNAL_ADD_LIBRARY( + ${LIBRARY_NAME} ${ARGN}) + KOKKOS_SET_LIBRARY_PROPERTIES(${LIBRARY_NAME}) + ENDIF() +ENDFUNCTION() + +FUNCTION(KOKKOS_ADD_INTERFACE_LIBRARY NAME) +IF (KOKKOS_HAS_TRILINOS) + TRIBITS_ADD_LIBRARY(${NAME} ${ARGN}) +ELSE() + CMAKE_PARSE_ARGUMENTS(PARSE + "" + "" + "HEADERS;SOURCES" + ${ARGN} + ) + + ADD_LIBRARY(${NAME} INTERFACE) + KOKKOS_INTERNAL_ADD_LIBRARY_INSTALL(${NAME}) + + INSTALL( + FILES ${PARSE_HEADERS} + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + ) + + INSTALL( + FILES ${PARSE_HEADERS} + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + COMPONENT ${PACKAGE_NAME} + ) +ENDIF() +ENDFUNCTION() + +FUNCTION(KOKKOS_LIB_INCLUDE_DIRECTORIES TARGET) + IF(KOKKOS_HAS_TRILINOS) + #ignore the target, tribits doesn't do anything directly with targets + TRIBITS_INCLUDE_DIRECTORIES(${ARGN}) + ELSE() #append to a list for later + KOKKOS_LIB_TYPE(${TARGET} INCTYPE) + FOREACH(DIR ${ARGN}) + TARGET_INCLUDE_DIRECTORIES(${TARGET} ${INCTYPE} $) + ENDFOREACH() + ENDIF() +ENDFUNCTION() + +FUNCTION(KOKKOS_LIB_COMPILE_OPTIONS TARGET) + IF(KOKKOS_HAS_TRILINOS) + #don't trust tribits to do this correctly + KOKKOS_TARGET_COMPILE_OPTIONS(${TARGET} ${ARGN}) + ELSE() + KOKKOS_LIB_TYPE(${TARGET} INCTYPE) + KOKKOS_TARGET_COMPILE_OPTIONS(${${PROJECT_NAME}_LIBRARY_NAME_PREFIX}${TARGET} ${INCTYPE} ${ARGN}) + ENDIF() +ENDFUNCTION() + +MACRO(KOKKOS_ADD_TEST_DIRECTORIES) + IF (KOKKOS_HAS_TRILINOS) + TRIBITS_ADD_TEST_DIRECTORIES(${ARGN}) + ELSE() + IF(KOKKOS_ENABLE_TESTS) + FOREACH(TEST_DIR ${ARGN}) + ADD_SUBDIRECTORY(${TEST_DIR}) + ENDFOREACH() + ENDIF() + ENDIF() +ENDMACRO() diff --git a/lib/kokkos/cmake/pgi.cmake b/lib/kokkos/cmake/pgi.cmake new file mode 100644 index 0000000000000000000000000000000000000000..e98e84955888496225e9268c7db47ef514f08a48 --- /dev/null +++ b/lib/kokkos/cmake/pgi.cmake @@ -0,0 +1,8 @@ + +function(kokkos_set_pgi_flags full_standard int_standard) + STRING(TOLOWER ${full_standard} FULL_LC_STANDARD) + STRING(TOLOWER ${int_standard} INT_LC_STANDARD) + SET(KOKKOS_CXX_STANDARD_FLAG "--c++${FULL_LC_STANDARD}" PARENT_SCOPE) + SET(KOKKOS_CXX_INTERMDIATE_STANDARD_FLAG "--c++${INT_LC_STANDARD}" PARENT_SCOPE) +endfunction() + diff --git a/lib/kokkos/cmake/tpls/FindTPLCUSPARSE.cmake b/lib/kokkos/cmake/tpls/FindTPLCUSPARSE.cmake index aad1e2bad7629f3f43ca91135752253a20ac9523..b8cee04804cd6cbfdb475b2631c1b1beddfa7b41 100644 --- a/lib/kokkos/cmake/tpls/FindTPLCUSPARSE.cmake +++ b/lib/kokkos/cmake/tpls/FindTPLCUSPARSE.cmake @@ -67,7 +67,7 @@ ELSE() IF(CUDA_cusparse_LIBRARY STREQUAL "CUDA_cusparse_LIBRARY-NOTFOUND") MESSAGE(FATAL_ERROR "\nCUSPARSE: could not find cuspasre library.") ENDIF() - ENDIF(CMAKE_VERSION VERSION_LESS "2.8.8") + ENDIF() GLOBAL_SET(TPL_CUSPARSE_LIBRARY_DIRS) GLOBAL_SET(TPL_CUSPARSE_INCLUDE_DIRS ${TPL_CUDA_INCLUDE_DIRS}) GLOBAL_SET(TPL_CUSPARSE_LIBRARIES ${CUDA_cusparse_LIBRARY}) diff --git a/lib/kokkos/cmake/tpls/FindTPLHWLOC.cmake b/lib/kokkos/cmake/tpls/FindTPLHWLOC.cmake index 715b3e9bde59379c632fbec7926b425e6189e74d..a4c55e1d7b695697b458a4c00e5cb5a72a531353 100644 --- a/lib/kokkos/cmake/tpls/FindTPLHWLOC.cmake +++ b/lib/kokkos/cmake/tpls/FindTPLHWLOC.cmake @@ -64,7 +64,7 @@ # Version: 1.3 # -TRIBITS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( HWLOC +KOKKOS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( HWLOC REQUIRED_HEADERS hwloc.h REQUIRED_LIBS_NAMES "hwloc" ) diff --git a/lib/kokkos/cmake/tpls/FindTPLPthread.cmake b/lib/kokkos/cmake/tpls/FindTPLPthread.cmake index fc401d7543357f18d225a33efe0cf3bb489170d7..4dc1a87e18681d1c84b0706b48c8d09731b21732 100644 --- a/lib/kokkos/cmake/tpls/FindTPLPthread.cmake +++ b/lib/kokkos/cmake/tpls/FindTPLPthread.cmake @@ -75,7 +75,7 @@ IF(USE_THREADS) SET(TPL_Pthread_LIBRARIES "${CMAKE_THREAD_LIBS_INIT}") SET(TPL_Pthread_LIBRARY_DIRS "") ELSE() - TRIBITS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( Pthread + KOKKOS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( Pthread REQUIRED_HEADERS pthread.h REQUIRED_LIBS_NAMES pthread ) diff --git a/lib/kokkos/cmake/tpls/FindTPLQTHREADS.cmake b/lib/kokkos/cmake/tpls/FindTPLQTHREADS.cmake deleted file mode 100644 index c312f2590bcd29197a0cf3fbd5e0b484579a09c2..0000000000000000000000000000000000000000 --- a/lib/kokkos/cmake/tpls/FindTPLQTHREADS.cmake +++ /dev/null @@ -1,69 +0,0 @@ -# @HEADER -# ************************************************************************ -# -# Trilinos: An Object-Oriented Solver Framework -# Copyright (2001) Sandia Corporation -# -# -# Copyright (2001) Sandia Corporation. Under the terms of Contract -# DE-AC04-94AL85000, there is a non-exclusive license for use of this -# work by or on behalf of the U.S. Government. Export of this program -# may require a license from the United States Government. -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# NOTICE: The United States Government is granted for itself and others -# acting on its behalf a paid-up, nonexclusive, irrevocable worldwide -# license in this data to reproduce, prepare derivative works, and -# perform publicly and display publicly. Beginning five (5) years from -# July 25, 2001, the United States Government is granted for itself and -# others acting on its behalf a paid-up, nonexclusive, irrevocable -# worldwide license in this data to reproduce, prepare derivative works, -# distribute copies to the public, perform publicly and display -# publicly, and to permit others to do so. -# -# NEITHER THE UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT -# OF ENERGY, NOR SANDIA CORPORATION, NOR ANY OF THEIR EMPLOYEES, MAKES -# ANY WARRANTY, EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR -# RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY -# INFORMATION, APPARATUS, PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS -# THAT ITS USE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS. -# -# ************************************************************************ -# @HEADER - - -#----------------------------------------------------------------------------- -# Hardware locality detection and control library. -# -# Acquisition information: -# Date checked: July 2014 -# Checked by: H. Carter Edwards -# Source: https://code.google.com/p/qthreads -# - -TRIBITS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( QTHREADS - REQUIRED_HEADERS qthread.h - REQUIRED_LIBS_NAMES "qthread" - ) diff --git a/lib/kokkos/cmake/tribits.cmake b/lib/kokkos/cmake/tribits.cmake deleted file mode 100644 index 1f467f0662e0c2154a53a0c5fcb46aea8be5de43..0000000000000000000000000000000000000000 --- a/lib/kokkos/cmake/tribits.cmake +++ /dev/null @@ -1,531 +0,0 @@ -INCLUDE(CMakeParseArguments) -INCLUDE(CTest) - -cmake_policy(SET CMP0054 NEW) - -MESSAGE(STATUS "The project name is: ${PROJECT_NAME}") - -IF(NOT DEFINED ${PROJECT_NAME}_ENABLE_OpenMP) - SET(${PROJECT_NAME}_ENABLE_OpenMP OFF) -ENDIF() - -IF(NOT DEFINED ${PROJECT_NAME}_ENABLE_HPX) - SET(${PROJECT_NAME}_ENABLE_HPX OFF) -ENDIF() - -IF(NOT DEFINED ${PROJECT_NAME}_ENABLE_DEBUG) - SET(${PROJECT_NAME}_ENABLE_DEBUG OFF) -ENDIF() - -IF(NOT DEFINED ${PROJECT_NAME}_ENABLE_CXX11) - SET(${PROJECT_NAME}_ENABLE_CXX11 ON) -ENDIF() - -IF(NOT DEFINED ${PROJECT_NAME}_ENABLE_TESTS) - SET(${PROJECT_NAME}_ENABLE_TESTS OFF) -ENDIF() - -IF(NOT DEFINED TPL_ENABLE_Pthread) - SET(TPL_ENABLE_Pthread OFF) -ENDIF() - -FUNCTION(ASSERT_DEFINED VARS) - FOREACH(VAR ${VARS}) - IF(NOT DEFINED ${VAR}) - MESSAGE(SEND_ERROR "Error, the variable ${VAR} is not defined!") - ENDIF() - ENDFOREACH() -ENDFUNCTION() - -MACRO(GLOBAL_SET VARNAME) - SET(${VARNAME} ${ARGN} CACHE INTERNAL "") -ENDMACRO() - -MACRO(PREPEND_GLOBAL_SET VARNAME) - ASSERT_DEFINED(${VARNAME}) - GLOBAL_SET(${VARNAME} ${ARGN} ${${VARNAME}}) -ENDMACRO() - -#FUNCTION(REMOVE_GLOBAL_DUPLICATES VARNAME) -# ASSERT_DEFINED(${VARNAME}) -# IF (${VARNAME}) -# SET(TMP ${${VARNAME}}) -# LIST(REMOVE_DUPLICATES TMP) -# GLOBAL_SET(${VARNAME} ${TMP}) -# ENDIF() -#ENDFUNCTION() - -#MACRO(TRIBITS_ADD_OPTION_AND_DEFINE USER_OPTION_NAME MACRO_DEFINE_NAME DOCSTRING DEFAULT_VALUE) -# MESSAGE(STATUS "TRIBITS_ADD_OPTION_AND_DEFINE: '${USER_OPTION_NAME}' '${MACRO_DEFINE_NAME}' '${DEFAULT_VALUE}'") -# SET( ${USER_OPTION_NAME} "${DEFAULT_VALUE}" CACHE BOOL "${DOCSTRING}" ) -# IF(NOT ${MACRO_DEFINE_NAME} STREQUAL "") -# IF(${USER_OPTION_NAME}) -# GLOBAL_SET(${MACRO_DEFINE_NAME} ON) -# ELSE() -# GLOBAL_SET(${MACRO_DEFINE_NAME} OFF) -# ENDIF() -# ENDIF() -#ENDMACRO() - -FUNCTION(TRIBITS_CONFIGURE_FILE PACKAGE_NAME_CONFIG_FILE) - - # Configure the file - CONFIGURE_FILE( - ${PACKAGE_SOURCE_DIR}/cmake/${PACKAGE_NAME_CONFIG_FILE}.in - ${CMAKE_CURRENT_BINARY_DIR}/${PACKAGE_NAME_CONFIG_FILE} - ) - -ENDFUNCTION() - -#MACRO(TRIBITS_ADD_DEBUG_OPTION) -# TRIBITS_ADD_OPTION_AND_DEFINE( -# ${PROJECT_NAME}_ENABLE_DEBUG -# HAVE_${PROJECT_NAME_UC}_DEBUG -# "Enable a host of runtime debug checking." -# OFF -# ) -#ENDMACRO() - - -MACRO(TRIBITS_ADD_TEST_DIRECTORIES) - IF(${${PROJECT_NAME}_ENABLE_TESTS}) - FOREACH(TEST_DIR ${ARGN}) - ADD_SUBDIRECTORY(${TEST_DIR}) - ENDFOREACH() - ENDIF() -ENDMACRO() - -MACRO(TRIBITS_ADD_EXAMPLE_DIRECTORIES) - IF(${PACKAGE_NAME}_ENABLE_EXAMPLES OR ${PARENT_PACKAGE_NAME}_ENABLE_EXAMPLES) - FOREACH(EXAMPLE_DIR ${ARGN}) - ADD_SUBDIRECTORY(${EXAMPLE_DIR}) - ENDFOREACH() - ENDIF() -ENDMACRO() - - -function(INCLUDE_DIRECTORIES) - cmake_parse_arguments(INCLUDE_DIRECTORIES "REQUIRED_DURING_INSTALLATION_TESTING" "" "" ${ARGN}) - _INCLUDE_DIRECTORIES(${INCLUDE_DIRECTORIES_UNPARSED_ARGUMENTS}) -endfunction() - - -MACRO(TARGET_TRANSFER_PROPERTY TARGET_NAME PROP_IN PROP_OUT) - SET(PROP_VALUES) - FOREACH(TARGET_X ${ARGN}) - LIST(APPEND PROP_VALUES "$") - ENDFOREACH() - SET_TARGET_PROPERTIES(${TARGET_NAME} PROPERTIES ${PROP_OUT} "${PROP_VALUES}") -ENDMACRO() - -MACRO(ADD_INTERFACE_LIBRARY LIB_NAME) - FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp "") - ADD_LIBRARY(${LIB_NAME} STATIC ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp) - SET_TARGET_PROPERTIES(${LIB_NAME} PROPERTIES INTERFACE TRUE) -ENDMACRO() - -# Older versions of cmake does not make include directories transitive -MACRO(TARGET_LINK_AND_INCLUDE_LIBRARIES TARGET_NAME) - TARGET_LINK_LIBRARIES(${TARGET_NAME} LINK_PUBLIC ${ARGN}) - FOREACH(DEP_LIB ${ARGN}) - TARGET_INCLUDE_DIRECTORIES(${TARGET_NAME} PUBLIC $) - TARGET_INCLUDE_DIRECTORIES(${TARGET_NAME} PUBLIC $) - ENDFOREACH() -ENDMACRO() - -FUNCTION(TRIBITS_ADD_LIBRARY LIBRARY_NAME) - - SET(options STATIC SHARED TESTONLY NO_INSTALL_LIB_OR_HEADERS CUDALIBRARY) - SET(oneValueArgs) - SET(multiValueArgs HEADERS HEADERS_INSTALL_SUBDIR NOINSTALLHEADERS SOURCES DEPLIBS IMPORTEDLIBS DEFINES ADDED_LIB_TARGET_NAME_OUT) - - CMAKE_PARSE_ARGUMENTS(PARSE "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - - IF(PARSE_HEADERS) - LIST(REMOVE_DUPLICATES PARSE_HEADERS) - ENDIF() - IF(PARSE_SOURCES) - LIST(REMOVE_DUPLICATES PARSE_SOURCES) - ENDIF() - - # Local variable to hold all of the libraries that will be directly linked - # to this library. - SET(LINK_LIBS ${${PACKAGE_NAME}_DEPS}) - - # Add dependent libraries passed directly in - - IF (PARSE_IMPORTEDLIBS) - LIST(APPEND LINK_LIBS ${PARSE_IMPORTEDLIBS}) - ENDIF() - - IF (PARSE_DEPLIBS) - LIST(APPEND LINK_LIBS ${PARSE_DEPLIBS}) - ENDIF() - - # Add the library and all the dependencies - - IF (PARSE_DEFINES) - ADD_DEFINITIONS(${PARSE_DEFINES}) - ENDIF() - - IF (PARSE_STATIC) - SET(STATIC_KEYWORD "STATIC") - ELSE() - SET(STATIC_KEYWORD) - ENDIF() - - IF (PARSE_SHARED) - SET(SHARED_KEYWORD "SHARED") - ELSE() - SET(SHARED_KEYWORD) - ENDIF() - - IF (PARSE_TESTONLY) - SET(EXCLUDE_FROM_ALL_KEYWORD "EXCLUDE_FROM_ALL") - ELSE() - SET(EXCLUDE_FROM_ALL_KEYWORD) - ENDIF() - IF (NOT PARSE_CUDALIBRARY) - ADD_LIBRARY( - ${LIBRARY_NAME} - ${STATIC_KEYWORD} - ${SHARED_KEYWORD} - ${EXCLUDE_FROM_ALL_KEYWORD} - ${PARSE_HEADERS} - ${PARSE_NOINSTALLHEADERS} - ${PARSE_SOURCES} - ) - ELSE() - CUDA_ADD_LIBRARY( - ${LIBRARY_NAME} - ${PARSE_HEADERS} - ${PARSE_NOINSTALLHEADERS} - ${PARSE_SOURCES} - ) - ENDIF() - - TARGET_LINK_AND_INCLUDE_LIBRARIES(${LIBRARY_NAME} ${LINK_LIBS}) - - IF (NOT PARSE_TESTONLY OR PARSE_NO_INSTALL_LIB_OR_HEADERS) - - INSTALL( - TARGETS ${LIBRARY_NAME} - EXPORT ${PROJECT_NAME} - RUNTIME DESTINATION bin - LIBRARY DESTINATION lib - ARCHIVE DESTINATION lib - COMPONENT ${PACKAGE_NAME} - ) - - INSTALL( - FILES ${PARSE_HEADERS} - EXPORT ${PROJECT_NAME} - DESTINATION include - COMPONENT ${PACKAGE_NAME} - ) - - INSTALL( - DIRECTORY ${PARSE_HEADERS_INSTALL_SUBDIR} - EXPORT ${PROJECT_NAME} - DESTINATION include - COMPONENT ${PACKAGE_NAME} - ) - - ENDIF() - - IF (NOT PARSE_TESTONLY) - PREPEND_GLOBAL_SET(${PACKAGE_NAME}_LIBS ${LIBRARY_NAME}) - REMOVE_GLOBAL_DUPLICATES(${PACKAGE_NAME}_LIBS) - ENDIF() - -ENDFUNCTION() - -FUNCTION(TRIBITS_ADD_EXECUTABLE EXE_NAME) - - SET(options NOEXEPREFIX NOEXESUFFIX ADD_DIR_TO_NAME INSTALLABLE TESTONLY) - SET(oneValueArgs ADDED_EXE_TARGET_NAME_OUT) - SET(multiValueArgs SOURCES CATEGORIES HOST XHOST HOSTTYPE XHOSTTYPE DIRECTORY TESTONLYLIBS IMPORTEDLIBS DEPLIBS COMM LINKER_LANGUAGE TARGET_DEFINES DEFINES) - - CMAKE_PARSE_ARGUMENTS(PARSE "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - - IF (PARSE_TARGET_DEFINES) - TARGET_COMPILE_DEFINITIONS(${EXE_NAME} PUBLIC ${PARSE_TARGET_DEFINES}) - ENDIF() - - SET(LINK_LIBS PACKAGE_${PACKAGE_NAME}) - - IF (PARSE_TESTONLYLIBS) - LIST(APPEND LINK_LIBS ${PARSE_TESTONLYLIBS}) - ENDIF() - - IF (PARSE_IMPORTEDLIBS) - LIST(APPEND LINK_LIBS ${PARSE_IMPORTEDLIBS}) - ENDIF() - - SET (EXE_SOURCES) - IF(PARSE_DIRECTORY) - FOREACH( SOURCE_FILE ${PARSE_SOURCES} ) - IF(IS_ABSOLUTE ${SOURCE_FILE}) - SET (EXE_SOURCES ${EXE_SOURCES} ${SOURCE_FILE}) - ELSE() - SET (EXE_SOURCES ${EXE_SOURCES} ${PARSE_DIRECTORY}/${SOURCE_FILE}) - ENDIF() - ENDFOREACH( ) - ELSE() - FOREACH( SOURCE_FILE ${PARSE_SOURCES} ) - SET (EXE_SOURCES ${EXE_SOURCES} ${SOURCE_FILE}) - ENDFOREACH( ) - ENDIF() - - SET(EXE_BINARY_NAME ${EXE_NAME}) - IF(DEFINED PACKAGE_NAME AND NOT PARSE_NOEXEPREFIX) - SET(EXE_BINARY_NAME ${PACKAGE_NAME}_${EXE_BINARY_NAME}) - ENDIF() - - # IF (PARSE_TESTONLY) - # SET(EXCLUDE_FROM_ALL_KEYWORD "EXCLUDE_FROM_ALL") - # ELSE() - # SET(EXCLUDE_FROM_ALL_KEYWORD) - # ENDIF() - ADD_EXECUTABLE(${EXE_BINARY_NAME} ${EXCLUDE_FROM_ALL_KEYWORD} ${EXE_SOURCES}) - - TARGET_LINK_AND_INCLUDE_LIBRARIES(${EXE_BINARY_NAME} ${LINK_LIBS}) - - IF(PARSE_ADDED_EXE_TARGET_NAME_OUT) - SET(${PARSE_ADDED_EXE_TARGET_NAME_OUT} ${EXE_BINARY_NAME} PARENT_SCOPE) - ENDIF() - - IF(PARSE_INSTALLABLE) - INSTALL( - TARGETS ${EXE_BINARY_NAME} - EXPORT ${PROJECT_NAME} - DESTINATION bin - ) - ENDIF() -ENDFUNCTION() - -IF(NOT TARGET check) - ADD_CUSTOM_TARGET(check COMMAND ${CMAKE_CTEST_COMMAND} -VV -C ${CMAKE_CFG_INTDIR}) -ENDIF() - -FUNCTION(TRIBITS_ADD_TEST) -ENDFUNCTION() -FUNCTION(TRIBITS_TPL_TENTATIVELY_ENABLE) -ENDFUNCTION() - -FUNCTION(TRIBITS_ADD_ADVANCED_TEST) - # TODO Write this -ENDFUNCTION() - -FUNCTION(TRIBITS_ADD_EXECUTABLE_AND_TEST EXE_NAME) - - SET(options STANDARD_PASS_OUTPUT WILL_FAIL) - SET(oneValueArgs PASS_REGULAR_EXPRESSION FAIL_REGULAR_EXPRESSION ENVIRONMENT TIMEOUT CATEGORIES ADDED_TESTS_NAMES_OUT ADDED_EXE_TARGET_NAME_OUT) - SET(multiValueArgs) - - CMAKE_PARSE_ARGUMENTS(PARSE "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - - TRIBITS_ADD_EXECUTABLE(${EXE_NAME} TESTONLY ADDED_EXE_TARGET_NAME_OUT TEST_NAME ${PARSE_UNPARSED_ARGUMENTS}) - - IF(WIN32) - ADD_TEST(NAME ${TEST_NAME} WORKING_DIRECTORY ${LIBRARY_OUTPUT_PATH} COMMAND ${TEST_NAME}${CMAKE_EXECUTABLE_SUFFIX}) - ELSE() - ADD_TEST(NAME ${TEST_NAME} COMMAND ${TEST_NAME}) - ENDIF() - ADD_DEPENDENCIES(check ${TEST_NAME}) - - IF(PARSE_FAIL_REGULAR_EXPRESSION) - SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES FAIL_REGULAR_EXPRESSION ${PARSE_FAIL_REGULAR_EXPRESSION}) - ENDIF() - - IF(PARSE_PASS_REGULAR_EXPRESSION) - SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES PASS_REGULAR_EXPRESSION ${PARSE_PASS_REGULAR_EXPRESSION}) - ENDIF() - - IF(PARSE_WILL_FAIL) - SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES WILL_FAIL ${PARSE_WILL_FAIL}) - ENDIF() - - IF(PARSE_ADDED_TESTS_NAMES_OUT) - SET(${PARSE_ADDED_TESTS_NAMES_OUT} ${TEST_NAME} PARENT_SCOPE) - ENDIF() - - IF(PARSE_ADDED_EXE_TARGET_NAME_OUT) - SET(${PARSE_ADDED_EXE_TARGET_NAME_OUT} ${TEST_NAME} PARENT_SCOPE) - ENDIF() - -ENDFUNCTION() - -MACRO(TIBITS_CREATE_IMPORTED_TPL_LIBRARY TPL_NAME) - ADD_INTERFACE_LIBRARY(TPL_LIB_${TPL_NAME}) - TARGET_LINK_LIBRARIES(TPL_LIB_${TPL_NAME} LINK_PUBLIC ${TPL_${TPL_NAME}_LIBRARIES}) - TARGET_INCLUDE_DIRECTORIES(TPL_LIB_${TPL_NAME} INTERFACE ${TPL_${TPL_NAME}_INCLUDE_DIRS}) -ENDMACRO() - -FUNCTION(TRIBITS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES TPL_NAME) - - SET(options MUST_FIND_ALL_LIBS MUST_FIND_ALL_HEADERS NO_PRINT_ENABLE_SUCCESS_FAIL) - SET(oneValueArgs) - SET(multiValueArgs REQUIRED_HEADERS REQUIRED_LIBS_NAMES) - - CMAKE_PARSE_ARGUMENTS(PARSE "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - - SET(_${TPL_NAME}_ENABLE_SUCCESS TRUE) - IF (PARSE_REQUIRED_LIBS_NAMES) - FIND_LIBRARY(TPL_${TPL_NAME}_LIBRARIES NAMES ${PARSE_REQUIRED_LIBS_NAMES}) - IF(NOT TPL_${TPL_NAME}_LIBRARIES) - SET(_${TPL_NAME}_ENABLE_SUCCESS FALSE) - ENDIF() - ENDIF() - IF (PARSE_REQUIRED_HEADERS) - FIND_PATH(TPL_${TPL_NAME}_INCLUDE_DIRS NAMES ${PARSE_REQUIRED_HEADERS}) - IF(NOT TPL_${TPL_NAME}_INCLUDE_DIRS) - SET(_${TPL_NAME}_ENABLE_SUCCESS FALSE) - ENDIF() - ENDIF() - - - IF (_${TPL_NAME}_ENABLE_SUCCESS) - TIBITS_CREATE_IMPORTED_TPL_LIBRARY(${TPL_NAME}) - ENDIF() - -ENDFUNCTION() - -#MACRO(TRIBITS_PROCESS_TPL_DEP_FILE TPL_FILE) -# GET_FILENAME_COMPONENT(TPL_NAME ${TPL_FILE} NAME_WE) -# INCLUDE("${TPL_FILE}") -# IF(TARGET TPL_LIB_${TPL_NAME}) -# MESSAGE(STATUS "Found tpl library: ${TPL_NAME}") -# SET(TPL_ENABLE_${TPL_NAME} TRUE) -# ELSE() -# MESSAGE(STATUS "Tpl library not found: ${TPL_NAME}") -# SET(TPL_ENABLE_${TPL_NAME} FALSE) -# ENDIF() -#ENDMACRO() - -MACRO(PREPEND_TARGET_SET VARNAME TARGET_NAME TYPE) - IF(TYPE STREQUAL "REQUIRED") - SET(REQUIRED TRUE) - ELSE() - SET(REQUIRED FALSE) - ENDIF() - IF(TARGET ${TARGET_NAME}) - PREPEND_GLOBAL_SET(${VARNAME} ${TARGET_NAME}) - ELSE() - IF(REQUIRED) - MESSAGE(FATAL_ERROR "Missing dependency ${TARGET_NAME}") - ENDIF() - ENDIF() -ENDMACRO() - -MACRO(TRIBITS_APPEND_PACKAGE_DEPS DEP_LIST TYPE) - FOREACH(DEP ${ARGN}) - PREPEND_GLOBAL_SET(${DEP_LIST} PACKAGE_${DEP}) - ENDFOREACH() -ENDMACRO() - -MACRO(TRIBITS_APPEND_TPLS_DEPS DEP_LIST TYPE) - FOREACH(DEP ${ARGN}) - PREPEND_TARGET_SET(${DEP_LIST} TPL_LIB_${DEP} ${TYPE}) - ENDFOREACH() -ENDMACRO() - -MACRO(TRIBITS_ENABLE_TPLS) - FOREACH(TPL ${ARGN}) - IF(TARGET ${TPL}) - GLOBAL_SET(${PACKAGE_NAME}_ENABLE_${TPL} TRUE) - ELSE() - GLOBAL_SET(${PACKAGE_NAME}_ENABLE_${TPL} FALSE) - ENDIF() - ENDFOREACH() -ENDMACRO() - -MACRO(TRIBITS_PACKAGE_DEFINE_DEPENDENCIES) - - SET(options) - SET(oneValueArgs) - SET(multiValueArgs - LIB_REQUIRED_PACKAGES - LIB_OPTIONAL_PACKAGES - TEST_REQUIRED_PACKAGES - TEST_OPTIONAL_PACKAGES - LIB_REQUIRED_TPLS - LIB_OPTIONAL_TPLS - TEST_REQUIRED_TPLS - TEST_OPTIONAL_TPLS - REGRESSION_EMAIL_LIST - SUBPACKAGES_DIRS_CLASSIFICATIONS_OPTREQS - ) - CMAKE_PARSE_ARGUMENTS(PARSE "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - - GLOBAL_SET(${PACKAGE_NAME}_DEPS "") - TRIBITS_APPEND_PACKAGE_DEPS(${PACKAGE_NAME}_DEPS REQUIRED ${PARSE_LIB_REQUIRED_PACKAGES}) - TRIBITS_APPEND_PACKAGE_DEPS(${PACKAGE_NAME}_DEPS OPTIONAL ${PARSE_LIB_OPTIONAL_PACKAGES}) - TRIBITS_APPEND_TPLS_DEPS(${PACKAGE_NAME}_DEPS REQUIRED ${PARSE_LIB_REQUIRED_TPLS}) - TRIBITS_APPEND_TPLS_DEPS(${PACKAGE_NAME}_DEPS OPTIONAL ${PARSE_LIB_OPTIONAL_TPLS}) - - GLOBAL_SET(${PACKAGE_NAME}_TEST_DEPS "") - TRIBITS_APPEND_PACKAGE_DEPS(${PACKAGE_NAME}_TEST_DEPS REQUIRED ${PARSE_TEST_REQUIRED_PACKAGES}) - TRIBITS_APPEND_PACKAGE_DEPS(${PACKAGE_NAME}_TEST_DEPS OPTIONAL ${PARSE_TEST_OPTIONAL_PACKAGES}) - TRIBITS_APPEND_TPLS_DEPS(${PACKAGE_NAME}_TEST_DEPS REQUIRED ${PARSE_TEST_REQUIRED_TPLS}) - TRIBITS_APPEND_TPLS_DEPS(${PACKAGE_NAME}_TEST_DEPS OPTIONAL ${PARSE_TEST_OPTIONAL_TPLS}) - - TRIBITS_ENABLE_TPLS(${PARSE_LIB_REQUIRED_TPLS} ${PARSE_LIB_OPTIONAL_TPLS} ${PARSE_TEST_REQUIRED_TPLS} ${PARSE_TEST_OPTIONAL_TPLS}) - -ENDMACRO() - -MACRO(TRIBITS_SUBPACKAGE NAME) - SET(PACKAGE_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) - SET(PARENT_PACKAGE_NAME ${PACKAGE_NAME}) - SET(PACKAGE_NAME ${PACKAGE_NAME}${NAME}) - STRING(TOUPPER ${PACKAGE_NAME} PACKAGE_NAME_UC) - SET(${PACKAGE_NAME}_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) - - ADD_INTERFACE_LIBRARY(PACKAGE_${PACKAGE_NAME}) - - GLOBAL_SET(${PACKAGE_NAME}_LIBS "") - - INCLUDE(${PACKAGE_SOURCE_DIR}/cmake/Dependencies.cmake) - -ENDMACRO(TRIBITS_SUBPACKAGE) - -MACRO(TRIBITS_SUBPACKAGE_POSTPROCESS) - TARGET_LINK_AND_INCLUDE_LIBRARIES(PACKAGE_${PACKAGE_NAME} ${${PACKAGE_NAME}_LIBS}) -ENDMACRO(TRIBITS_SUBPACKAGE_POSTPROCESS) - -MACRO(TRIBITS_PACKAGE_DECL NAME) - - SET(PACKAGE_NAME ${NAME}) - SET(${PACKAGE_NAME}_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) - STRING(TOUPPER ${PACKAGE_NAME} PACKAGE_NAME_UC) - - #SET(TRIBITS_DEPS_DIR "${CMAKE_SOURCE_DIR}/cmake/deps") - #FILE(GLOB TPLS_FILES "${TRIBITS_DEPS_DIR}/*.cmake") - #FOREACH(TPL_FILE ${TPLS_FILES}) - # TRIBITS_PROCESS_TPL_DEP_FILE(${TPL_FILE}) - #ENDFOREACH() - -ENDMACRO() - - -MACRO(TRIBITS_PROCESS_SUBPACKAGES) - FILE(GLOB SUBPACKAGES RELATIVE ${CMAKE_SOURCE_DIR} */cmake/Dependencies.cmake) - FOREACH(SUBPACKAGE ${SUBPACKAGES}) - GET_FILENAME_COMPONENT(SUBPACKAGE_CMAKE ${SUBPACKAGE} DIRECTORY) - GET_FILENAME_COMPONENT(SUBPACKAGE_DIR ${SUBPACKAGE_CMAKE} DIRECTORY) - ADD_SUBDIRECTORY(${CMAKE_BINARY_DIR}/../${SUBPACKAGE_DIR}) - ENDFOREACH() -ENDMACRO(TRIBITS_PROCESS_SUBPACKAGES) - -MACRO(TRIBITS_PACKAGE_DEF) -ENDMACRO(TRIBITS_PACKAGE_DEF) - -MACRO(TRIBITS_EXCLUDE_AUTOTOOLS_FILES) -ENDMACRO(TRIBITS_EXCLUDE_AUTOTOOLS_FILES) - -MACRO(TRIBITS_EXCLUDE_FILES) -ENDMACRO(TRIBITS_EXCLUDE_FILES) - -MACRO(TRIBITS_PACKAGE_POSTPROCESS) -ENDMACRO(TRIBITS_PACKAGE_POSTPROCESS) - diff --git a/lib/kokkos/containers/CMakeLists.txt b/lib/kokkos/containers/CMakeLists.txt index c37aa3e3e21e22ef1a1e8160d885d91884d9a62d..2bfaea7a13ebca4d3f971afe36b8935ebed21e13 100644 --- a/lib/kokkos/containers/CMakeLists.txt +++ b/lib/kokkos/containers/CMakeLists.txt @@ -1,13 +1,10 @@ - - -TRIBITS_SUBPACKAGE(Containers) - - -IF(KOKKOS_HAS_TRILINOS) - ADD_SUBDIRECTORY(src) -ENDIF() - -TRIBITS_ADD_TEST_DIRECTORIES(unit_tests) -TRIBITS_ADD_TEST_DIRECTORIES(performance_tests) - -TRIBITS_SUBPACKAGE_POSTPROCESS() + + +KOKKOS_SUBPACKAGE(Containers) + +ADD_SUBDIRECTORY(src) + +KOKKOS_ADD_TEST_DIRECTORIES(unit_tests) +KOKKOS_ADD_TEST_DIRECTORIES(performance_tests) + +KOKKOS_SUBPACKAGE_POSTPROCESS() diff --git a/lib/kokkos/containers/performance_tests/CMakeLists.txt b/lib/kokkos/containers/performance_tests/CMakeLists.txt index 3c6584bc346b9e48a245e39038a99396a10c1a14..ca76808190d43be58fccd9bceb58b9faaa7fc167 100644 --- a/lib/kokkos/containers/performance_tests/CMakeLists.txt +++ b/lib/kokkos/containers/performance_tests/CMakeLists.txt @@ -1,49 +1,62 @@ -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -INCLUDE_DIRECTORIES(REQUIRED_DURING_INSTALLATION_TESTING ${CMAKE_CURRENT_SOURCE_DIR}) -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../src ) - -IF(NOT KOKKOS_HAS_TRILINOS) - IF(KOKKOS_SEPARATE_LIBS) - set(TEST_LINK_TARGETS kokkoscore) - ELSE() - set(TEST_LINK_TARGETS kokkos) - ENDIF() +KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) +KOKKOS_INCLUDE_DIRECTORIES(REQUIRED_DURING_INSTALLATION_TESTING ${CMAKE_CURRENT_SOURCE_DIR}) +KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../src ) + +IF(Kokkos_ENABLE_CUDA) + SET(SOURCES + TestMain.cpp + TestCuda.cpp + ) + + KOKKOS_ADD_TEST_EXECUTABLE( PerfTestExec_Cuda + SOURCES ${SOURCES} + ) + + KOKKOS_ADD_TEST( NAME PerformanceTest_Cuda + EXE PerfTestExec_Cuda + ) ENDIF() -SET(SOURCES - TestMain.cpp - TestCuda.cpp - ) - -IF(Kokkos_ENABLE_Pthread) - LIST( APPEND SOURCES TestThreads.cpp) +IF(Kokkos_ENABLE_PTHREAD) + SET(SOURCES + TestMain.cpp + TestThreads.cpp + ) + KOKKOS_ADD_TEST_EXECUTABLE( PerfTestExec_Threads + SOURCES ${SOURCES} + ) + + KOKKOS_ADD_TEST( NAME PerformanceTest_Threads + EXE PerfTestExec_Threads + ) ENDIF() -IF(Kokkos_ENABLE_OpenMP) - LIST( APPEND SOURCES TestOpenMP.cpp) +IF(Kokkos_ENABLE_OPENMP) + SET(SOURCES + TestMain.cpp + TestOpenMP.cpp + ) + KOKKOS_ADD_TEST_EXECUTABLE( PerfTestExec_OpenMP + SOURCES ${SOURCES} + ) + + KOKKOS_ADD_TEST( NAME PerformanceTest_OpenMP + EXE PerfTestExec_OpenMP + ) ENDIF() IF(Kokkos_ENABLE_HPX) - LIST( APPEND SOURCES TestHPX.cpp) + SET(SOURCES + TestMain.cpp + TestHPX.cpp + ) + KOKKOS_ADD_TEST_EXECUTABLE( PerfTestExec_HPX + SOURCES ${SOURCES} + ) + + KOKKOS_ADD_TEST( NAME PerformanceTest_HPX + EXE PerfTestExec_HPX + ) ENDIF() -# Per #374, we always want to build this test, but we only want to run -# it as a PERFORMANCE test. That's why we separate building the test -# from running the test. - -TRIBITS_ADD_EXECUTABLE( - PerfTestExec - SOURCES ${SOURCES} - COMM serial mpi - TESTONLYLIBS kokkos_gtest ${TEST_LINK_TARGETS} - ) - -TRIBITS_ADD_TEST( - PerformanceTest - NAME PerfTestExec - COMM serial mpi - NUM_MPI_PROCS 1 - CATEGORIES PERFORMANCE - FAIL_REGULAR_EXPRESSION " FAILED " - ) diff --git a/lib/kokkos/containers/performance_tests/TestCuda.cpp b/lib/kokkos/containers/performance_tests/TestCuda.cpp index 351fb86df3d48836ff45286401e52a6f1f684fd7..697a006c3c05f373cc9bdbcfa1efd18a46f2b516 100644 --- a/lib/kokkos/containers/performance_tests/TestCuda.cpp +++ b/lib/kokkos/containers/performance_tests/TestCuda.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -42,7 +43,7 @@ */ #include -#if defined( KOKKOS_ENABLE_CUDA ) +#if defined(KOKKOS_ENABLE_CUDA) #include #include @@ -66,45 +67,38 @@ namespace Performance { class cuda : public ::testing::Test { -protected: - static void SetUpTestCase() - { + protected: + static void SetUpTestCase() { std::cout << std::setprecision(5) << std::scientific; Kokkos::InitArguments args(-1, -1, 0); Kokkos::initialize(args); } - static void TearDownTestCase() - { - Kokkos::finalize(); - } + static void TearDownTestCase() { Kokkos::finalize(); } }; -TEST_F( cuda, dynrankview_perf ) -{ +TEST_F(cuda, dynrankview_perf) { std::cout << "Cuda" << std::endl; std::cout << " DynRankView vs View: Initialization Only " << std::endl; - test_dynrankview_op_perf( 40960 ); + test_dynrankview_op_perf(40960); } -TEST_F( cuda, global_2_local) -{ +TEST_F(cuda, global_2_local) { std::cout << "Cuda" << std::endl; std::cout << "size, create, generate, fill, find" << std::endl; - for (unsigned i=Performance::begin_id_size; i<=Performance::end_id_size; i *= Performance::id_step) + for (unsigned i = Performance::begin_id_size; i <= Performance::end_id_size; + i *= Performance::id_step) test_global_to_local_ids(i); } -TEST_F( cuda, unordered_map_performance_near) -{ - Perf::run_performance_tests("cuda-near"); +TEST_F(cuda, unordered_map_performance_near) { + Perf::run_performance_tests("cuda-near"); } -TEST_F( cuda, unordered_map_performance_far) -{ - Perf::run_performance_tests("cuda-far"); +TEST_F(cuda, unordered_map_performance_far) { + Perf::run_performance_tests("cuda-far"); } -} +} // namespace Performance #else void KOKKOS_CONTAINERS_PERFORMANCE_TESTS_TESTCUDA_PREVENT_EMPTY_LINK_ERROR() {} -#endif /* #if defined( KOKKOS_ENABLE_CUDA ) */ +#endif /* #if defined( KOKKOS_ENABLE_CUDA ) */ diff --git a/lib/kokkos/containers/performance_tests/TestDynRankView.hpp b/lib/kokkos/containers/performance_tests/TestDynRankView.hpp index db6274e0579747079f3274b454ade63ad1a051b3..ee13f7e58ba2bcbf530a2d1234ef43e84ddc3c99 100644 --- a/lib/kokkos/containers/performance_tests/TestDynRankView.hpp +++ b/lib/kokkos/containers/performance_tests/TestDynRankView.hpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -49,109 +50,102 @@ #include -// Compare performance of DynRankView to View, specific focus on the parenthesis operators +// Compare performance of DynRankView to View, specific focus on the parenthesis +// operators namespace Performance { -//View functor +// View functor template struct InitViewFunctor { - typedef Kokkos::View inviewtype; + typedef Kokkos::View inviewtype; inviewtype _inview; - InitViewFunctor( inviewtype &inview_ ) : _inview(inview_) - {} + InitViewFunctor(inviewtype &inview_) : _inview(inview_) {} KOKKOS_INLINE_FUNCTION void operator()(const int i) const { for (unsigned j = 0; j < _inview.extent(1); ++j) { for (unsigned k = 0; k < _inview.extent(2); ++k) { - _inview(i,j,k) = i/2 -j*j + k/3; + _inview(i, j, k) = i / 2 - j * j + k / 3; } } } - struct SumComputationTest - { - typedef Kokkos::View inviewtype; + struct SumComputationTest { + typedef Kokkos::View inviewtype; inviewtype _inview; - typedef Kokkos::View outviewtype; + typedef Kokkos::View outviewtype; outviewtype _outview; KOKKOS_INLINE_FUNCTION - SumComputationTest(inviewtype &inview_ , outviewtype &outview_) : _inview(inview_), _outview(outview_) {} + SumComputationTest(inviewtype &inview_, outviewtype &outview_) + : _inview(inview_), _outview(outview_) {} KOKKOS_INLINE_FUNCTION void operator()(const int i) const { for (unsigned j = 0; j < _inview.extent(1); ++j) { for (unsigned k = 0; k < _inview.extent(2); ++k) { - _outview(i) += _inview(i,j,k) ; + _outview(i) += _inview(i, j, k); } } } }; - }; template struct InitStrideViewFunctor { - typedef Kokkos::View inviewtype; + typedef Kokkos::View inviewtype; inviewtype _inview; - InitStrideViewFunctor( inviewtype &inview_ ) : _inview(inview_) - {} + InitStrideViewFunctor(inviewtype &inview_) : _inview(inview_) {} KOKKOS_INLINE_FUNCTION void operator()(const int i) const { for (unsigned j = 0; j < _inview.extent(1); ++j) { for (unsigned k = 0; k < _inview.extent(2); ++k) { - _inview(i,j,k) = i/2 -j*j + k/3; + _inview(i, j, k) = i / 2 - j * j + k / 3; } } } - }; template struct InitViewRank7Functor { - typedef Kokkos::View inviewtype; + typedef Kokkos::View inviewtype; inviewtype _inview; - InitViewRank7Functor( inviewtype &inview_ ) : _inview(inview_) - {} + InitViewRank7Functor(inviewtype &inview_) : _inview(inview_) {} KOKKOS_INLINE_FUNCTION void operator()(const int i) const { for (unsigned j = 0; j < _inview.extent(1); ++j) { for (unsigned k = 0; k < _inview.extent(2); ++k) { - _inview(i,j,k,0,0,0,0) = i/2 -j*j + k/3; + _inview(i, j, k, 0, 0, 0, 0) = i / 2 - j * j + k / 3; } } } - }; -//DynRankView functor +// DynRankView functor template struct InitDynRankViewFunctor { typedef Kokkos::DynRankView inviewtype; inviewtype _inview; - InitDynRankViewFunctor( inviewtype &inview_ ) : _inview(inview_) - {} + InitDynRankViewFunctor(inviewtype &inview_) : _inview(inview_) {} KOKKOS_INLINE_FUNCTION void operator()(const int i) const { for (unsigned j = 0; j < _inview.extent(1); ++j) { for (unsigned k = 0; k < _inview.extent(2); ++k) { - _inview(i,j,k) = i/2 -j*j + k/3; + _inview(i, j, k) = i / 2 - j * j + k / 3; } } } - struct SumComputationTest - { + struct SumComputationTest { typedef Kokkos::DynRankView inviewtype; inviewtype _inview; @@ -159,108 +153,121 @@ struct InitDynRankViewFunctor { outviewtype _outview; KOKKOS_INLINE_FUNCTION - SumComputationTest(inviewtype &inview_ , outviewtype &outview_) : _inview(inview_), _outview(outview_) {} + SumComputationTest(inviewtype &inview_, outviewtype &outview_) + : _inview(inview_), _outview(outview_) {} KOKKOS_INLINE_FUNCTION void operator()(const int i) const { for (unsigned j = 0; j < _inview.extent(1); ++j) { for (unsigned k = 0; k < _inview.extent(2); ++k) { - _outview(i) += _inview(i,j,k) ; + _outview(i) += _inview(i, j, k); } } } }; - }; - template -void test_dynrankview_op_perf( const int par_size ) -{ - +void test_dynrankview_op_perf(const int par_size) { typedef DeviceType execution_space; typedef typename execution_space::size_type size_type; const size_type dim_2 = 90; const size_type dim_3 = 30; - double elapsed_time_view = 0; - double elapsed_time_compview = 0; + double elapsed_time_view = 0; + double elapsed_time_compview = 0; double elapsed_time_strideview = 0; double elapsed_time_view_rank7 = 0; - double elapsed_time_drview = 0; + double elapsed_time_drview = 0; double elapsed_time_compdrview = 0; Kokkos::Timer timer; { - Kokkos::View testview("testview",par_size,dim_2,dim_3); + Kokkos::View testview("testview", par_size, dim_2, + dim_3); typedef InitViewFunctor FunctorType; timer.reset(); - Kokkos::RangePolicy policy(0,par_size); - Kokkos::parallel_for( policy , FunctorType(testview) ); + Kokkos::RangePolicy policy(0, par_size); + Kokkos::parallel_for(policy, FunctorType(testview)); DeviceType().fence(); elapsed_time_view = timer.seconds(); std::cout << " View time (init only): " << elapsed_time_view << std::endl; - timer.reset(); - Kokkos::View sumview("sumview",par_size); - Kokkos::parallel_for( policy , typename FunctorType::SumComputationTest(testview, sumview) ); + Kokkos::View sumview("sumview", par_size); + Kokkos::parallel_for( + policy, typename FunctorType::SumComputationTest(testview, sumview)); DeviceType().fence(); elapsed_time_compview = timer.seconds(); - std::cout << " View sum computation time: " << elapsed_time_view << std::endl; - + std::cout << " View sum computation time: " << elapsed_time_view + << std::endl; - Kokkos::View teststrideview = Kokkos::subview(testview, Kokkos::ALL, Kokkos::ALL,Kokkos::ALL); + Kokkos::View teststrideview = + Kokkos::subview(testview, Kokkos::ALL, Kokkos::ALL, Kokkos::ALL); typedef InitStrideViewFunctor FunctorStrideType; timer.reset(); - Kokkos::parallel_for( policy , FunctorStrideType(teststrideview) ); + Kokkos::parallel_for(policy, FunctorStrideType(teststrideview)); DeviceType().fence(); elapsed_time_strideview = timer.seconds(); - std::cout << " Strided View time (init only): " << elapsed_time_strideview << std::endl; + std::cout << " Strided View time (init only): " << elapsed_time_strideview + << std::endl; } { - Kokkos::View testview("testview",par_size,dim_2,dim_3,1,1,1,1); + Kokkos::View testview("testview", par_size, + dim_2, dim_3, 1, 1, 1, 1); typedef InitViewRank7Functor FunctorType; timer.reset(); - Kokkos::RangePolicy policy(0,par_size); - Kokkos::parallel_for( policy , FunctorType(testview) ); + Kokkos::RangePolicy policy(0, par_size); + Kokkos::parallel_for(policy, FunctorType(testview)); DeviceType().fence(); elapsed_time_view_rank7 = timer.seconds(); - std::cout << " View Rank7 time (init only): " << elapsed_time_view_rank7 << std::endl; + std::cout << " View Rank7 time (init only): " << elapsed_time_view_rank7 + << std::endl; } { - Kokkos::DynRankView testdrview("testdrview",par_size,dim_2,dim_3); + Kokkos::DynRankView testdrview("testdrview", par_size, + dim_2, dim_3); typedef InitDynRankViewFunctor FunctorType; timer.reset(); - Kokkos::RangePolicy policy(0,par_size); - Kokkos::parallel_for( policy , FunctorType(testdrview) ); + Kokkos::RangePolicy policy(0, par_size); + Kokkos::parallel_for(policy, FunctorType(testdrview)); DeviceType().fence(); elapsed_time_drview = timer.seconds(); - std::cout << " DynRankView time (init only): " << elapsed_time_drview << std::endl; + std::cout << " DynRankView time (init only): " << elapsed_time_drview + << std::endl; timer.reset(); - Kokkos::DynRankView sumview("sumview",par_size); - Kokkos::parallel_for( policy , typename FunctorType::SumComputationTest(testdrview, sumview) ); + Kokkos::DynRankView sumview("sumview", par_size); + Kokkos::parallel_for( + policy, typename FunctorType::SumComputationTest(testdrview, sumview)); DeviceType().fence(); elapsed_time_compdrview = timer.seconds(); - std::cout << " DynRankView sum computation time: " << elapsed_time_compdrview << std::endl; - + std::cout << " DynRankView sum computation time: " + << elapsed_time_compdrview << std::endl; } - std::cout << " Ratio of View to DynRankView time: " << elapsed_time_view / elapsed_time_drview << std::endl; //expect < 1 - std::cout << " Ratio of View to DynRankView sum computation time: " << elapsed_time_compview / elapsed_time_compdrview << std::endl; //expect < 1 - std::cout << " Ratio of View to View Rank7 time: " << elapsed_time_view / elapsed_time_view_rank7 << std::endl; //expect < 1 - std::cout << " Ratio of StrideView to DynRankView time: " << elapsed_time_strideview / elapsed_time_drview << std::endl; //expect < 1 - std::cout << " Ratio of DynRankView to View Rank7 time: " << elapsed_time_drview / elapsed_time_view_rank7 << std::endl; //expect ? + std::cout << " Ratio of View to DynRankView time: " + << elapsed_time_view / elapsed_time_drview + << std::endl; // expect < 1 + std::cout << " Ratio of View to DynRankView sum computation time: " + << elapsed_time_compview / elapsed_time_compdrview + << std::endl; // expect < 1 + std::cout << " Ratio of View to View Rank7 time: " + << elapsed_time_view / elapsed_time_view_rank7 + << std::endl; // expect < 1 + std::cout << " Ratio of StrideView to DynRankView time: " + << elapsed_time_strideview / elapsed_time_drview + << std::endl; // expect < 1 + std::cout << " Ratio of DynRankView to View Rank7 time: " + << elapsed_time_drview / elapsed_time_view_rank7 + << std::endl; // expect ? timer.reset(); -} //end test_dynrankview - +} // end test_dynrankview -} //end Performance +} // namespace Performance #endif - diff --git a/lib/kokkos/containers/performance_tests/TestGlobal2LocalIds.hpp b/lib/kokkos/containers/performance_tests/TestGlobal2LocalIds.hpp index 98997b32394b7a9d71a1e78e3041993e25bd4193..0d2ee4bc8d0ccff968363e81e3b6bb353a79492f 100644 --- a/lib/kokkos/containers/performance_tests/TestGlobal2LocalIds.hpp +++ b/lib/kokkos/containers/performance_tests/TestGlobal2LocalIds.hpp @@ -1,12 +1,13 @@ //@HEADER // ************************************************************************ -// -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: @@ -22,10 +23,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -35,7 +36,7 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// +// // ************************************************************************ //@HEADER @@ -54,153 +55,137 @@ namespace Performance { static const unsigned begin_id_size = 256u; -static const unsigned end_id_size = 1u << 22; -static const unsigned id_step = 2u; +static const unsigned end_id_size = 1u << 22; +static const unsigned id_step = 2u; -union helper -{ +union helper { uint32_t word; uint8_t byte[4]; }; - template -struct generate_ids -{ +struct generate_ids { typedef Device execution_space; typedef typename execution_space::size_type size_type; - typedef Kokkos::View local_id_view; + typedef Kokkos::View local_id_view; local_id_view local_2_global; - generate_ids( local_id_view & ids) - : local_2_global(ids) - { + generate_ids(local_id_view& ids) : local_2_global(ids) { Kokkos::parallel_for(local_2_global.extent(0), *this); } - KOKKOS_INLINE_FUNCTION - void operator()(size_type i) const - { - + void operator()(size_type i) const { helper x = {static_cast(i)}; // shuffle the bytes of i to create a unique, semi-random global_id x.word = ~x.word; uint8_t tmp = x.byte[3]; - x.byte[3] = x.byte[1]; - x.byte[1] = tmp; + x.byte[3] = x.byte[1]; + x.byte[1] = tmp; - tmp = x.byte[2]; + tmp = x.byte[2]; x.byte[2] = x.byte[0]; x.byte[0] = tmp; local_2_global[i] = x.word; } - }; template -struct fill_map -{ +struct fill_map { typedef Device execution_space; typedef typename execution_space::size_type size_type; - typedef Kokkos::View local_id_view; - typedef Kokkos::UnorderedMap global_id_view; + typedef Kokkos::View + local_id_view; + typedef Kokkos::UnorderedMap + global_id_view; global_id_view global_2_local; local_id_view local_2_global; - fill_map( global_id_view gIds, local_id_view lIds) - : global_2_local(gIds) , local_2_global(lIds) - { + fill_map(global_id_view gIds, local_id_view lIds) + : global_2_local(gIds), local_2_global(lIds) { Kokkos::parallel_for(local_2_global.extent(0), *this); } KOKKOS_INLINE_FUNCTION - void operator()(size_type i) const - { - global_2_local.insert( local_2_global[i], i); + void operator()(size_type i) const { + global_2_local.insert(local_2_global[i], i); } - }; template -struct find_test -{ +struct find_test { typedef Device execution_space; typedef typename execution_space::size_type size_type; - typedef Kokkos::View local_id_view; - typedef Kokkos::UnorderedMap global_id_view; + typedef Kokkos::View + local_id_view; + typedef Kokkos::UnorderedMap + global_id_view; global_id_view global_2_local; local_id_view local_2_global; typedef size_t value_type; - find_test( global_id_view gIds, local_id_view lIds, value_type & num_errors) - : global_2_local(gIds) , local_2_global(lIds) - { + find_test(global_id_view gIds, local_id_view lIds, value_type& num_errors) + : global_2_local(gIds), local_2_global(lIds) { Kokkos::parallel_reduce(local_2_global.extent(0), *this, num_errors); } KOKKOS_INLINE_FUNCTION - void init(value_type & v) const - { v = 0; } + void init(value_type& v) const { v = 0; } KOKKOS_INLINE_FUNCTION - void join(volatile value_type & dst, volatile value_type const & src) const - { dst += src; } + void join(volatile value_type& dst, volatile value_type const& src) const { + dst += src; + } KOKKOS_INLINE_FUNCTION - void operator()(size_type i, value_type & num_errors) const - { - uint32_t index = global_2_local.find( local_2_global[i] ); + void operator()(size_type i, value_type& num_errors) const { + uint32_t index = global_2_local.find(local_2_global[i]); - if ( global_2_local.value_at(index) != i) ++num_errors; + if (global_2_local.value_at(index) != i) ++num_errors; } - }; template -void test_global_to_local_ids(unsigned num_ids) -{ - +void test_global_to_local_ids(unsigned num_ids) { typedef Device execution_space; typedef typename execution_space::size_type size_type; - typedef Kokkos::View local_id_view; - typedef Kokkos::UnorderedMap global_id_view; + typedef Kokkos::View local_id_view; + typedef Kokkos::UnorderedMap + global_id_view; - //size + // size std::cout << num_ids << ", "; double elasped_time = 0; Kokkos::Timer timer; local_id_view local_2_global("local_ids", num_ids); - global_id_view global_2_local((3u*num_ids)/2u); + global_id_view global_2_local((3u * num_ids) / 2u); - //create + // create elasped_time = timer.seconds(); std::cout << elasped_time << ", "; timer.reset(); // generate unique ids - { - generate_ids gen(local_2_global); - } + { generate_ids gen(local_2_global); } Device().fence(); // generate elasped_time = timer.seconds(); std::cout << elasped_time << ", "; timer.reset(); - { - fill_map fill(global_2_local, local_2_global); - } + { fill_map fill(global_2_local, local_2_global); } Device().fence(); // fill @@ -208,11 +193,9 @@ void test_global_to_local_ids(unsigned num_ids) std::cout << elasped_time << ", "; timer.reset(); - size_t num_errors = 0; - for (int i=0; i<100; ++i) - { - find_test find(global_2_local, local_2_global,num_errors); + for (int i = 0; i < 100; ++i) { + find_test find(global_2_local, local_2_global, num_errors); } Device().fence(); @@ -220,12 +203,9 @@ void test_global_to_local_ids(unsigned num_ids) elasped_time = timer.seconds(); std::cout << elasped_time << std::endl; - ASSERT_EQ( num_errors, 0u); + ASSERT_EQ(num_errors, 0u); } +} // namespace Performance -} // namespace Performance - - -#endif //KOKKOS_TEST_GLOBAL_TO_LOCAL_IDS_HPP - +#endif // KOKKOS_TEST_GLOBAL_TO_LOCAL_IDS_HPP diff --git a/lib/kokkos/containers/performance_tests/TestHPX.cpp b/lib/kokkos/containers/performance_tests/TestHPX.cpp index 0f43377cee499f04d14736663c92aa32e49c0a58..48be466bfa9c8f73eac25187a665d46230667faf 100644 --- a/lib/kokkos/containers/performance_tests/TestHPX.cpp +++ b/lib/kokkos/containers/performance_tests/TestHPX.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -42,7 +43,7 @@ */ #include -#if defined( KOKKOS_ENABLE_HPX ) +#if defined(KOKKOS_ENABLE_HPX) #include @@ -61,70 +62,63 @@ #include #include - namespace Performance { class hpx : public ::testing::Test { -protected: - static void SetUpTestCase() - { + protected: + static void SetUpTestCase() { std::cout << std::setprecision(5) << std::scientific; Kokkos::initialize(); - Kokkos::print_configuration( std::cout ); + Kokkos::print_configuration(std::cout); } - static void TearDownTestCase() - { - Kokkos::finalize(); - } + static void TearDownTestCase() { Kokkos::finalize(); } }; -TEST_F( hpx, dynrankview_perf ) -{ +TEST_F(hpx, dynrankview_perf) { std::cout << "HPX" << std::endl; std::cout << " DynRankView vs View: Initialization Only " << std::endl; - test_dynrankview_op_perf( 8192 ); + test_dynrankview_op_perf(8192); } -TEST_F( hpx, global_2_local) -{ +TEST_F(hpx, global_2_local) { std::cout << "HPX" << std::endl; std::cout << "size, create, generate, fill, find" << std::endl; - for (unsigned i=Performance::begin_id_size; i<=Performance::end_id_size; i *= Performance::id_step) + for (unsigned i = Performance::begin_id_size; i <= Performance::end_id_size; + i *= Performance::id_step) test_global_to_local_ids(i); } -TEST_F( hpx, unordered_map_performance_near) -{ +TEST_F(hpx, unordered_map_performance_near) { unsigned num_hpx = 4; std::ostringstream base_file_name; base_file_name << "hpx-" << num_hpx << "-near"; - Perf::run_performance_tests(base_file_name.str()); + Perf::run_performance_tests( + base_file_name.str()); } -TEST_F( hpx, unordered_map_performance_far) -{ +TEST_F(hpx, unordered_map_performance_far) { unsigned num_hpx = 4; std::ostringstream base_file_name; base_file_name << "hpx-" << num_hpx << "-far"; - Perf::run_performance_tests(base_file_name.str()); + Perf::run_performance_tests( + base_file_name.str()); } -TEST_F( hpx, scatter_view) -{ +TEST_F(hpx, scatter_view) { std::cout << "ScatterView data-duplicated test:\n"; Perf::test_scatter_view(10, 1000 * 1000); -//std::cout << "ScatterView atomics test:\n"; -//Perf::test_scatter_view(10, 1000 * 1000); + Kokkos::Experimental::ScatterDuplicated, + Kokkos::Experimental::ScatterNonAtomic>(10, + 1000 * 1000); + // std::cout << "ScatterView atomics test:\n"; + // Perf::test_scatter_view(10, 1000 * 1000); } -} // namespace test +} // namespace Performance #else void KOKKOS_CONTAINERS_PERFORMANCE_TESTS_TESTHPX_PREVENT_EMPTY_LINK_ERROR() {} #endif - diff --git a/lib/kokkos/containers/performance_tests/TestMain.cpp b/lib/kokkos/containers/performance_tests/TestMain.cpp index 217b01a57afccf1bc7658ce41214d77fbffffd67..e3c8edb0457d2cc0bcaffdf0449da008b47d5dcb 100644 --- a/lib/kokkos/containers/performance_tests/TestMain.cpp +++ b/lib/kokkos/containers/performance_tests/TestMain.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -47,7 +48,6 @@ #include int main(int argc, char *argv[]) { - ::testing::InitGoogleTest(&argc,argv); + ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } - diff --git a/lib/kokkos/containers/performance_tests/TestOpenMP.cpp b/lib/kokkos/containers/performance_tests/TestOpenMP.cpp index e6218074eafd965554c3ebe60136386c488fd882..a9c8639ed4b03eab0eaca89a4139c5262af942eb 100644 --- a/lib/kokkos/containers/performance_tests/TestOpenMP.cpp +++ b/lib/kokkos/containers/performance_tests/TestOpenMP.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -42,7 +43,7 @@ */ #include -#if defined( KOKKOS_ENABLE_OPENMP ) +#if defined(KOKKOS_ENABLE_OPENMP) #include @@ -61,82 +62,72 @@ #include #include - namespace Performance { class openmp : public ::testing::Test { -protected: - static void SetUpTestCase() - { + protected: + static void SetUpTestCase() { std::cout << std::setprecision(5) << std::scientific; Kokkos::initialize(); - Kokkos::OpenMP::print_configuration( std::cout ); + Kokkos::OpenMP::print_configuration(std::cout); } - static void TearDownTestCase() - { - Kokkos::finalize(); - } + static void TearDownTestCase() { Kokkos::finalize(); } }; -TEST_F( openmp, dynrankview_perf ) -{ +TEST_F(openmp, dynrankview_perf) { std::cout << "OpenMP" << std::endl; std::cout << " DynRankView vs View: Initialization Only " << std::endl; - test_dynrankview_op_perf( 8192 ); + test_dynrankview_op_perf(8192); } -TEST_F( openmp, global_2_local) -{ +TEST_F(openmp, global_2_local) { std::cout << "OpenMP" << std::endl; std::cout << "size, create, generate, fill, find" << std::endl; - for (unsigned i=Performance::begin_id_size; i<=Performance::end_id_size; i *= Performance::id_step) + for (unsigned i = Performance::begin_id_size; i <= Performance::end_id_size; + i *= Performance::id_step) test_global_to_local_ids(i); } -TEST_F( openmp, unordered_map_performance_near) -{ +TEST_F(openmp, unordered_map_performance_near) { unsigned num_openmp = 4; if (Kokkos::hwloc::available()) { num_openmp = Kokkos::hwloc::get_available_numa_count() * - Kokkos::hwloc::get_available_cores_per_numa() * - Kokkos::hwloc::get_available_threads_per_core(); - + Kokkos::hwloc::get_available_cores_per_numa() * + Kokkos::hwloc::get_available_threads_per_core(); } std::ostringstream base_file_name; base_file_name << "openmp-" << num_openmp << "-near"; - Perf::run_performance_tests(base_file_name.str()); + Perf::run_performance_tests(base_file_name.str()); } -TEST_F( openmp, unordered_map_performance_far) -{ +TEST_F(openmp, unordered_map_performance_far) { unsigned num_openmp = 4; if (Kokkos::hwloc::available()) { num_openmp = Kokkos::hwloc::get_available_numa_count() * - Kokkos::hwloc::get_available_cores_per_numa() * - Kokkos::hwloc::get_available_threads_per_core(); - + Kokkos::hwloc::get_available_cores_per_numa() * + Kokkos::hwloc::get_available_threads_per_core(); } std::ostringstream base_file_name; base_file_name << "openmp-" << num_openmp << "-far"; - Perf::run_performance_tests(base_file_name.str()); + Perf::run_performance_tests(base_file_name.str()); } -TEST_F( openmp, scatter_view) -{ +TEST_F(openmp, scatter_view) { std::cout << "ScatterView data-duplicated test:\n"; Perf::test_scatter_view(10, 1000 * 1000); -//std::cout << "ScatterView atomics test:\n"; -//Perf::test_scatter_view(10, 1000 * 1000); + Kokkos::Experimental::ScatterDuplicated, + Kokkos::Experimental::ScatterNonAtomic>(10, + 1000 * 1000); + // std::cout << "ScatterView atomics test:\n"; + // Perf::test_scatter_view(10, 1000 * 1000); } -} // namespace test +} // namespace Performance #else -void KOKKOS_CONTAINERS_PERFORMANCE_TESTS_TESTOPENMP_PREVENT_EMPTY_LINK_ERROR() {} +void KOKKOS_CONTAINERS_PERFORMANCE_TESTS_TESTOPENMP_PREVENT_EMPTY_LINK_ERROR() { +} #endif - diff --git a/lib/kokkos/containers/performance_tests/TestROCm.cpp b/lib/kokkos/containers/performance_tests/TestROCm.cpp index 3cf9f3bd143c5848af12e7ddb717ab8e4d59ce8e..55b770b49cb9bef5ed78003530b421800e66370b 100644 --- a/lib/kokkos/containers/performance_tests/TestROCm.cpp +++ b/lib/kokkos/containers/performance_tests/TestROCm.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -42,7 +43,7 @@ */ #include -#if defined( KOKKOS_ENABLE_ROCM ) +#if defined(KOKKOS_ENABLE_ROCM) #include #include @@ -66,15 +67,14 @@ namespace Performance { class rocm : public ::testing::Test { -protected: - static void SetUpTestCase() - { + protected: + static void SetUpTestCase() { std::cout << std::setprecision(5) << std::scientific; Kokkos::HostSpace::execution_space::initialize(); - Kokkos::Experimental::ROCm::initialize( Kokkos::Experimental::ROCm::SelectDevice(0) ); + Kokkos::Experimental::ROCm::initialize( + Kokkos::Experimental::ROCm::SelectDevice(0)); } - static void TearDownTestCase() - { + static void TearDownTestCase() { Kokkos::Experimental::ROCm::finalize(); Kokkos::HostSpace::execution_space::finalize(); } @@ -97,17 +97,15 @@ TEST_F( rocm, global_2_local) } #endif -TEST_F( rocm, unordered_map_performance_near) -{ - Perf::run_performance_tests("rocm-near"); +TEST_F(rocm, unordered_map_performance_near) { + Perf::run_performance_tests("rocm-near"); } -TEST_F( rocm, unordered_map_performance_far) -{ - Perf::run_performance_tests("rocm-far"); +TEST_F(rocm, unordered_map_performance_far) { + Perf::run_performance_tests("rocm-far"); } -} +} // namespace Performance #else void KOKKOS_CONTAINERS_PERFORMANCE_TESTS_TESTROCM_PREVENT_EMPTY_LINK_ERROR() {} -#endif /* #if defined( KOKKOS_ENABLE_ROCM ) */ +#endif /* #if defined( KOKKOS_ENABLE_ROCM ) */ diff --git a/lib/kokkos/containers/performance_tests/TestScatterView.hpp b/lib/kokkos/containers/performance_tests/TestScatterView.hpp index bd9121bb829a903bc8cb6e78b51668bcc45448e4..3d4c57f3e2c844e21fc0a1a491fc1fe3608c8f2b 100644 --- a/lib/kokkos/containers/performance_tests/TestScatterView.hpp +++ b/lib/kokkos/containers/performance_tests/TestScatterView.hpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -49,67 +50,68 @@ namespace Perf { -template -void test_scatter_view(int m, int n) -{ - Kokkos::View original_view("original_view", n); +template +void test_scatter_view(int m, int n) { + Kokkos::View original_view("original_view", + n); { - auto scatter_view = Kokkos::Experimental::create_scatter_view - < Kokkos::Experimental::ScatterSum - , duplication - , contribution - > (original_view); + auto scatter_view = Kokkos::Experimental::create_scatter_view< + Kokkos::Experimental::ScatterSum, duplication, contribution>( + original_view); Kokkos::Experimental::UniqueToken< - ExecSpace, Kokkos::Experimental::UniqueTokenScope::Global> - unique_token{ExecSpace()}; - //auto internal_view = scatter_view.internal_view; + ExecSpace, Kokkos::Experimental::UniqueTokenScope::Global> + unique_token{ExecSpace()}; + // auto internal_view = scatter_view.internal_view; auto policy = Kokkos::RangePolicy(0, n); for (int foo = 0; foo < 5; ++foo) { - { - auto num_threads = unique_token.size(); - std::cout << "num_threads " << num_threads << '\n'; - Kokkos::View hand_coded_duplicate_view("hand_coded_duplicate", num_threads, n); - auto f2 = KOKKOS_LAMBDA(int i) { - auto thread_id = unique_token.acquire(); - for (int j = 0; j < 10; ++j) { - auto k = (i + j) % n; - hand_coded_duplicate_view(thread_id, k, 0) += 4.2; - hand_coded_duplicate_view(thread_id, k, 1) += 2.0; - hand_coded_duplicate_view(thread_id, k, 2) += 1.0; + { + auto num_threads = unique_token.size(); + std::cout << "num_threads " << num_threads << '\n'; + Kokkos::View + hand_coded_duplicate_view("hand_coded_duplicate", num_threads, n); + auto f2 = KOKKOS_LAMBDA(int i) { + auto thread_id = unique_token.acquire(); + for (int j = 0; j < 10; ++j) { + auto k = (i + j) % n; + hand_coded_duplicate_view(thread_id, k, 0) += 4.2; + hand_coded_duplicate_view(thread_id, k, 1) += 2.0; + hand_coded_duplicate_view(thread_id, k, 2) += 1.0; + } + }; + Kokkos::Timer timer; + timer.reset(); + for (int k = 0; k < m; ++k) { + Kokkos::parallel_for(policy, f2, + "hand_coded_duplicate_scatter_view_test"); } - }; - Kokkos::Timer timer; - timer.reset(); - for (int k = 0; k < m; ++k) { - Kokkos::parallel_for(policy, f2, "hand_coded_duplicate_scatter_view_test"); + Kokkos::fence(); + auto t = timer.seconds(); + std::cout << "hand-coded test took " << t << " seconds\n"; } - Kokkos::fence(); - auto t = timer.seconds(); - std::cout << "hand-coded test took " << t << " seconds\n"; - } - { - auto f = KOKKOS_LAMBDA(int i) { - auto scatter_access = scatter_view.access(); - for (int j = 0; j < 10; ++j) { - auto k = (i + j) % n; - scatter_access(k, 0) += 4.2; - scatter_access(k, 1) += 2.0; - scatter_access(k, 2) += 1.0; + { + auto f = KOKKOS_LAMBDA(int i) { + auto scatter_access = scatter_view.access(); + for (int j = 0; j < 10; ++j) { + auto k = (i + j) % n; + scatter_access(k, 0) += 4.2; + scatter_access(k, 1) += 2.0; + scatter_access(k, 2) += 1.0; + } + }; + Kokkos::Timer timer; + timer.reset(); + for (int k = 0; k < m; ++k) { + Kokkos::parallel_for(policy, f, "scatter_view_test"); } - }; - Kokkos::Timer timer; - timer.reset(); - for (int k = 0; k < m; ++k) { - Kokkos::parallel_for(policy, f, "scatter_view_test"); + Kokkos::fence(); + auto t = timer.seconds(); + std::cout << "test took " << t << " seconds\n"; } - Kokkos::fence(); - auto t = timer.seconds(); - std::cout << "test took " << t << " seconds\n"; } } - } } -} +} // namespace Perf #endif diff --git a/lib/kokkos/containers/performance_tests/TestThreads.cpp b/lib/kokkos/containers/performance_tests/TestThreads.cpp index 6a02e67b252781f7320223ad8fb87cadfef3355e..2f3740453966700655e37ecbf1e46552d7b18ae7 100644 --- a/lib/kokkos/containers/performance_tests/TestThreads.cpp +++ b/lib/kokkos/containers/performance_tests/TestThreads.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -42,7 +43,7 @@ */ #include -#if defined( KOKKOS_ENABLE_THREADS ) +#if defined(KOKKOS_ENABLE_THREADS) #include @@ -65,9 +66,8 @@ namespace Performance { class threads : public ::testing::Test { -protected: - static void SetUpTestCase() - { + protected: + static void SetUpTestCase() { std::cout << std::setprecision(5) << std::scientific; unsigned num_threads = 4; @@ -76,66 +76,57 @@ protected: num_threads = Kokkos::hwloc::get_available_numa_count() * Kokkos::hwloc::get_available_cores_per_numa() * Kokkos::hwloc::get_available_threads_per_core(); - } std::cout << "Threads: " << num_threads << std::endl; - Kokkos::initialize( Kokkos::InitArguments(num_threads) ); + Kokkos::initialize(Kokkos::InitArguments(num_threads)); } - static void TearDownTestCase() - { - Kokkos::finalize(); - } + static void TearDownTestCase() { Kokkos::finalize(); } }; -TEST_F( threads, dynrankview_perf ) -{ +TEST_F(threads, dynrankview_perf) { std::cout << "Threads" << std::endl; std::cout << " DynRankView vs View: Initialization Only " << std::endl; - test_dynrankview_op_perf( 8192 ); + test_dynrankview_op_perf(8192); } -TEST_F( threads, global_2_local) -{ +TEST_F(threads, global_2_local) { std::cout << "Threads" << std::endl; std::cout << "size, create, generate, fill, find" << std::endl; - for (unsigned i=Performance::begin_id_size; i<=Performance::end_id_size; i *= Performance::id_step) + for (unsigned i = Performance::begin_id_size; i <= Performance::end_id_size; + i *= Performance::id_step) test_global_to_local_ids(i); } -TEST_F( threads, unordered_map_performance_near) -{ +TEST_F(threads, unordered_map_performance_near) { unsigned num_threads = 4; if (Kokkos::hwloc::available()) { num_threads = Kokkos::hwloc::get_available_numa_count() * Kokkos::hwloc::get_available_cores_per_numa() * Kokkos::hwloc::get_available_threads_per_core(); - } std::ostringstream base_file_name; base_file_name << "threads-" << num_threads << "-near"; - Perf::run_performance_tests(base_file_name.str()); + Perf::run_performance_tests(base_file_name.str()); } -TEST_F( threads, unordered_map_performance_far) -{ +TEST_F(threads, unordered_map_performance_far) { unsigned num_threads = 4; if (Kokkos::hwloc::available()) { num_threads = Kokkos::hwloc::get_available_numa_count() * Kokkos::hwloc::get_available_cores_per_numa() * Kokkos::hwloc::get_available_threads_per_core(); - } std::ostringstream base_file_name; base_file_name << "threads-" << num_threads << "-far"; - Perf::run_performance_tests(base_file_name.str()); + Perf::run_performance_tests(base_file_name.str()); } -} // namespace Performance +} // namespace Performance #else -void KOKKOS_CONTAINERS_PERFORMANCE_TESTS_TESTTHREADS_PREVENT_EMPTY_LINK_ERROR() {} +void KOKKOS_CONTAINERS_PERFORMANCE_TESTS_TESTTHREADS_PREVENT_EMPTY_LINK_ERROR() { +} #endif - diff --git a/lib/kokkos/containers/performance_tests/TestUnorderedMapPerformance.hpp b/lib/kokkos/containers/performance_tests/TestUnorderedMapPerformance.hpp index 8d09281ed36cc99690c40784397a448cf1c8e1d8..9057842340bc7bfd6f59f57b35b5c9536918d554 100644 --- a/lib/kokkos/containers/performance_tests/TestUnorderedMapPerformance.hpp +++ b/lib/kokkos/containers/performance_tests/TestUnorderedMapPerformance.hpp @@ -1,10 +1,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -22,10 +23,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -50,12 +51,10 @@ #include #include - namespace Perf { template -struct UnorderedMapTest -{ +struct UnorderedMapTest { typedef Device execution_space; typedef Kokkos::UnorderedMap map_type; typedef typename map_type::histogram_type histogram_type; @@ -68,22 +67,22 @@ struct UnorderedMapTest uint32_t capacity; uint32_t inserts; uint32_t collisions; - double seconds; + double seconds; map_type map; histogram_type histogram; - UnorderedMapTest( uint32_t arg_capacity, uint32_t arg_inserts, uint32_t arg_collisions) - : capacity(arg_capacity) - , inserts(arg_inserts) - , collisions(arg_collisions) - , seconds(0) - , map(capacity) - , histogram(map.get_histogram()) - { - Kokkos::Timer wall_clock ; + UnorderedMapTest(uint32_t arg_capacity, uint32_t arg_inserts, + uint32_t arg_collisions) + : capacity(arg_capacity), + inserts(arg_inserts), + collisions(arg_collisions), + seconds(0), + map(capacity), + histogram(map.get_histogram()) { + Kokkos::Timer wall_clock; wall_clock.reset(); - value_type v = {}; + value_type v = {}; int loop_count = 0; do { ++loop_count; @@ -92,81 +91,79 @@ struct UnorderedMapTest Kokkos::parallel_reduce(inserts, *this, v); if (v.failed_count > 0u) { - const uint32_t new_capacity = map.capacity() + ((map.capacity()*3ull)/20u) + v.failed_count/collisions ; - map.rehash( new_capacity ); + const uint32_t new_capacity = map.capacity() + + ((map.capacity() * 3ull) / 20u) + + v.failed_count / collisions; + map.rehash(new_capacity); } } while (v.failed_count > 0u); seconds = wall_clock.seconds(); - switch (loop_count) - { - case 1u: std::cout << " \033[0;32m" << loop_count << "\033[0m "; break; - case 2u: std::cout << " \033[1;31m" << loop_count << "\033[0m "; break; - default: std::cout << " \033[0;31m" << loop_count << "\033[0m "; break; + switch (loop_count) { + case 1u: std::cout << " \033[0;32m" << loop_count << "\033[0m "; break; + case 2u: std::cout << " \033[1;31m" << loop_count << "\033[0m "; break; + default: std::cout << " \033[0;31m" << loop_count << "\033[0m "; break; } - std::cout << std::setprecision(2) << std::fixed << std::setw(5) << (1e9*(seconds/(inserts))) << "; " << std::flush; + std::cout << std::setprecision(2) << std::fixed << std::setw(5) + << (1e9 * (seconds / (inserts))) << "; " << std::flush; histogram.calculate(); Device().fence(); } - void print(std::ostream & metrics_out, std::ostream & length_out, std::ostream & distance_out, std::ostream & block_distance_out) - { + void print(std::ostream& metrics_out, std::ostream& length_out, + std::ostream& distance_out, std::ostream& block_distance_out) { metrics_out << map.capacity() << " , "; - metrics_out << inserts/collisions << " , "; - metrics_out << (100.0 * inserts/collisions) / map.capacity() << " , "; + metrics_out << inserts / collisions << " , "; + metrics_out << (100.0 * inserts / collisions) / map.capacity() << " , "; metrics_out << inserts << " , "; metrics_out << (map.failed_insert() ? "true" : "false") << " , "; metrics_out << collisions << " , "; - metrics_out << 1e9*(seconds/inserts) << " , "; + metrics_out << 1e9 * (seconds / inserts) << " , "; metrics_out << seconds << std::endl; length_out << map.capacity() << " , "; - length_out << ((100.0 *inserts/collisions) / map.capacity()) << " , "; + length_out << ((100.0 * inserts / collisions) / map.capacity()) << " , "; length_out << collisions << " , "; histogram.print_length(length_out); distance_out << map.capacity() << " , "; - distance_out << ((100.0 *inserts/collisions) / map.capacity()) << " , "; + distance_out << ((100.0 * inserts / collisions) / map.capacity()) << " , "; distance_out << collisions << " , "; histogram.print_distance(distance_out); block_distance_out << map.capacity() << " , "; - block_distance_out << ((100.0 *inserts/collisions) / map.capacity()) << " , "; + block_distance_out << ((100.0 * inserts / collisions) / map.capacity()) + << " , "; block_distance_out << collisions << " , "; histogram.print_block_distance(block_distance_out); } - KOKKOS_INLINE_FUNCTION - void init( value_type & v ) const - { + void init(value_type& v) const { v.failed_count = 0; - v.max_list = 0; + v.max_list = 0; } KOKKOS_INLINE_FUNCTION - void join( volatile value_type & dst, const volatile value_type & src ) const - { + void join(volatile value_type& dst, const volatile value_type& src) const { dst.failed_count += src.failed_count; dst.max_list = src.max_list < dst.max_list ? dst.max_list : src.max_list; } KOKKOS_INLINE_FUNCTION - void operator()(uint32_t i, value_type & v) const - { - const uint32_t key = Near ? i/collisions : i%(inserts/collisions); - typename map_type::insert_result result = map.insert(key,i); + void operator()(uint32_t i, value_type& v) const { + const uint32_t key = Near ? i / collisions : i % (inserts / collisions); + typename map_type::insert_result result = map.insert(key, i); v.failed_count += !result.failed() ? 0 : 1; - v.max_list = result.list_position() < v.max_list ? v.max_list : result.list_position(); + v.max_list = result.list_position() < v.max_list ? v.max_list + : result.list_position(); } - }; template -void run_performance_tests(std::string const & base_file_name) -{ +void run_performance_tests(std::string const& base_file_name) { #if 0 std::string metrics_file_name = base_file_name + std::string("-metrics.csv"); std::string length_file_name = base_file_name + std::string("-length.csv"); @@ -254,7 +251,6 @@ void run_performance_tests(std::string const & base_file_name) #endif } +} // namespace Perf -} // namespace Perf - -#endif //KOKKOS_TEST_UNORDERED_MAP_PERFORMANCE_HPP +#endif // KOKKOS_TEST_UNORDERED_MAP_PERFORMANCE_HPP diff --git a/lib/kokkos/containers/src/CMakeLists.txt b/lib/kokkos/containers/src/CMakeLists.txt index e68fcad5e945fce8010cd3b0a6ff62298eb77de0..0c9d24d641bb61d99514a0a78a8405f5b7a86f3d 100644 --- a/lib/kokkos/containers/src/CMakeLists.txt +++ b/lib/kokkos/containers/src/CMakeLists.txt @@ -1,47 +1,34 @@ - -TRIBITS_CONFIGURE_FILE(${PACKAGE_NAME}_config.h) - -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) - -#----------------------------------------------------------------------------- - -SET(TRILINOS_INCDIR ${CMAKE_INSTALL_PREFIX}/${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}) - -if(KOKKOS_LEGACY_TRIBITS) - - SET(HEADERS "") - SET(SOURCES "") - - SET(HEADERS_IMPL "") - - FILE(GLOB HEADERS *.hpp) - FILE(GLOB HEADERS_IMPL impl/*.hpp) - FILE(GLOB SOURCES impl/*.cpp) - - INSTALL(FILES ${HEADERS_IMPL} DESTINATION ${TRILINOS_INCDIR}/impl/) - - TRIBITS_ADD_LIBRARY( - kokkoscontainers - HEADERS ${HEADERS} - NOINSTALLHEADERS ${HEADERS_IMPL} - SOURCES ${SOURCES} - DEPLIBS - ) - -else() - - INSTALL ( - DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/" - DESTINATION ${TRILINOS_INCDIR} - FILES_MATCHING PATTERN "*.hpp" - ) - - TRIBITS_ADD_LIBRARY( - kokkoscontainers - SOURCES ${KOKKOS_CONTAINERS_SRCS} - DEPLIBS - ) - -endif() -#----------------------------------------------------------------------------- + +KOKKOS_CONFIGURE_FILE(${PACKAGE_NAME}_config.h) + +#need these here for now +KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) +KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) + +#----------------------------------------------------------------------------- + +SET(KOKKOS_CONTAINERS_SRCS) +APPEND_GLOB(KOKKOS_CONTAINERS_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/impl/*.cpp) + +INSTALL ( + DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/" + DESTINATION ${KOKKOS_HEADER_DIR} + FILES_MATCHING PATTERN "*.hpp" +) + +KOKKOS_ADD_LIBRARY( + kokkoscontainers + SOURCES ${KOKKOS_CONTAINERS_SRCS} +) + +SET_TARGET_PROPERTIES(kokkoscontainers PROPERTIES VERSION ${Kokkos_VERSION}) + +KOKKOS_LIB_INCLUDE_DIRECTORIES(kokkoscontainers + ${KOKKOS_TOP_BUILD_DIR} + ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_SOURCE_DIR} +) +KOKKOS_LINK_INTERNAL_LIBRARY(kokkoscontainers kokkoscore) + +#----------------------------------------------------------------------------- + diff --git a/lib/kokkos/containers/src/Kokkos_Bitset.hpp b/lib/kokkos/containers/src/Kokkos_Bitset.hpp index 4d78430fc634ed7dad2a904a4dc5282e16501637..3596c7653ac1de08f748ef35de2070af4f16630f 100644 --- a/lib/kokkos/containers/src/Kokkos_Bitset.hpp +++ b/lib/kokkos/containers/src/Kokkos_Bitset.hpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -53,27 +54,25 @@ namespace Kokkos { -template +template class Bitset; -template +template class ConstBitset; template -void deep_copy( Bitset & dst, Bitset const& src); +void deep_copy(Bitset& dst, Bitset const& src); template -void deep_copy( Bitset & dst, ConstBitset const& src); +void deep_copy(Bitset& dst, ConstBitset const& src); template -void deep_copy( ConstBitset & dst, ConstBitset const& src); - +void deep_copy(ConstBitset& dst, ConstBitset const& src); /// A thread safe view to a bitset template -class Bitset -{ -public: +class Bitset { + public: typedef Device execution_space; typedef unsigned size_type; @@ -81,98 +80,88 @@ public: enum { MOVE_HINT_BACKWARD = 2u }; enum { - BIT_SCAN_FORWARD_MOVE_HINT_FORWARD = 0u - , BIT_SCAN_REVERSE_MOVE_HINT_FORWARD = BIT_SCAN_REVERSE - , BIT_SCAN_FORWARD_MOVE_HINT_BACKWARD = MOVE_HINT_BACKWARD - , BIT_SCAN_REVERSE_MOVE_HINT_BACKWARD = BIT_SCAN_REVERSE | MOVE_HINT_BACKWARD + BIT_SCAN_FORWARD_MOVE_HINT_FORWARD = 0u, + BIT_SCAN_REVERSE_MOVE_HINT_FORWARD = BIT_SCAN_REVERSE, + BIT_SCAN_FORWARD_MOVE_HINT_BACKWARD = MOVE_HINT_BACKWARD, + BIT_SCAN_REVERSE_MOVE_HINT_BACKWARD = BIT_SCAN_REVERSE | MOVE_HINT_BACKWARD }; -private: - enum { block_size = static_cast(sizeof(unsigned)*CHAR_BIT) }; - enum { block_mask = block_size-1u }; + private: + enum { block_size = static_cast(sizeof(unsigned) * CHAR_BIT) }; + enum { block_mask = block_size - 1u }; enum { block_shift = Kokkos::Impl::integral_power_of_two(block_size) }; -public: - - + public: /// constructor /// arg_size := number of bit in set Bitset(unsigned arg_size = 0u) - : m_size(arg_size) - , m_last_block_mask(0u) - , m_blocks("Bitset", ((m_size + block_mask) >> block_shift) ) - { - for (int i=0, end = static_cast(m_size & block_mask); i < end; ++i) { + : m_size(arg_size), + m_last_block_mask(0u), + m_blocks("Bitset", ((m_size + block_mask) >> block_shift)) { + for (int i = 0, end = static_cast(m_size & block_mask); i < end; ++i) { m_last_block_mask |= 1u << i; } } KOKKOS_INLINE_FUNCTION - Bitset (const Bitset&) = default; + Bitset(const Bitset&) = default; KOKKOS_INLINE_FUNCTION - Bitset& operator= (const Bitset&) = default; + Bitset& operator=(const Bitset&) = default; KOKKOS_INLINE_FUNCTION - Bitset (Bitset&&) = default; + Bitset(Bitset&&) = default; KOKKOS_INLINE_FUNCTION - Bitset& operator= (Bitset&&) = default; - + Bitset& operator=(Bitset&&) = default; + KOKKOS_INLINE_FUNCTION - ~Bitset () = default; + ~Bitset() = default; /// number of bits in the set /// can be call from the host or the device KOKKOS_FORCEINLINE_FUNCTION - unsigned size() const - { return m_size; } + unsigned size() const { return m_size; } /// number of bits which are set to 1 /// can only be called from the host - unsigned count() const - { - Impl::BitsetCount< Bitset > f(*this); + unsigned count() const { + Impl::BitsetCount > f(*this); return f.apply(); } /// set all bits to 1 /// can only be called from the host - void set() - { - Kokkos::deep_copy(m_blocks, ~0u ); + void set() { + Kokkos::deep_copy(m_blocks, ~0u); if (m_last_block_mask) { - //clear the unused bits in the last block - typedef Kokkos::Impl::DeepCopy< typename execution_space::memory_space, Kokkos::HostSpace > raw_deep_copy; - raw_deep_copy( m_blocks.data() + (m_blocks.extent(0) -1u), &m_last_block_mask, sizeof(unsigned)); + // clear the unused bits in the last block + typedef Kokkos::Impl::DeepCopy + raw_deep_copy; + raw_deep_copy(m_blocks.data() + (m_blocks.extent(0) - 1u), + &m_last_block_mask, sizeof(unsigned)); } } /// set all bits to 0 /// can only be called from the host - void reset() - { - Kokkos::deep_copy(m_blocks, 0u ); - } + void reset() { Kokkos::deep_copy(m_blocks, 0u); } /// set all bits to 0 /// can only be called from the host - void clear() - { - Kokkos::deep_copy(m_blocks, 0u ); - } + void clear() { Kokkos::deep_copy(m_blocks, 0u); } /// set i'th bit to 1 /// can only be called from the device KOKKOS_FORCEINLINE_FUNCTION - bool set( unsigned i ) const - { - if ( i < m_size ) { - unsigned * block_ptr = &m_blocks[ i >> block_shift ]; - const unsigned mask = 1u << static_cast( i & block_mask ); + bool set(unsigned i) const { + if (i < m_size) { + unsigned* block_ptr = &m_blocks[i >> block_shift]; + const unsigned mask = 1u << static_cast(i & block_mask); - return !( atomic_fetch_or( block_ptr, mask ) & mask ); + return !(atomic_fetch_or(block_ptr, mask) & mask); } return false; } @@ -180,13 +169,12 @@ public: /// set i'th bit to 0 /// can only be called from the device KOKKOS_FORCEINLINE_FUNCTION - bool reset( unsigned i ) const - { - if ( i < m_size ) { - unsigned * block_ptr = &m_blocks[ i >> block_shift ]; - const unsigned mask = 1u << static_cast( i & block_mask ); + bool reset(unsigned i) const { + if (i < m_size) { + unsigned* block_ptr = &m_blocks[i >> block_shift]; + const unsigned mask = 1u << static_cast(i & block_mask); - return atomic_fetch_and( block_ptr, ~mask ) & mask; + return atomic_fetch_and(block_ptr, ~mask) & mask; } return false; } @@ -194,11 +182,10 @@ public: /// return true if the i'th bit set to 1 /// can only be called from the device KOKKOS_FORCEINLINE_FUNCTION - bool test( unsigned i ) const - { - if ( i < m_size ) { - const unsigned block = volatile_load(&m_blocks[ i >> block_shift ]); - const unsigned mask = 1u << static_cast( i & block_mask ); + bool test(unsigned i) const { + if (i < m_size) { + const unsigned block = volatile_load(&m_blocks[i >> block_shift]); + const unsigned mask = 1u << static_cast(i & block_mask); return block & mask; } return false; @@ -208,90 +195,93 @@ public: /// returns the max number of times those functions should be call /// when searching for an available bit KOKKOS_FORCEINLINE_FUNCTION - unsigned max_hint() const - { - return m_blocks.extent(0); - } + unsigned max_hint() const { return m_blocks.extent(0); } /// find a bit set to 1 near the hint - /// returns a pair< bool, unsigned> where if result.first is true then result.second is the bit found - /// and if result.first is false the result.second is a new hint + /// returns a pair< bool, unsigned> where if result.first is true then + /// result.second is the bit found and if result.first is false the + /// result.second is a new hint KOKKOS_INLINE_FUNCTION - Kokkos::pair find_any_set_near( unsigned hint , unsigned scan_direction = BIT_SCAN_FORWARD_MOVE_HINT_FORWARD ) const - { - const unsigned block_idx = (hint >> block_shift) < m_blocks.extent(0) ? (hint >> block_shift) : 0; + Kokkos::pair find_any_set_near( + unsigned hint, + unsigned scan_direction = BIT_SCAN_FORWARD_MOVE_HINT_FORWARD) const { + const unsigned block_idx = + (hint >> block_shift) < m_blocks.extent(0) ? (hint >> block_shift) : 0; const unsigned offset = hint & block_mask; - unsigned block = volatile_load(&m_blocks[ block_idx ]); - block = !m_last_block_mask || (block_idx < (m_blocks.extent(0)-1)) ? block : block & m_last_block_mask ; + unsigned block = volatile_load(&m_blocks[block_idx]); + block = !m_last_block_mask || (block_idx < (m_blocks.extent(0) - 1)) + ? block + : block & m_last_block_mask; return find_any_helper(block_idx, offset, block, scan_direction); } /// find a bit set to 0 near the hint - /// returns a pair< bool, unsigned> where if result.first is true then result.second is the bit found - /// and if result.first is false the result.second is a new hint + /// returns a pair< bool, unsigned> where if result.first is true then + /// result.second is the bit found and if result.first is false the + /// result.second is a new hint KOKKOS_INLINE_FUNCTION - Kokkos::pair find_any_unset_near( unsigned hint , unsigned scan_direction = BIT_SCAN_FORWARD_MOVE_HINT_FORWARD ) const - { + Kokkos::pair find_any_unset_near( + unsigned hint, + unsigned scan_direction = BIT_SCAN_FORWARD_MOVE_HINT_FORWARD) const { const unsigned block_idx = hint >> block_shift; - const unsigned offset = hint & block_mask; - unsigned block = volatile_load(&m_blocks[ block_idx ]); - block = !m_last_block_mask || (block_idx < (m_blocks.extent(0)-1) ) ? ~block : ~block & m_last_block_mask ; + const unsigned offset = hint & block_mask; + unsigned block = volatile_load(&m_blocks[block_idx]); + block = !m_last_block_mask || (block_idx < (m_blocks.extent(0) - 1)) + ? ~block + : ~block & m_last_block_mask; return find_any_helper(block_idx, offset, block, scan_direction); } -private: - + private: KOKKOS_FORCEINLINE_FUNCTION - Kokkos::pair find_any_helper(unsigned block_idx, unsigned offset, unsigned block, unsigned scan_direction) const - { - Kokkos::pair result( block > 0u, 0); + Kokkos::pair find_any_helper(unsigned block_idx, + unsigned offset, unsigned block, + unsigned scan_direction) const { + Kokkos::pair result(block > 0u, 0); if (!result.first) { - result.second = update_hint( block_idx, offset, scan_direction ); - } - else { - result.second = scan_block( (block_idx << block_shift) - , offset - , block - , scan_direction - ); + result.second = update_hint(block_idx, offset, scan_direction); + } else { + result.second = + scan_block((block_idx << block_shift), offset, block, scan_direction); } return result; } - KOKKOS_FORCEINLINE_FUNCTION - unsigned scan_block(unsigned block_start, int offset, unsigned block, unsigned scan_direction ) const - { - offset = !(scan_direction & BIT_SCAN_REVERSE) ? offset : (offset + block_mask) & block_mask; + unsigned scan_block(unsigned block_start, int offset, unsigned block, + unsigned scan_direction) const { + offset = !(scan_direction & BIT_SCAN_REVERSE) + ? offset + : (offset + block_mask) & block_mask; block = Impl::rotate_right(block, offset); - return ((( !(scan_direction & BIT_SCAN_REVERSE) ? - Impl::bit_scan_forward(block) : - ::Kokkos::log2(block) - ) + offset - ) & block_mask - ) + block_start; + return (((!(scan_direction & BIT_SCAN_REVERSE) + ? Impl::bit_scan_forward(block) + : ::Kokkos::log2(block)) + + offset) & + block_mask) + + block_start; } KOKKOS_FORCEINLINE_FUNCTION - unsigned update_hint( long long block_idx, unsigned offset, unsigned scan_direction ) const - { + unsigned update_hint(long long block_idx, unsigned offset, + unsigned scan_direction) const { block_idx += scan_direction & MOVE_HINT_BACKWARD ? -1 : 1; block_idx = block_idx >= 0 ? block_idx : m_blocks.extent(0) - 1; - block_idx = block_idx < static_cast(m_blocks.extent(0)) ? block_idx : 0; + block_idx = + block_idx < static_cast(m_blocks.extent(0)) ? block_idx : 0; - return static_cast(block_idx)*block_size + offset; + return static_cast(block_idx) * block_size + offset; } -private: - + private: unsigned m_size; unsigned m_last_block_mask; - View< unsigned *, execution_space, MemoryTraits > m_blocks; + View > m_blocks; -private: + private: template friend class Bitset; @@ -302,87 +292,72 @@ private: friend struct Impl::BitsetCount; template - friend void deep_copy( Bitset & dst, Bitset const& src); + friend void deep_copy(Bitset& dst, Bitset const& src); template - friend void deep_copy( Bitset & dst, ConstBitset const& src); + friend void deep_copy(Bitset& dst, + ConstBitset const& src); }; /// a thread-safe view to a const bitset /// i.e. can only test bits template -class ConstBitset -{ -public: +class ConstBitset { + public: typedef Device execution_space; typedef unsigned size_type; -private: - enum { block_size = static_cast(sizeof(unsigned)*CHAR_BIT) }; - enum { block_mask = block_size -1u }; + private: + enum { block_size = static_cast(sizeof(unsigned) * CHAR_BIT) }; + enum { block_mask = block_size - 1u }; enum { block_shift = Kokkos::Impl::integral_power_of_two(block_size) }; -public: - ConstBitset() - : m_size (0) - {} + public: + ConstBitset() : m_size(0) {} ConstBitset(Bitset const& rhs) - : m_size(rhs.m_size) - , m_blocks(rhs.m_blocks) - {} + : m_size(rhs.m_size), m_blocks(rhs.m_blocks) {} ConstBitset(ConstBitset const& rhs) - : m_size( rhs.m_size ) - , m_blocks( rhs.m_blocks ) - {} + : m_size(rhs.m_size), m_blocks(rhs.m_blocks) {} - ConstBitset & operator = (Bitset const & rhs) - { - this->m_size = rhs.m_size; + ConstBitset& operator=(Bitset const& rhs) { + this->m_size = rhs.m_size; this->m_blocks = rhs.m_blocks; return *this; } - ConstBitset & operator = (ConstBitset const & rhs) - { - this->m_size = rhs.m_size; + ConstBitset& operator=(ConstBitset const& rhs) { + this->m_size = rhs.m_size; this->m_blocks = rhs.m_blocks; return *this; } - KOKKOS_FORCEINLINE_FUNCTION - unsigned size() const - { - return m_size; - } + unsigned size() const { return m_size; } - unsigned count() const - { - Impl::BitsetCount< ConstBitset > f(*this); + unsigned count() const { + Impl::BitsetCount > f(*this); return f.apply(); } KOKKOS_FORCEINLINE_FUNCTION - bool test( unsigned i ) const - { - if ( i < m_size ) { - const unsigned block = m_blocks[ i >> block_shift ]; - const unsigned mask = 1u << static_cast( i & block_mask ); + bool test(unsigned i) const { + if (i < m_size) { + const unsigned block = m_blocks[i >> block_shift]; + const unsigned mask = 1u << static_cast(i & block_mask); return block & mask; } return false; } -private: - + private: unsigned m_size; - View< const unsigned *, execution_space, MemoryTraits > m_blocks; + View > m_blocks; -private: + private: template friend class ConstBitset; @@ -390,47 +365,56 @@ private: friend struct Impl::BitsetCount; template - friend void deep_copy( Bitset & dst, ConstBitset const& src); + friend void deep_copy(Bitset& dst, + ConstBitset const& src); template - friend void deep_copy( ConstBitset & dst, ConstBitset const& src); + friend void deep_copy(ConstBitset& dst, + ConstBitset const& src); }; - template -void deep_copy( Bitset & dst, Bitset const& src) -{ +void deep_copy(Bitset& dst, Bitset const& src) { if (dst.size() != src.size()) { - throw std::runtime_error("Error: Cannot deep_copy bitsets of different sizes!"); + throw std::runtime_error( + "Error: Cannot deep_copy bitsets of different sizes!"); } - typedef Kokkos::Impl::DeepCopy< typename DstDevice::memory_space, typename SrcDevice::memory_space > raw_deep_copy; - raw_deep_copy(dst.m_blocks.data(), src.m_blocks.data(), sizeof(unsigned)*src.m_blocks.extent(0)); + typedef Kokkos::Impl::DeepCopy + raw_deep_copy; + raw_deep_copy(dst.m_blocks.data(), src.m_blocks.data(), + sizeof(unsigned) * src.m_blocks.extent(0)); } template -void deep_copy( Bitset & dst, ConstBitset const& src) -{ +void deep_copy(Bitset& dst, ConstBitset const& src) { if (dst.size() != src.size()) { - throw std::runtime_error("Error: Cannot deep_copy bitsets of different sizes!"); + throw std::runtime_error( + "Error: Cannot deep_copy bitsets of different sizes!"); } - typedef Kokkos::Impl::DeepCopy< typename DstDevice::memory_space, typename SrcDevice::memory_space > raw_deep_copy; - raw_deep_copy(dst.m_blocks.data(), src.m_blocks.data(), sizeof(unsigned)*src.m_blocks.extent(0)); + typedef Kokkos::Impl::DeepCopy + raw_deep_copy; + raw_deep_copy(dst.m_blocks.data(), src.m_blocks.data(), + sizeof(unsigned) * src.m_blocks.extent(0)); } template -void deep_copy( ConstBitset & dst, ConstBitset const& src) -{ +void deep_copy(ConstBitset& dst, ConstBitset const& src) { if (dst.size() != src.size()) { - throw std::runtime_error("Error: Cannot deep_copy bitsets of different sizes!"); + throw std::runtime_error( + "Error: Cannot deep_copy bitsets of different sizes!"); } - typedef Kokkos::Impl::DeepCopy< typename DstDevice::memory_space, typename SrcDevice::memory_space > raw_deep_copy; - raw_deep_copy(dst.m_blocks.data(), src.m_blocks.data(), sizeof(unsigned)*src.m_blocks.extent(0)); + typedef Kokkos::Impl::DeepCopy + raw_deep_copy; + raw_deep_copy(dst.m_blocks.data(), src.m_blocks.data(), + sizeof(unsigned) * src.m_blocks.extent(0)); } -} // namespace Kokkos - -#endif //KOKKOS_BITSET_HPP +} // namespace Kokkos +#endif // KOKKOS_BITSET_HPP diff --git a/lib/kokkos/containers/src/Kokkos_DualView.hpp b/lib/kokkos/containers/src/Kokkos_DualView.hpp index d9b14d67a2f02854962c2f3d127f1d80da67a46d..d8a3ebc1ae947b6ef617d106de8d954ca4bfbc77 100644 --- a/lib/kokkos/containers/src/Kokkos_DualView.hpp +++ b/lib/kokkos/containers/src/Kokkos_DualView.hpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -90,47 +91,41 @@ namespace Kokkos { * behavior. Please see the documentation of Kokkos::View for * examples. The default suffices for most users. */ -template< class DataType , - class Arg1Type = void , - class Arg2Type = void , +template -class DualView : public ViewTraits< DataType , Arg1Type , Arg2Type, Arg3Type > -{ -template< class , class , class , class > friend class DualView ; -public: +class DualView : public ViewTraits { + template + friend class DualView; + + public: //! \name Typedefs for device types and various Kokkos::View specializations. //@{ - typedef ViewTraits< DataType , Arg1Type , Arg2Type, Arg3Type > traits ; + typedef ViewTraits traits; //! The Kokkos Host Device type; - typedef typename traits::host_mirror_space host_mirror_space ; + typedef typename traits::host_mirror_space host_mirror_space; //! The type of a Kokkos::View on the device. - typedef View< typename traits::data_type , - Arg1Type , - Arg2Type , - Arg3Type > t_dev ; + typedef View t_dev; /// \typedef t_host /// \brief The type of a Kokkos::View host mirror of \c t_dev. - typedef typename t_dev::HostMirror t_host ; + typedef typename t_dev::HostMirror t_host; //! The type of a const View on the device. //! The type of a Kokkos::View on the device. - typedef View< typename traits::const_data_type , - Arg1Type , - Arg2Type , - Arg3Type > t_dev_const ; + typedef View + t_dev_const; /// \typedef t_host_const /// \brief The type of a const View host mirror of \c t_dev_const. typedef typename t_dev_const::HostMirror t_host_const; //! The type of a const, random-access View on the device. - typedef View< typename traits::const_data_type , - typename traits::array_layout , - typename traits::device_type , - Kokkos::MemoryTraits > t_dev_const_randomread ; + typedef View > + t_dev_const_randomread; /// \typedef t_host_const_randomread /// \brief The type of a const, random-access View host mirror of @@ -138,39 +133,36 @@ public: typedef typename t_dev_const_randomread::HostMirror t_host_const_randomread; //! The type of an unmanaged View on the device. - typedef View< typename traits::data_type , - typename traits::array_layout , - typename traits::device_type , - MemoryUnmanaged> t_dev_um; + typedef View + t_dev_um; //! The type of an unmanaged View host mirror of \c t_dev_um. - typedef View< typename t_host::data_type , - typename t_host::array_layout , - typename t_host::device_type , - MemoryUnmanaged> t_host_um; + typedef View + t_host_um; //! The type of a const unmanaged View on the device. - typedef View< typename traits::const_data_type , - typename traits::array_layout , - typename traits::device_type , - MemoryUnmanaged> t_dev_const_um; + typedef View + t_dev_const_um; //! The type of a const unmanaged View host mirror of \c t_dev_const_um. - typedef View t_host_const_um; + typedef View + t_host_const_um; //! The type of a const, random-access View on the device. - typedef View< typename t_host::const_data_type , - typename t_host::array_layout , - typename t_host::device_type , - Kokkos::MemoryTraits > t_dev_const_randomread_um ; + typedef View > + t_dev_const_randomread_um; /// \typedef t_host_const_randomread /// \brief The type of a const, random-access View host mirror of /// \c t_dev_const_randomread. - typedef typename t_dev_const_randomread::HostMirror t_host_const_randomread_um; + typedef + typename t_dev_const_randomread::HostMirror t_host_const_randomread_um; //@} //! \name The two View instances. @@ -184,18 +176,20 @@ public: //@{ #ifndef KOKKOS_ENABLE_DEPRECATED_CODE -protected: + protected: // modified_flags[0] -> host // modified_flags[1] -> device - typedef View t_modified_flags; + typedef View t_modified_flags; t_modified_flags modified_flags; -public: + public: #else - typedef View t_modified_flags; - typedef View t_modified_flag; + typedef View + t_modified_flags; + typedef View + t_modified_flag; t_modified_flags modified_flags; - t_modified_flag modified_host,modified_device; + t_modified_flag modified_host, modified_device; #endif //@} @@ -208,11 +202,11 @@ public: /// default constructors. The "modified" flags are both initialized /// to "unmodified." #ifndef KOKKOS_ENABLE_DEPRECATED_CODE - DualView () = default; + DualView() = default; #else - DualView ():modified_flags (t_modified_flags("DualView::modified_flags")) { - modified_host = t_modified_flag(modified_flags,0); - modified_device = t_modified_flag(modified_flags,1); + DualView() : modified_flags(t_modified_flags("DualView::modified_flags")) { + modified_host = t_modified_flag(modified_flags, 0); + modified_device = t_modified_flag(modified_flags, 1); } #endif @@ -225,52 +219,52 @@ public: /// View objects. For example, if the View has three dimensions, /// the first three integer arguments will be nonzero, and you may /// omit the integer arguments that follow. - DualView (const std::string& label, - const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) - : d_view (label, n0, n1, n2, n3, n4, n5, n6, n7) - , h_view (create_mirror_view (d_view)) // without UVM, host View mirrors - , modified_flags (t_modified_flags("DualView::modified_flags")) - { + DualView(const std::string& label, + const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) + : d_view(label, n0, n1, n2, n3, n4, n5, n6, n7), + h_view(create_mirror_view(d_view)) // without UVM, host View mirrors + , + modified_flags(t_modified_flags("DualView::modified_flags")) { #ifdef KOKKOS_ENABLE_DEPRECATED_CODE - modified_host = t_modified_flag(modified_flags,0); - modified_device = t_modified_flag(modified_flags,1); + modified_host = t_modified_flag(modified_flags, 0); + modified_device = t_modified_flag(modified_flags, 1); #endif } //! Copy constructor (shallow copy) - template - DualView (const DualView& src) : - d_view (src.d_view), - h_view (src.h_view), - modified_flags (src.modified_flags) + template + DualView(const DualView& src) + : d_view(src.d_view), + h_view(src.h_view), + modified_flags(src.modified_flags) #ifdef KOKKOS_ENABLE_DEPRECATED_CODE - , modified_host(src.modified_host) - , modified_device(src.modified_device) + , + modified_host(src.modified_host), + modified_device(src.modified_device) #endif - {} + { + } //! Subview constructor - template< class SD, class S1 , class S2 , class S3 - , class Arg0 , class ... Args > - DualView( const DualView & src - , const Arg0 & arg0 - , Args ... args - ) - : d_view( Kokkos::subview( src.d_view , arg0 , args ... ) ) - , h_view( Kokkos::subview( src.h_view , arg0 , args ... ) ) - , modified_flags (src.modified_flags) + template + DualView(const DualView& src, const Arg0& arg0, Args... args) + : d_view(Kokkos::subview(src.d_view, arg0, args...)), + h_view(Kokkos::subview(src.h_view, arg0, args...)), + modified_flags(src.modified_flags) #ifdef KOKKOS_ENABLE_DEPRECATED_CODE - , modified_host(src.modified_host) - , modified_device(src.modified_device) + , + modified_host(src.modified_host), + modified_device(src.modified_device) #endif - {} + { + } /// \brief Create DualView from existing device and host View objects. /// @@ -282,34 +276,34 @@ public: /// /// \param d_view_ Device View /// \param h_view_ Host View (must have type t_host = t_dev::HostMirror) - DualView (const t_dev& d_view_, const t_host& h_view_) : - d_view (d_view_), - h_view (h_view_), - modified_flags (t_modified_flags("DualView::modified_flags")) - { - if ( int(d_view.rank) != int(h_view.rank) || - d_view.extent(0) != h_view.extent(0) || - d_view.extent(1) != h_view.extent(1) || - d_view.extent(2) != h_view.extent(2) || - d_view.extent(3) != h_view.extent(3) || - d_view.extent(4) != h_view.extent(4) || - d_view.extent(5) != h_view.extent(5) || - d_view.extent(6) != h_view.extent(6) || - d_view.extent(7) != h_view.extent(7) || - d_view.stride_0() != h_view.stride_0() || - d_view.stride_1() != h_view.stride_1() || - d_view.stride_2() != h_view.stride_2() || - d_view.stride_3() != h_view.stride_3() || - d_view.stride_4() != h_view.stride_4() || - d_view.stride_5() != h_view.stride_5() || - d_view.stride_6() != h_view.stride_6() || - d_view.stride_7() != h_view.stride_7() || - d_view.span() != h_view.span() ) { - Kokkos::Impl::throw_runtime_exception("DualView constructed with incompatible views"); + DualView(const t_dev& d_view_, const t_host& h_view_) + : d_view(d_view_), + h_view(h_view_), + modified_flags(t_modified_flags("DualView::modified_flags")) { + if (int(d_view.rank) != int(h_view.rank) || + d_view.extent(0) != h_view.extent(0) || + d_view.extent(1) != h_view.extent(1) || + d_view.extent(2) != h_view.extent(2) || + d_view.extent(3) != h_view.extent(3) || + d_view.extent(4) != h_view.extent(4) || + d_view.extent(5) != h_view.extent(5) || + d_view.extent(6) != h_view.extent(6) || + d_view.extent(7) != h_view.extent(7) || + d_view.stride_0() != h_view.stride_0() || + d_view.stride_1() != h_view.stride_1() || + d_view.stride_2() != h_view.stride_2() || + d_view.stride_3() != h_view.stride_3() || + d_view.stride_4() != h_view.stride_4() || + d_view.stride_5() != h_view.stride_5() || + d_view.stride_6() != h_view.stride_6() || + d_view.stride_7() != h_view.stride_7() || + d_view.span() != h_view.span()) { + Kokkos::Impl::throw_runtime_exception( + "DualView constructed with incompatible views"); } #ifdef KOKKOS_ENABLE_DEPRECATED_CODE - modified_host = t_modified_flag(modified_flags,0); - modified_device = t_modified_flag(modified_flags,1); + modified_host = t_modified_flag(modified_flags, 0); + modified_device = t_modified_flag(modified_flags, 1); #endif } @@ -326,119 +320,133 @@ public: /// /// For example, suppose you create a DualView on Cuda, like this: /// \code - /// typedef Kokkos::DualView dual_view_type; - /// dual_view_type DV ("my dual view", 100); - /// \endcode - /// If you want to get the CUDA device View, do this: - /// \code - /// typename dual_view_type::t_dev cudaView = DV.view (); - /// \endcode - /// and if you want to get the host mirror of that View, do this: - /// \code - /// typedef typename Kokkos::HostSpace::execution_space host_device_type; - /// typename dual_view_type::t_host hostView = DV.view (); - /// \endcode - template< class Device > - KOKKOS_INLINE_FUNCTION - const typename Impl::if_c< - std::is_same::value, - t_dev, - t_host>::type& view () const - { - #ifndef KOKKOS_ENABLE_DEPRECATED_CODE - constexpr bool device_is_memspace = std::is_same::value; - constexpr bool device_is_execspace = std::is_same::value; - constexpr bool device_exec_is_t_dev_exec = std::is_same::value; - constexpr bool device_mem_is_t_dev_mem = std::is_same::value; - constexpr bool device_exec_is_t_host_exec = std::is_same::value; - constexpr bool device_mem_is_t_host_mem = std::is_same::value; - constexpr bool device_is_t_host_device = std::is_same::value; - constexpr bool device_is_t_dev_device = std::is_same::value; + /// typedef Kokkos::DualView + /// dual_view_type; dual_view_type DV ("my dual view", 100); \endcode If you + /// want to get the CUDA device View, do this: \code typename + /// dual_view_type::t_dev cudaView = DV.view (); \endcode and if + /// you want to get the host mirror of that View, do this: \code typedef + /// typename Kokkos::HostSpace::execution_space host_device_type; typename + /// dual_view_type::t_host hostView = DV.view (); \endcode + template + KOKKOS_INLINE_FUNCTION const typename Impl::if_c< + std::is_same::value, + t_dev, t_host>::type& + view() const { +#ifndef KOKKOS_ENABLE_DEPRECATED_CODE + constexpr bool device_is_memspace = + std::is_same::value; + constexpr bool device_is_execspace = + std::is_same::value; + constexpr bool device_exec_is_t_dev_exec = + std::is_same::value; + constexpr bool device_mem_is_t_dev_mem = + std::is_same::value; + constexpr bool device_exec_is_t_host_exec = + std::is_same::value; + constexpr bool device_mem_is_t_host_mem = + std::is_same::value; + constexpr bool device_is_t_host_device = + std::is_same::value; + constexpr bool device_is_t_dev_device = + std::is_same::value; static_assert( device_is_t_dev_device || device_is_t_host_device || - (device_is_memspace && (device_mem_is_t_dev_mem || device_mem_is_t_host_mem) ) || - (device_is_execspace && (device_exec_is_t_dev_exec || device_exec_is_t_host_exec) ) || - ( - (!device_is_execspace && !device_is_memspace) && ( - (device_mem_is_t_dev_mem || device_mem_is_t_host_mem) || - (device_exec_is_t_dev_exec || device_exec_is_t_host_exec) - ) - ) - , - "Template parameter to .view() must exactly match one of the DualView's device types or one of the execution or memory spaces"); - #endif - - return Impl::if_c< - std::is_same< - typename t_dev::memory_space, - typename Device::memory_space>::value, - t_dev, - t_host >::select (d_view , h_view); + (device_is_memspace && + (device_mem_is_t_dev_mem || device_mem_is_t_host_mem)) || + (device_is_execspace && + (device_exec_is_t_dev_exec || device_exec_is_t_host_exec)) || + ((!device_is_execspace && !device_is_memspace) && + ((device_mem_is_t_dev_mem || device_mem_is_t_host_mem) || + (device_exec_is_t_dev_exec || device_exec_is_t_host_exec))), + "Template parameter to .view() must exactly match one of the " + "DualView's device types or one of the execution or memory spaces"); +#endif + + return Impl::if_c::value, + t_dev, t_host>::select(d_view, h_view); } KOKKOS_INLINE_FUNCTION - t_host view_host() const { - return h_view; - } + t_host view_host() const { return h_view; } KOKKOS_INLINE_FUNCTION - t_dev view_device() const { - return d_view; - } + t_dev view_device() const { return d_view; } - template + template static int get_device_side() { - constexpr bool device_is_memspace = std::is_same::value; - constexpr bool device_is_execspace = std::is_same::value; - constexpr bool device_exec_is_t_dev_exec = std::is_same::value; - constexpr bool device_mem_is_t_dev_mem = std::is_same::value; - constexpr bool device_exec_is_t_host_exec = std::is_same::value; - constexpr bool device_mem_is_t_host_mem = std::is_same::value; - constexpr bool device_is_t_host_device = std::is_same::value; - constexpr bool device_is_t_dev_device = std::is_same::value; - - #ifndef KOKKOS_ENABLE_DEPRECATED_CODE + constexpr bool device_is_memspace = + std::is_same::value; + constexpr bool device_is_execspace = + std::is_same::value; + constexpr bool device_exec_is_t_dev_exec = + std::is_same::value; + constexpr bool device_mem_is_t_dev_mem = + std::is_same::value; + constexpr bool device_exec_is_t_host_exec = + std::is_same::value; + constexpr bool device_mem_is_t_host_mem = + std::is_same::value; + constexpr bool device_is_t_host_device = + std::is_same::value; + constexpr bool device_is_t_dev_device = + std::is_same::value; + +#ifndef KOKKOS_ENABLE_DEPRECATED_CODE static_assert( device_is_t_dev_device || device_is_t_host_device || - (device_is_memspace && (device_mem_is_t_dev_mem || device_mem_is_t_host_mem) ) || - (device_is_execspace && (device_exec_is_t_dev_exec || device_exec_is_t_host_exec) ) || - ( - (!device_is_execspace && !device_is_memspace) && ( - (device_mem_is_t_dev_mem || device_mem_is_t_host_mem) || - (device_exec_is_t_dev_exec || device_exec_is_t_host_exec) - ) - ) - , - "Template parameter to .sync() must exactly match one of the DualView's device types or one of the execution or memory spaces"); - #endif + (device_is_memspace && + (device_mem_is_t_dev_mem || device_mem_is_t_host_mem)) || + (device_is_execspace && + (device_exec_is_t_dev_exec || device_exec_is_t_host_exec)) || + ((!device_is_execspace && !device_is_memspace) && + ((device_mem_is_t_dev_mem || device_mem_is_t_host_mem) || + (device_exec_is_t_dev_exec || device_exec_is_t_host_exec))), + "Template parameter to .sync() must exactly match one of the " + "DualView's device types or one of the execution or memory spaces"); +#endif - #ifndef KOKKOS_ENABLE_DEPRECATED_CODE +#ifndef KOKKOS_ENABLE_DEPRECATED_CODE int dev = -1; - #else +#else int dev = 0; - #endif - if(device_is_t_dev_device) dev = 1; - else if(device_is_t_host_device) dev = 0; +#endif + if (device_is_t_dev_device) + dev = 1; + else if (device_is_t_host_device) + dev = 0; else { - if(device_is_memspace) { - if(device_mem_is_t_dev_mem) dev = 1; - if(device_mem_is_t_host_mem) dev = 0; - if(device_mem_is_t_host_mem && device_mem_is_t_dev_mem) dev = -1; + if (device_is_memspace) { + if (device_mem_is_t_dev_mem) dev = 1; + if (device_mem_is_t_host_mem) dev = 0; + if (device_mem_is_t_host_mem && device_mem_is_t_dev_mem) dev = -1; } - if(device_is_execspace) { - if(device_exec_is_t_dev_exec) dev = 1; - if(device_exec_is_t_host_exec) dev = 0; - if(device_exec_is_t_host_exec && device_exec_is_t_dev_exec) dev = -1; + if (device_is_execspace) { + if (device_exec_is_t_dev_exec) dev = 1; + if (device_exec_is_t_host_exec) dev = 0; + if (device_exec_is_t_host_exec && device_exec_is_t_dev_exec) dev = -1; } - if(!device_is_execspace && !device_is_memspace) { - if(device_mem_is_t_dev_mem) dev = 1; - if(device_mem_is_t_host_mem) dev = 0; - if(device_mem_is_t_host_mem && device_mem_is_t_dev_mem) dev = -1; - if(device_exec_is_t_dev_exec) dev = 1; - if(device_exec_is_t_host_exec) dev = 0; - if(device_exec_is_t_host_exec && device_exec_is_t_dev_exec) dev = -1; + if (!device_is_execspace && !device_is_memspace) { + if (device_mem_is_t_dev_mem) dev = 1; + if (device_mem_is_t_host_mem) dev = 0; + if (device_mem_is_t_host_mem && device_mem_is_t_dev_mem) dev = -1; + if (device_exec_is_t_dev_exec) dev = 1; + if (device_exec_is_t_host_exec) dev = 0; + if (device_exec_is_t_host_exec && device_exec_is_t_dev_exec) dev = -1; } } return dev; @@ -461,88 +469,94 @@ public: /// the data in either View. You must manually mark modified data /// as modified, by calling the modify() method with the /// appropriate template parameter. - template - void sync( const typename Impl::enable_if< - ( std::is_same< typename traits::data_type , typename traits::non_const_data_type>::value) || - ( std::is_same< Device , int>::value) - , int >::type& = 0) - { - if(modified_flags.data()==NULL) return; + template + void sync(const typename Impl::enable_if< + (std::is_same::value) || + (std::is_same::value), + int>::type& = 0) { + if (modified_flags.data() == NULL) return; int dev = get_device_side(); - if (dev == 1) { // if Device is the same as DualView's device type + if (dev == 1) { // if Device is the same as DualView's device type if ((modified_flags(0) > 0) && (modified_flags(0) >= modified_flags(1))) { - deep_copy (d_view, h_view); + deep_copy(d_view, h_view); modified_flags(0) = modified_flags(1) = 0; } } - if (dev == 0) { // hopefully Device is the same as DualView's host type + if (dev == 0) { // hopefully Device is the same as DualView's host type if ((modified_flags(1) > 0) && (modified_flags(1) >= modified_flags(0))) { - deep_copy (h_view, d_view); + deep_copy(h_view, d_view); modified_flags(0) = modified_flags(1) = 0; } } - if(std::is_same::value) { + if (std::is_same::value) { typename t_dev::execution_space().fence(); typename t_host::execution_space().fence(); } } - template - void sync ( const typename Impl::enable_if< - ( ! std::is_same< typename traits::data_type , typename traits::non_const_data_type>::value ) || - ( std::is_same< Device , int>::value) - , int >::type& = 0 ) - { - if(modified_flags.data()==NULL) return; + template + void sync(const typename Impl::enable_if< + (!std::is_same::value) || + (std::is_same::value), + int>::type& = 0) { + if (modified_flags.data() == NULL) return; int dev = get_device_side(); - if (dev == 1) { // if Device is the same as DualView's device type + if (dev == 1) { // if Device is the same as DualView's device type if ((modified_flags(0) > 0) && (modified_flags(0) >= modified_flags(1))) { - Impl::throw_runtime_exception("Calling sync on a DualView with a const datatype."); + Impl::throw_runtime_exception( + "Calling sync on a DualView with a const datatype."); } } - if (dev == 0){ // hopefully Device is the same as DualView's host type + if (dev == 0) { // hopefully Device is the same as DualView's host type if ((modified_flags(1) > 0) && (modified_flags(1) >= modified_flags(0))) { - Impl::throw_runtime_exception("Calling sync on a DualView with a const datatype."); + Impl::throw_runtime_exception( + "Calling sync on a DualView with a const datatype."); } } } void sync_host() { - if( ! std::is_same< typename traits::data_type , typename traits::non_const_data_type>::value ) - Impl::throw_runtime_exception("Calling sync_host on a DualView with a const datatype."); - if(modified_flags.data()==NULL) return; - if(modified_flags(1) > modified_flags(0)) { - deep_copy (h_view, d_view); + if (!std::is_same::value) + Impl::throw_runtime_exception( + "Calling sync_host on a DualView with a const datatype."); + if (modified_flags.data() == NULL) return; + if (modified_flags(1) > modified_flags(0)) { + deep_copy(h_view, d_view); modified_flags(1) = modified_flags(0) = 0; } } void sync_device() { - if( ! std::is_same< typename traits::data_type , typename traits::non_const_data_type>::value ) - Impl::throw_runtime_exception("Calling sync_device on a DualView with a const datatype."); - if(modified_flags.data()==NULL) return; - if(modified_flags(0) > modified_flags(1)) { - deep_copy (d_view, h_view); + if (!std::is_same::value) + Impl::throw_runtime_exception( + "Calling sync_device on a DualView with a const datatype."); + if (modified_flags.data() == NULL) return; + if (modified_flags(0) > modified_flags(1)) { + deep_copy(d_view, h_view); modified_flags(1) = modified_flags(0) = 0; } } - template - bool need_sync() const - { - if(modified_flags.data()==NULL) return false; + template + bool need_sync() const { + if (modified_flags.data() == NULL) return false; int dev = get_device_side(); - if (dev == 1) { // if Device is the same as DualView's device type + if (dev == 1) { // if Device is the same as DualView's device type if ((modified_flags(0) > 0) && (modified_flags(0) >= modified_flags(1))) { return true; } } - if (dev == 0){ // hopefully Device is the same as DualView's host type + if (dev == 0) { // hopefully Device is the same as DualView's host type if ((modified_flags(1) > 0) && (modified_flags(1) >= modified_flags(0))) { return true; } @@ -551,13 +565,13 @@ public: } inline bool need_sync_host() const { - if(modified_flags.data()==NULL) return false; - return modified_flags(0) - void modify () { - if(modified_flags.data()==NULL) return; + template + void modify() { + if (modified_flags.data() == NULL) return; int dev = get_device_side(); - if (dev == 1) { // if Device is the same as DualView's device type + if (dev == 1) { // if Device is the same as DualView's device type // Increment the device's modified count. - modified_flags(1) = (modified_flags(1) > modified_flags(0) ? - modified_flags(1) : modified_flags(0)) + 1; + modified_flags(1) = + (modified_flags(1) > modified_flags(0) ? modified_flags(1) + : modified_flags(0)) + + 1; } - if (dev == 0) { // hopefully Device is the same as DualView's host type + if (dev == 0) { // hopefully Device is the same as DualView's host type // Increment the host's modified count. - modified_flags(0) = (modified_flags(1) > modified_flags(0) ? - modified_flags(1) : modified_flags(0)) + 1; + modified_flags(0) = + (modified_flags(1) > modified_flags(0) ? modified_flags(1) + : modified_flags(0)) + + 1; } #ifdef KOKKOS_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK @@ -594,10 +612,12 @@ public: } inline void modify_host() { - if(modified_flags.data()!=NULL) { - modified_flags(0) = (modified_flags(1) > modified_flags(0) ? - modified_flags(1) : modified_flags(0)) + 1; - #ifdef KOKKOS_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK + if (modified_flags.data() != NULL) { + modified_flags(0) = + (modified_flags(1) > modified_flags(0) ? modified_flags(1) + : modified_flags(0)) + + 1; +#ifdef KOKKOS_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK if (modified_flags(0) && modified_flags(1)) { std::string msg = "Kokkos::DualView::modify_host ERROR: "; msg += "Concurrent modification of host and device views "; @@ -606,15 +626,17 @@ public: msg += "\"\n"; Kokkos::abort(msg.c_str()); } - #endif +#endif } } inline void modify_device() { - if(modified_flags.data()!=NULL) { - modified_flags(1) = (modified_flags(1) > modified_flags(0) ? - modified_flags(1) : modified_flags(0)) + 1; - #ifdef KOKKOS_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK + if (modified_flags.data() != NULL) { + modified_flags(1) = + (modified_flags(1) > modified_flags(0) ? modified_flags(1) + : modified_flags(0)) + + 1; +#ifdef KOKKOS_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK if (modified_flags(0) && modified_flags(1)) { std::string msg = "Kokkos::DualView::modify_device ERROR: "; msg += "Concurrent modification of host and device views "; @@ -623,12 +645,12 @@ public: msg += "\"\n"; Kokkos::abort(msg.c_str()); } - #endif +#endif } } inline void clear_sync_state() { - if(modified_flags.data()!=NULL) + if (modified_flags.data() != NULL) modified_flags(1) = modified_flags(0) = 0; } @@ -641,75 +663,72 @@ public: /// This discards any existing contents of the objects, and resets /// their modified flags. It does not copy the old contents /// of either View into the new View objects. - void realloc( const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG ) { - ::Kokkos::realloc(d_view,n0,n1,n2,n3,n4,n5,n6,n7); - h_view = create_mirror_view( d_view ); - - /* Reset dirty flags */ - if(modified_flags.data()==NULL) { - modified_flags = t_modified_flags("DualView::modified_flags"); - } else - modified_flags(1) = modified_flags(0) = 0; + void realloc(const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) { + ::Kokkos::realloc(d_view, n0, n1, n2, n3, n4, n5, n6, n7); + h_view = create_mirror_view(d_view); + + /* Reset dirty flags */ + if (modified_flags.data() == NULL) { + modified_flags = t_modified_flags("DualView::modified_flags"); + } else + modified_flags(1) = modified_flags(0) = 0; } /// \brief Resize both views, copying old contents into new if necessary. /// /// This method only copies the old contents into the new View /// objects for the device which was last marked as modified. - void resize( const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG ) { - if(modified_flags.data()==NULL) { - modified_flags = t_modified_flags("DualView::modified_flags"); - } - if(modified_flags(1) >= modified_flags(0)) { - /* Resize on Device */ - ::Kokkos::resize(d_view,n0,n1,n2,n3,n4,n5,n6,n7); - h_view = create_mirror_view( d_view ); - - /* Mark Device copy as modified */ - modified_flags(1) = modified_flags(1)+1; - - } else { - /* Realloc on Device */ - - ::Kokkos::realloc(d_view,n0,n1,n2,n3,n4,n5,n6,n7); - - const bool sizeMismatch = ( h_view.extent(0) != n0 ) || - ( h_view.extent(1) != n1 ) || - ( h_view.extent(2) != n2 ) || - ( h_view.extent(3) != n3 ) || - ( h_view.extent(4) != n4 ) || - ( h_view.extent(5) != n5 ) || - ( h_view.extent(6) != n6 ) || - ( h_view.extent(7) != n7 ); - if ( sizeMismatch ) - ::Kokkos::resize(h_view,n0,n1,n2,n3,n4,n5,n6,n7); - - t_host temp_view = create_mirror_view( d_view ); - - /* Remap on Host */ - Kokkos::deep_copy( temp_view , h_view ); - - h_view = temp_view; - - d_view = create_mirror_view( typename t_dev::execution_space(), h_view ); - - /* Mark Host copy as modified */ - modified_flags(0) = modified_flags(0)+1; - } + void resize(const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) { + if (modified_flags.data() == NULL) { + modified_flags = t_modified_flags("DualView::modified_flags"); + } + if (modified_flags(1) >= modified_flags(0)) { + /* Resize on Device */ + ::Kokkos::resize(d_view, n0, n1, n2, n3, n4, n5, n6, n7); + h_view = create_mirror_view(d_view); + + /* Mark Device copy as modified */ + modified_flags(1) = modified_flags(1) + 1; + + } else { + /* Realloc on Device */ + + ::Kokkos::realloc(d_view, n0, n1, n2, n3, n4, n5, n6, n7); + + const bool sizeMismatch = + (h_view.extent(0) != n0) || (h_view.extent(1) != n1) || + (h_view.extent(2) != n2) || (h_view.extent(3) != n3) || + (h_view.extent(4) != n4) || (h_view.extent(5) != n5) || + (h_view.extent(6) != n6) || (h_view.extent(7) != n7); + if (sizeMismatch) + ::Kokkos::resize(h_view, n0, n1, n2, n3, n4, n5, n6, n7); + + t_host temp_view = create_mirror_view(d_view); + + /* Remap on Host */ + Kokkos::deep_copy(temp_view, h_view); + + h_view = temp_view; + + d_view = create_mirror_view(typename t_dev::execution_space(), h_view); + + /* Mark Host copy as modified */ + modified_flags(0) = modified_flags(0) + 1; + } } //@} @@ -718,37 +737,35 @@ public: #ifdef KOKKOS_ENABLE_DEPRECATED_CODE //! The allocation size (same as Kokkos::View::capacity). - size_t capacity() const { - return d_view.span(); - } + size_t capacity() const { return d_view.span(); } #endif //! The allocation size (same as Kokkos::View::span). - KOKKOS_INLINE_FUNCTION constexpr size_t span() const { - return d_view.span(); - } + KOKKOS_INLINE_FUNCTION constexpr size_t span() const { return d_view.span(); } - KOKKOS_INLINE_FUNCTION bool span_is_contiguous() const { - return d_view.span_is_contiguous(); + KOKKOS_INLINE_FUNCTION bool span_is_contiguous() const { + return d_view.span_is_contiguous(); } //! Get stride(s) for each dimension. - template< typename iType> + template void stride(iType* stride_) const { d_view.stride(stride_); } - template< typename iType > - KOKKOS_INLINE_FUNCTION constexpr - typename std::enable_if< std::is_integral::value , size_t >::type - extent( const iType & r ) const - { return d_view.extent(r); } + template + KOKKOS_INLINE_FUNCTION constexpr + typename std::enable_if::value, size_t>::type + extent(const iType& r) const { + return d_view.extent(r); + } - template< typename iType > - KOKKOS_INLINE_FUNCTION constexpr - typename std::enable_if< std::is_integral::value , int >::type - extent_int( const iType & r ) const - { return static_cast(d_view.extent(r)); } + template + KOKKOS_INLINE_FUNCTION constexpr + typename std::enable_if::value, int>::type + extent_int(const iType& r) const { + return static_cast(d_view.extent(r)); + } #ifdef KOKKOS_ENABLE_DEPRECATED_CODE /* Deprecate all 'dimension' functions in favor of @@ -756,27 +773,27 @@ public: */ /* \brief return size of dimension 0 */ - size_t dimension_0() const {return d_view.extent(0);} + size_t dimension_0() const { return d_view.extent(0); } /* \brief return size of dimension 1 */ - size_t dimension_1() const {return d_view.extent(1);} + size_t dimension_1() const { return d_view.extent(1); } /* \brief return size of dimension 2 */ - size_t dimension_2() const {return d_view.extent(2);} + size_t dimension_2() const { return d_view.extent(2); } /* \brief return size of dimension 3 */ - size_t dimension_3() const {return d_view.extent(3);} + size_t dimension_3() const { return d_view.extent(3); } /* \brief return size of dimension 4 */ - size_t dimension_4() const {return d_view.extent(4);} + size_t dimension_4() const { return d_view.extent(4); } /* \brief return size of dimension 5 */ - size_t dimension_5() const {return d_view.extent(5);} + size_t dimension_5() const { return d_view.extent(5); } /* \brief return size of dimension 6 */ - size_t dimension_6() const {return d_view.extent(6);} + size_t dimension_6() const { return d_view.extent(6); } /* \brief return size of dimension 7 */ - size_t dimension_7() const {return d_view.extent(7);} + size_t dimension_7() const { return d_view.extent(7); } #endif //@} }; -} // namespace Kokkos +} // namespace Kokkos //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- @@ -787,32 +804,24 @@ public: namespace Kokkos { namespace Impl { -template< class D, class A1, class A2, class A3, class ... Args > +template struct DualViewSubview { + typedef typename Kokkos::Impl::ViewMapping< + void, Kokkos::ViewTraits, Args...>::traits_type dst_traits; - typedef typename Kokkos::Impl::ViewMapping - < void - , Kokkos::ViewTraits< D, A1, A2, A3 > - , Args ... - >::traits_type dst_traits ; - - typedef Kokkos::DualView - < typename dst_traits::data_type - , typename dst_traits::array_layout - , typename dst_traits::device_type - , typename dst_traits::memory_traits - > type ; + typedef Kokkos::DualView< + typename dst_traits::data_type, typename dst_traits::array_layout, + typename dst_traits::device_type, typename dst_traits::memory_traits> + type; }; } /* namespace Impl */ - -template< class D , class A1 , class A2 , class A3 , class ... Args > -typename Impl::DualViewSubview::type -subview( const DualView & src , Args ... args ) -{ - return typename - Impl::DualViewSubview::type( src , args ... ); +template +typename Impl::DualViewSubview::type subview( + const DualView& src, Args... args) { + return typename Impl::DualViewSubview::type(src, + args...); } } /* namespace Kokkos */ @@ -826,40 +835,35 @@ namespace Kokkos { // Partial specialization of Kokkos::deep_copy() for DualView objects. // -template< class DT , class DL , class DD , class DM , - class ST , class SL , class SD , class SM > -void -deep_copy (DualView dst, // trust me, this must not be a reference - const DualView& src ) -{ - if ( src.need_sync_device() ) { - deep_copy (dst.h_view, src.h_view); +template +void deep_copy( + DualView dst, // trust me, this must not be a reference + const DualView& src) { + if (src.need_sync_device()) { + deep_copy(dst.h_view, src.h_view); dst.modify_host(); - } - else { - deep_copy (dst.d_view, src.d_view); + } else { + deep_copy(dst.d_view, src.d_view); dst.modify_device(); - } + } } -template< class ExecutionSpace , - class DT , class DL , class DD , class DM , - class ST , class SL , class SD , class SM > -void -deep_copy (const ExecutionSpace& exec , - DualView dst, // trust me, this must not be a reference - const DualView& src ) -{ - if ( src.need_sync_device() ) { - deep_copy (exec, dst.h_view, src.h_view); +template +void deep_copy( + const ExecutionSpace& exec, + DualView dst, // trust me, this must not be a reference + const DualView& src) { + if (src.need_sync_device()) { + deep_copy(exec, dst.h_view, src.h_view); dst.modify_host(); } else { - deep_copy (exec, dst.d_view, src.d_view); + deep_copy(exec, dst.d_view, src.d_view); dst.modify_device(); } } -} // namespace Kokkos +} // namespace Kokkos #endif - diff --git a/lib/kokkos/containers/src/Kokkos_DynRankView.hpp b/lib/kokkos/containers/src/Kokkos_DynRankView.hpp index d1e6704a573dfd3c26ad5eaa37d69ab1dc678459..0ceb9d5d397e77fcfc8ef0a119e99cd6386a184b 100644 --- a/lib/kokkos/containers/src/Kokkos_DynRankView.hpp +++ b/lib/kokkos/containers/src/Kokkos_DynRankView.hpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -56,255 +57,240 @@ namespace Kokkos { -template< typename DataType , class ... Properties > -class DynRankView; //forward declare +template +class DynRankView; // forward declare namespace Impl { template struct DynRankDimTraits { - - enum : size_t{unspecified = KOKKOS_INVALID_INDEX}; + enum : size_t { unspecified = KOKKOS_INVALID_INDEX }; // Compute the rank of the view from the nonzero dimension arguments. KOKKOS_INLINE_FUNCTION - static size_t computeRank( const size_t N0 - , const size_t N1 - , const size_t N2 - , const size_t N3 - , const size_t N4 - , const size_t N5 - , const size_t N6 - , const size_t /* N7 */) - { - return - ( (N6 == unspecified && N5 == unspecified && N4 == unspecified && N3 == unspecified && N2 == unspecified && N1 == unspecified && N0 == unspecified) ? 0 - : ( (N6 == unspecified && N5 == unspecified && N4 == unspecified && N3 == unspecified && N2 == unspecified && N1 == unspecified) ? 1 - : ( (N6 == unspecified && N5 == unspecified && N4 == unspecified && N3 == unspecified && N2 == unspecified) ? 2 - : ( (N6 == unspecified && N5 == unspecified && N4 == unspecified && N3 == unspecified) ? 3 - : ( (N6 == unspecified && N5 == unspecified && N4 == unspecified) ? 4 - : ( (N6 == unspecified && N5 == unspecified) ? 5 - : ( (N6 == unspecified) ? 6 - : 7 ) ) ) ) ) ) ); + static size_t computeRank(const size_t N0, const size_t N1, const size_t N2, + const size_t N3, const size_t N4, const size_t N5, + const size_t N6, const size_t /* N7 */) { + return ( + (N6 == unspecified && N5 == unspecified && N4 == unspecified && + N3 == unspecified && N2 == unspecified && N1 == unspecified && + N0 == unspecified) + ? 0 + : ((N6 == unspecified && N5 == unspecified && N4 == unspecified && + N3 == unspecified && N2 == unspecified && N1 == unspecified) + ? 1 + : ((N6 == unspecified && N5 == unspecified && + N4 == unspecified && N3 == unspecified && + N2 == unspecified) + ? 2 + : ((N6 == unspecified && N5 == unspecified && + N4 == unspecified && N3 == unspecified) + ? 3 + : ((N6 == unspecified && N5 == unspecified && + N4 == unspecified) + ? 4 + : ((N6 == unspecified && + N5 == unspecified) + ? 5 + : ((N6 == unspecified) + ? 6 + : 7))))))); } // Compute the rank of the view from the nonzero layout arguments. template - KOKKOS_INLINE_FUNCTION - static size_t computeRank( const Layout& layout ) - { - return computeRank( layout.dimension[0] - , layout.dimension[1] - , layout.dimension[2] - , layout.dimension[3] - , layout.dimension[4] - , layout.dimension[5] - , layout.dimension[6] - , layout.dimension[7] ); + KOKKOS_INLINE_FUNCTION static size_t computeRank(const Layout& layout) { + return computeRank(layout.dimension[0], layout.dimension[1], + layout.dimension[2], layout.dimension[3], + layout.dimension[4], layout.dimension[5], + layout.dimension[6], layout.dimension[7]); } // Extra overload to match that for specialize types v2 - template - KOKKOS_INLINE_FUNCTION - static size_t computeRank( const Kokkos::Impl::ViewCtorProp& /* prop */, const Layout& layout ) - { + template + KOKKOS_INLINE_FUNCTION static size_t computeRank( + const Kokkos::Impl::ViewCtorProp& /* prop */, + const Layout& layout) { return computeRank(layout); } // Create the layout for the rank-7 view. // Non-strided Layout template - KOKKOS_INLINE_FUNCTION - static typename std::enable_if< (std::is_same::value || std::is_same::value) , Layout >::type createLayout( const Layout& layout ) - { - return Layout( layout.dimension[0] != unspecified ? layout.dimension[0] : 1 - , layout.dimension[1] != unspecified ? layout.dimension[1] : 1 - , layout.dimension[2] != unspecified ? layout.dimension[2] : 1 - , layout.dimension[3] != unspecified ? layout.dimension[3] : 1 - , layout.dimension[4] != unspecified ? layout.dimension[4] : 1 - , layout.dimension[5] != unspecified ? layout.dimension[5] : 1 - , layout.dimension[6] != unspecified ? layout.dimension[6] : 1 - , layout.dimension[7] != unspecified ? layout.dimension[7] : 1 - ); + KOKKOS_INLINE_FUNCTION static typename std::enable_if< + (std::is_same::value || + std::is_same::value), + Layout>::type + createLayout(const Layout& layout) { + return Layout(layout.dimension[0] != unspecified ? layout.dimension[0] : 1, + layout.dimension[1] != unspecified ? layout.dimension[1] : 1, + layout.dimension[2] != unspecified ? layout.dimension[2] : 1, + layout.dimension[3] != unspecified ? layout.dimension[3] : 1, + layout.dimension[4] != unspecified ? layout.dimension[4] : 1, + layout.dimension[5] != unspecified ? layout.dimension[5] : 1, + layout.dimension[6] != unspecified ? layout.dimension[6] : 1, + layout.dimension[7] != unspecified ? layout.dimension[7] : 1); } // LayoutStride template - KOKKOS_INLINE_FUNCTION - static typename std::enable_if< (std::is_same::value) , Layout>::type createLayout( const Layout& layout ) - { - return Layout( layout.dimension[0] != unspecified ? layout.dimension[0] : 1 - , layout.stride[0] - , layout.dimension[1] != unspecified ? layout.dimension[1] : 1 - , layout.stride[1] - , layout.dimension[2] != unspecified ? layout.dimension[2] : 1 - , layout.stride[2] - , layout.dimension[3] != unspecified ? layout.dimension[3] : 1 - , layout.stride[3] - , layout.dimension[4] != unspecified ? layout.dimension[4] : 1 - , layout.stride[4] - , layout.dimension[5] != unspecified ? layout.dimension[5] : 1 - , layout.stride[5] - , layout.dimension[6] != unspecified ? layout.dimension[6] : 1 - , layout.stride[6] - , layout.dimension[7] != unspecified ? layout.dimension[7] : 1 - , layout.stride[7] - ); + KOKKOS_INLINE_FUNCTION static typename std::enable_if< + (std::is_same::value), Layout>::type + createLayout(const Layout& layout) { + return Layout(layout.dimension[0] != unspecified ? layout.dimension[0] : 1, + layout.stride[0], + layout.dimension[1] != unspecified ? layout.dimension[1] : 1, + layout.stride[1], + layout.dimension[2] != unspecified ? layout.dimension[2] : 1, + layout.stride[2], + layout.dimension[3] != unspecified ? layout.dimension[3] : 1, + layout.stride[3], + layout.dimension[4] != unspecified ? layout.dimension[4] : 1, + layout.stride[4], + layout.dimension[5] != unspecified ? layout.dimension[5] : 1, + layout.stride[5], + layout.dimension[6] != unspecified ? layout.dimension[6] : 1, + layout.stride[6], + layout.dimension[7] != unspecified ? layout.dimension[7] : 1, + layout.stride[7]); } // Extra overload to match that for specialize types - template - KOKKOS_INLINE_FUNCTION - static typename std::enable_if< (std::is_same::value || std::is_same::value || std::is_same::value) , typename Traits::array_layout >::type createLayout( const Kokkos::Impl::ViewCtorProp& /* prop */, const typename Traits::array_layout& layout ) - { - return createLayout( layout ); + template + KOKKOS_INLINE_FUNCTION static typename std::enable_if< + (std::is_same::value || + std::is_same::value || + std::is_same::value), + typename Traits::array_layout>::type + createLayout(const Kokkos::Impl::ViewCtorProp& /* prop */, + const typename Traits::array_layout& layout) { + return createLayout(layout); } // Create a view from the given dimension arguments. // This is only necessary because the shmem constructor doesn't take a layout. - // NDE shmem View's are not compatible with the added view_alloc value_type / fad_dim deduction functionality + // NDE shmem View's are not compatible with the added view_alloc value_type + // / fad_dim deduction functionality template - static ViewType createView( const ViewArg& arg - , const size_t N0 - , const size_t N1 - , const size_t N2 - , const size_t N3 - , const size_t N4 - , const size_t N5 - , const size_t N6 - , const size_t N7 ) - { - return ViewType( arg - , N0 != unspecified ? N0 : 1 - , N1 != unspecified ? N1 : 1 - , N2 != unspecified ? N2 : 1 - , N3 != unspecified ? N3 : 1 - , N4 != unspecified ? N4 : 1 - , N5 != unspecified ? N5 : 1 - , N6 != unspecified ? N6 : 1 - , N7 != unspecified ? N7 : 1 ); + static ViewType createView(const ViewArg& arg, const size_t N0, + const size_t N1, const size_t N2, const size_t N3, + const size_t N4, const size_t N5, const size_t N6, + const size_t N7) { + return ViewType(arg, N0 != unspecified ? N0 : 1, N1 != unspecified ? N1 : 1, + N2 != unspecified ? N2 : 1, N3 != unspecified ? N3 : 1, + N4 != unspecified ? N4 : 1, N5 != unspecified ? N5 : 1, + N6 != unspecified ? N6 : 1, N7 != unspecified ? N7 : 1); } }; - // Non-strided Layout - template - KOKKOS_INLINE_FUNCTION - static typename std::enable_if< (std::is_same::value || std::is_same::value) && std::is_integral::value , Layout >::type - reconstructLayout( const Layout& layout , iType dynrank ) - { - return Layout( dynrank > 0 ? layout.dimension[0] :KOKKOS_INVALID_INDEX - , dynrank > 1 ? layout.dimension[1] :KOKKOS_INVALID_INDEX - , dynrank > 2 ? layout.dimension[2] :KOKKOS_INVALID_INDEX - , dynrank > 3 ? layout.dimension[3] :KOKKOS_INVALID_INDEX - , dynrank > 4 ? layout.dimension[4] :KOKKOS_INVALID_INDEX - , dynrank > 5 ? layout.dimension[5] :KOKKOS_INVALID_INDEX - , dynrank > 6 ? layout.dimension[6] :KOKKOS_INVALID_INDEX - , dynrank > 7 ? layout.dimension[7] :KOKKOS_INVALID_INDEX - ); - } - - // LayoutStride - template - KOKKOS_INLINE_FUNCTION - static typename std::enable_if< (std::is_same::value) && std::is_integral::value , Layout >::type - reconstructLayout( const Layout& layout , iType dynrank ) - { - return Layout( dynrank > 0 ? layout.dimension[0] :KOKKOS_INVALID_INDEX - , dynrank > 0 ? layout.stride[0] : (0) - , dynrank > 1 ? layout.dimension[1] :KOKKOS_INVALID_INDEX - , dynrank > 1 ? layout.stride[1] : (0) - , dynrank > 2 ? layout.dimension[2] :KOKKOS_INVALID_INDEX - , dynrank > 2 ? layout.stride[2] : (0) - , dynrank > 3 ? layout.dimension[3] :KOKKOS_INVALID_INDEX - , dynrank > 3 ? layout.stride[3] : (0) - , dynrank > 4 ? layout.dimension[4] :KOKKOS_INVALID_INDEX - , dynrank > 4 ? layout.stride[4] : (0) - , dynrank > 5 ? layout.dimension[5] :KOKKOS_INVALID_INDEX - , dynrank > 5 ? layout.stride[5] : (0) - , dynrank > 6 ? layout.dimension[6] :KOKKOS_INVALID_INDEX - , dynrank > 6 ? layout.stride[6] : (0) - , dynrank > 7 ? layout.dimension[7] :KOKKOS_INVALID_INDEX - , dynrank > 7 ? layout.stride[7] : (0) - ); - } +// Non-strided Layout +template +KOKKOS_INLINE_FUNCTION static + typename std::enable_if<(std::is_same::value || + std::is_same::value) && + std::is_integral::value, + Layout>::type + reconstructLayout(const Layout& layout, iType dynrank) { + return Layout(dynrank > 0 ? layout.dimension[0] : KOKKOS_INVALID_INDEX, + dynrank > 1 ? layout.dimension[1] : KOKKOS_INVALID_INDEX, + dynrank > 2 ? layout.dimension[2] : KOKKOS_INVALID_INDEX, + dynrank > 3 ? layout.dimension[3] : KOKKOS_INVALID_INDEX, + dynrank > 4 ? layout.dimension[4] : KOKKOS_INVALID_INDEX, + dynrank > 5 ? layout.dimension[5] : KOKKOS_INVALID_INDEX, + dynrank > 6 ? layout.dimension[6] : KOKKOS_INVALID_INDEX, + dynrank > 7 ? layout.dimension[7] : KOKKOS_INVALID_INDEX); +} +// LayoutStride +template +KOKKOS_INLINE_FUNCTION static typename std::enable_if< + (std::is_same::value) && + std::is_integral::value, + Layout>::type +reconstructLayout(const Layout& layout, iType dynrank) { + return Layout(dynrank > 0 ? layout.dimension[0] : KOKKOS_INVALID_INDEX, + dynrank > 0 ? layout.stride[0] : (0), + dynrank > 1 ? layout.dimension[1] : KOKKOS_INVALID_INDEX, + dynrank > 1 ? layout.stride[1] : (0), + dynrank > 2 ? layout.dimension[2] : KOKKOS_INVALID_INDEX, + dynrank > 2 ? layout.stride[2] : (0), + dynrank > 3 ? layout.dimension[3] : KOKKOS_INVALID_INDEX, + dynrank > 3 ? layout.stride[3] : (0), + dynrank > 4 ? layout.dimension[4] : KOKKOS_INVALID_INDEX, + dynrank > 4 ? layout.stride[4] : (0), + dynrank > 5 ? layout.dimension[5] : KOKKOS_INVALID_INDEX, + dynrank > 5 ? layout.stride[5] : (0), + dynrank > 6 ? layout.dimension[6] : KOKKOS_INVALID_INDEX, + dynrank > 6 ? layout.stride[6] : (0), + dynrank > 7 ? layout.dimension[7] : KOKKOS_INVALID_INDEX, + dynrank > 7 ? layout.stride[7] : (0)); +} /** \brief Debug bounds-checking routines */ // Enhanced debug checking - most infrastructure matches that of functions in // Kokkos_ViewMapping; additional checks for extra arguments beyond rank are 0 -template< unsigned , typename iType0 , class MapType > -KOKKOS_INLINE_FUNCTION -bool dyn_rank_view_verify_operator_bounds( const iType0 & , const MapType & ) -{ return true ; } - -template< unsigned R , typename iType0 , class MapType , typename iType1 , class ... Args > -KOKKOS_INLINE_FUNCTION -bool dyn_rank_view_verify_operator_bounds - ( const iType0 & rank - , const MapType & map - , const iType1 & i - , Args ... args - ) -{ - if ( static_cast(R) < rank ) { - return ( size_t(i) < map.extent(R) ) - && dyn_rank_view_verify_operator_bounds( rank , map , args ... ); - } - else if ( i != 0 ) { - printf("DynRankView Debug Bounds Checking Error: at rank %u\n Extra arguments beyond the rank must be zero \n",R); - return ( false ) - && dyn_rank_view_verify_operator_bounds( rank , map , args ... ); - } - else { - return ( true ) - && dyn_rank_view_verify_operator_bounds( rank , map , args ... ); +template +KOKKOS_INLINE_FUNCTION bool dyn_rank_view_verify_operator_bounds( + const iType0&, const MapType&) { + return true; +} + +template +KOKKOS_INLINE_FUNCTION bool dyn_rank_view_verify_operator_bounds( + const iType0& rank, const MapType& map, const iType1& i, Args... args) { + if (static_cast(R) < rank) { + return (size_t(i) < map.extent(R)) && + dyn_rank_view_verify_operator_bounds(rank, map, args...); + } else if (i != 0) { + printf( + "DynRankView Debug Bounds Checking Error: at rank %u\n Extra " + "arguments beyond the rank must be zero \n", + R); + return (false) && + dyn_rank_view_verify_operator_bounds(rank, map, args...); + } else { + return (true) && + dyn_rank_view_verify_operator_bounds(rank, map, args...); } } -template< unsigned , class MapType > -inline -void dyn_rank_view_error_operator_bounds( char * , int , const MapType & ) -{} - -template< unsigned R , class MapType , class iType , class ... Args > -inline -void dyn_rank_view_error_operator_bounds - ( char * buf - , int len - , const MapType & map - , const iType & i - , Args ... args - ) -{ - const int n = - snprintf(buf,len," %ld < %ld %c" - , static_cast(i) - , static_cast( map.extent(R) ) - , ( sizeof...(Args) ? ',' : ')' ) - ); - dyn_rank_view_error_operator_bounds(buf+n,len-n,map,args...); +template +inline void dyn_rank_view_error_operator_bounds(char*, int, const MapType&) {} + +template +inline void dyn_rank_view_error_operator_bounds(char* buf, int len, + const MapType& map, + const iType& i, Args... args) { + const int n = snprintf( + buf, len, " %ld < %ld %c", static_cast(i), + static_cast(map.extent(R)), (sizeof...(Args) ? ',' : ')')); + dyn_rank_view_error_operator_bounds(buf + n, len - n, map, args...); } // op_rank = rank of the operator version that was called -template< typename MemorySpace - , typename iType0 , typename iType1 , class MapType , class ... Args > -KOKKOS_INLINE_FUNCTION -void dyn_rank_view_verify_operator_bounds - ( const iType0 & op_rank , const iType1 & rank - , const Kokkos::Impl::SharedAllocationTracker & tracker - , const MapType & map , Args ... args ) -{ - if ( static_cast(rank) > op_rank ) { - Kokkos::abort( "DynRankView Bounds Checking Error: Need at least rank arguments to the operator()" ); - } - - if ( ! dyn_rank_view_verify_operator_bounds<0>( rank , map , args ... ) ) { -#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) +template +KOKKOS_INLINE_FUNCTION void dyn_rank_view_verify_operator_bounds( + const iType0& op_rank, const iType1& rank, + const Kokkos::Impl::SharedAllocationTracker& tracker, const MapType& map, + Args... args) { + if (static_cast(rank) > op_rank) { + Kokkos::abort( + "DynRankView Bounds Checking Error: Need at least rank arguments to " + "the operator()"); + } + + if (!dyn_rank_view_verify_operator_bounds<0>(rank, map, args...)) { +#if defined(KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST) enum { LEN = 1024 }; - char buffer[ LEN ]; + char buffer[LEN]; const std::string label = tracker.template get_label(); - int n = snprintf(buffer,LEN,"DynRankView bounds error of view %s (", label.c_str()); - dyn_rank_view_error_operator_bounds<0>( buffer + n , LEN - n , map , args ... ); + int n = snprintf(buffer, LEN, "DynRankView bounds error of view %s (", + label.c_str()); + dyn_rank_view_error_operator_bounds<0>(buffer + n, LEN - n, map, args...); Kokkos::Impl::throw_runtime_exception(std::string(buffer)); #else Kokkos::abort("DynRankView bounds error"); @@ -312,86 +298,84 @@ void dyn_rank_view_verify_operator_bounds } } - /** \brief Assign compatible default mappings */ struct ViewToDynRankViewTag {}; -} // namespace Impl +} // namespace Impl namespace Impl { -template< class DstTraits , class SrcTraits > -class ViewMapping< DstTraits , SrcTraits , - typename std::enable_if<( - std::is_same< typename DstTraits::memory_space , typename SrcTraits::memory_space >::value - && - std::is_same< typename DstTraits::specialize , void >::value - && - std::is_same< typename SrcTraits::specialize , void >::value - && - ( - std::is_same< typename DstTraits::array_layout , typename SrcTraits::array_layout >::value - || - ( - ( - std::is_same< typename DstTraits::array_layout , Kokkos::LayoutLeft >::value || - std::is_same< typename DstTraits::array_layout , Kokkos::LayoutRight >::value || - std::is_same< typename DstTraits::array_layout , Kokkos::LayoutStride >::value - ) - && - ( - std::is_same< typename SrcTraits::array_layout , Kokkos::LayoutLeft >::value || - std::is_same< typename SrcTraits::array_layout , Kokkos::LayoutRight >::value || - std::is_same< typename SrcTraits::array_layout , Kokkos::LayoutStride >::value - ) - ) - ) - ) , Kokkos::Impl::ViewToDynRankViewTag >::type > -{ -private: - - enum { is_assignable_value_type = - std::is_same< typename DstTraits::value_type - , typename SrcTraits::value_type >::value || - std::is_same< typename DstTraits::value_type - , typename SrcTraits::const_value_type >::value }; - - enum { is_assignable_layout = - std::is_same< typename DstTraits::array_layout - , typename SrcTraits::array_layout >::value || - std::is_same< typename DstTraits::array_layout - , Kokkos::LayoutStride >::value - }; +template +class ViewMapping< + DstTraits, SrcTraits, + typename std::enable_if< + (std::is_same::value && + std::is_same::value && + std::is_same::value && + (std::is_same::value || + ((std::is_same::value || + std::is_same::value || + std::is_same::value) && + (std::is_same::value || + std::is_same::value || + std::is_same::value)))), + Kokkos::Impl::ViewToDynRankViewTag>::type> { + private: + enum { + is_assignable_value_type = + std::is_same::value || + std::is_same::value + }; -public: + enum { + is_assignable_layout = + std::is_same::value || + std::is_same::value + }; - enum { is_assignable = is_assignable_value_type && - is_assignable_layout }; + public: + enum { is_assignable = is_assignable_value_type && is_assignable_layout }; - typedef ViewMapping< DstTraits , typename DstTraits::specialize > DstType ; - typedef ViewMapping< SrcTraits , typename SrcTraits::specialize > SrcType ; + typedef ViewMapping DstType; + typedef ViewMapping SrcType; - template < typename DT , typename ... DP , typename ST , typename ... SP > - KOKKOS_INLINE_FUNCTION - static void assign( Kokkos::DynRankView< DT , DP...> & dst , const Kokkos::View< ST , SP... > & src ) - { - static_assert( is_assignable_value_type - , "View assignment must have same value type or const = non-const" ); + template + KOKKOS_INLINE_FUNCTION static void assign( + Kokkos::DynRankView& dst, const Kokkos::View& src) { + static_assert( + is_assignable_value_type, + "View assignment must have same value type or const = non-const"); - static_assert( is_assignable_layout - , "View assignment must have compatible layout or have rank <= 1" ); + static_assert( + is_assignable_layout, + "View assignment must have compatible layout or have rank <= 1"); // Removed dimension checks... - typedef typename DstType::offset_type dst_offset_type ; - dst.m_map.m_impl_offset = dst_offset_type(std::integral_constant() , src.layout() ); //Check this for integer input1 for padding, etc - dst.m_map.m_impl_handle = Kokkos::Impl::ViewDataHandle< DstTraits >::assign( src.m_map.m_impl_handle , src.m_track ); - dst.m_track.assign( src.m_track , DstTraits::is_managed ); - dst.m_rank = src.Rank ; - } + typedef typename DstType::offset_type dst_offset_type; + dst.m_map.m_impl_offset = dst_offset_type( + std::integral_constant(), + src.layout()); // Check this for integer input1 for padding, etc + dst.m_map.m_impl_handle = Kokkos::Impl::ViewDataHandle::assign( + src.m_map.m_impl_handle, src.m_track); + dst.m_track.assign(src.m_track, DstTraits::is_managed); + dst.m_rank = src.Rank; + } }; -} //end Impl +} // namespace Impl /* \class DynRankView * \brief Container that creates a Kokkos view with rank determined at runtime. @@ -400,7 +384,8 @@ public: * Changes from View * 1. The rank of the DynRankView is returned by the method rank() * 2. Max rank of a DynRankView is 7 - * 3. subview called with 'subview(...)' or 'subdynrankview(...)' (backward compatibility) + * 3. subview called with 'subview(...)' or 'subdynrankview(...)' (backward + * compatibility) * 4. Every subview is returned with LayoutStride * 5. Copy and Copy-Assign View to DynRankView * 6. deep_copy between Views and DynRankViews @@ -408,93 +393,99 @@ public: * */ -template< class > struct is_dyn_rank_view : public std::false_type {}; - -template< class D, class ... P > -struct is_dyn_rank_view< Kokkos::DynRankView > : public std::true_type {}; +template +struct is_dyn_rank_view : public std::false_type {}; +template +struct is_dyn_rank_view > : public std::true_type { +}; -template< typename DataType , class ... Properties > -class DynRankView : public ViewTraits< DataType , Properties ... > -{ - static_assert( !std::is_array::value && !std::is_pointer::value , "Cannot template DynRankView with array or pointer datatype - must be pod" ); - -private: - template < class , class ... > friend class DynRankView ; - template < class , class ... > friend class Kokkos::Impl::ViewMapping ; +template +class DynRankView : public ViewTraits { + static_assert(!std::is_array::value && + !std::is_pointer::value, + "Cannot template DynRankView with array or pointer datatype - " + "must be pod"); -public: - typedef ViewTraits< DataType , Properties ... > drvtraits ; + private: + template + friend class DynRankView; + template + friend class Kokkos::Impl::ViewMapping; - typedef View< DataType******* , Properties...> view_type ; + public: + typedef ViewTraits drvtraits; - typedef ViewTraits< DataType******* , Properties ... > traits ; + typedef View view_type; + typedef ViewTraits traits; -private: - typedef Kokkos::Impl::ViewMapping< traits , typename traits::specialize > map_type ; - typedef Kokkos::Impl::SharedAllocationTracker track_type ; + private: + typedef Kokkos::Impl::ViewMapping + map_type; + typedef Kokkos::Impl::SharedAllocationTracker track_type; - track_type m_track ; - map_type m_map ; + track_type m_track; + map_type m_map; unsigned m_rank; -public: + public: KOKKOS_INLINE_FUNCTION - view_type & DownCast() const { return ( view_type & ) (*this); } + view_type& DownCast() const { return (view_type&)(*this); } KOKKOS_INLINE_FUNCTION - const view_type & ConstDownCast() const { return (const view_type & ) (*this); } + const view_type& ConstDownCast() const { return (const view_type&)(*this); } - //Types below - at least the HostMirror requires the value_type, NOT the rank 7 data_type of the traits + // Types below - at least the HostMirror requires the value_type, NOT the rank + // 7 data_type of the traits /** \brief Compatible view of array of scalar types */ - typedef DynRankView< typename drvtraits::scalar_array_type , - typename drvtraits::array_layout , - typename drvtraits::device_type , - typename drvtraits::memory_traits > - array_type ; + typedef DynRankView< + typename drvtraits::scalar_array_type, typename drvtraits::array_layout, + typename drvtraits::device_type, typename drvtraits::memory_traits> + array_type; /** \brief Compatible view of const data type */ - typedef DynRankView< typename drvtraits::const_data_type , - typename drvtraits::array_layout , - typename drvtraits::device_type , - typename drvtraits::memory_traits > - const_type ; + typedef DynRankView< + typename drvtraits::const_data_type, typename drvtraits::array_layout, + typename drvtraits::device_type, typename drvtraits::memory_traits> + const_type; /** \brief Compatible view of non-const data type */ - typedef DynRankView< typename drvtraits::non_const_data_type , - typename drvtraits::array_layout , - typename drvtraits::device_type , - typename drvtraits::memory_traits > - non_const_type ; + typedef DynRankView< + typename drvtraits::non_const_data_type, typename drvtraits::array_layout, + typename drvtraits::device_type, typename drvtraits::memory_traits> + non_const_type; /** \brief Compatible HostMirror view */ - typedef DynRankView< typename drvtraits::non_const_data_type , - typename drvtraits::array_layout , - typename drvtraits::host_mirror_space > - HostMirror ; - + typedef DynRankView + HostMirror; //---------------------------------------- // Domain rank and extents -// enum { Rank = map_type::Rank }; //Will be dyn rank of 7 always, keep the enum? + // enum { Rank = map_type::Rank }; //Will be dyn rank of 7 always, keep the + // enum? - template< typename iType > + template KOKKOS_INLINE_FUNCTION constexpr - typename std::enable_if< std::is_integral::value , size_t >::type - extent( const iType & r ) const - { return m_map.extent(r); } + typename std::enable_if::value, size_t>::type + extent(const iType& r) const { + return m_map.extent(r); + } - template< typename iType > + template KOKKOS_INLINE_FUNCTION constexpr - typename std::enable_if< std::is_integral::value , int >::type - extent_int( const iType & r ) const - { return static_cast(m_map.extent(r)); } + typename std::enable_if::value, int>::type + extent_int(const iType& r) const { + return static_cast(m_map.extent(r)); + } - KOKKOS_INLINE_FUNCTION constexpr - typename traits::array_layout layout() const - { return m_map.layout(); } + KOKKOS_INLINE_FUNCTION constexpr typename traits::array_layout layout() + const { + return m_map.layout(); + } //---------------------------------------- /* Deprecate all 'dimension' functions in favor of @@ -502,421 +493,572 @@ public: */ #ifdef KOKKOS_ENABLE_DEPRECATED_CODE - template< typename iType > + template KOKKOS_INLINE_FUNCTION constexpr - typename std::enable_if< std::is_integral::value , size_t >::type - dimension( const iType & r ) const { return extent( r ); } - - KOKKOS_INLINE_FUNCTION constexpr size_t dimension_0() const { return m_map.dimension_0(); } - KOKKOS_INLINE_FUNCTION constexpr size_t dimension_1() const { return m_map.dimension_1(); } - KOKKOS_INLINE_FUNCTION constexpr size_t dimension_2() const { return m_map.dimension_2(); } - KOKKOS_INLINE_FUNCTION constexpr size_t dimension_3() const { return m_map.dimension_3(); } - KOKKOS_INLINE_FUNCTION constexpr size_t dimension_4() const { return m_map.dimension_4(); } - KOKKOS_INLINE_FUNCTION constexpr size_t dimension_5() const { return m_map.dimension_5(); } - KOKKOS_INLINE_FUNCTION constexpr size_t dimension_6() const { return m_map.dimension_6(); } - KOKKOS_INLINE_FUNCTION constexpr size_t dimension_7() const { return m_map.dimension_7(); } + typename std::enable_if::value, size_t>::type + dimension(const iType& r) const { + return extent(r); + } + + KOKKOS_INLINE_FUNCTION constexpr size_t dimension_0() const { + return m_map.dimension_0(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t dimension_1() const { + return m_map.dimension_1(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t dimension_2() const { + return m_map.dimension_2(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t dimension_3() const { + return m_map.dimension_3(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t dimension_4() const { + return m_map.dimension_4(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t dimension_5() const { + return m_map.dimension_5(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t dimension_6() const { + return m_map.dimension_6(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t dimension_7() const { + return m_map.dimension_7(); + } #endif //---------------------------------------- - KOKKOS_INLINE_FUNCTION constexpr size_t size() const { return m_map.extent(0) * - m_map.extent(1) * - m_map.extent(2) * - m_map.extent(3) * - m_map.extent(4) * - m_map.extent(5) * - m_map.extent(6) * - m_map.extent(7); } - - KOKKOS_INLINE_FUNCTION constexpr size_t stride_0() const { return m_map.stride_0(); } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_1() const { return m_map.stride_1(); } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_2() const { return m_map.stride_2(); } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_3() const { return m_map.stride_3(); } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_4() const { return m_map.stride_4(); } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_5() const { return m_map.stride_5(); } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_6() const { return m_map.stride_6(); } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_7() const { return m_map.stride_7(); } - - template< typename iType > - KOKKOS_INLINE_FUNCTION void stride( iType * const s ) const { m_map.stride(s); } + KOKKOS_INLINE_FUNCTION constexpr size_t size() const { + return m_map.extent(0) * m_map.extent(1) * m_map.extent(2) * + m_map.extent(3) * m_map.extent(4) * m_map.extent(5) * + m_map.extent(6) * m_map.extent(7); + } + + KOKKOS_INLINE_FUNCTION constexpr size_t stride_0() const { + return m_map.stride_0(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t stride_1() const { + return m_map.stride_1(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t stride_2() const { + return m_map.stride_2(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t stride_3() const { + return m_map.stride_3(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t stride_4() const { + return m_map.stride_4(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t stride_5() const { + return m_map.stride_5(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t stride_6() const { + return m_map.stride_6(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t stride_7() const { + return m_map.stride_7(); + } + + template + KOKKOS_INLINE_FUNCTION void stride(iType* const s) const { + m_map.stride(s); + } //---------------------------------------- // Range span is the span which contains all members. - typedef typename map_type::reference_type reference_type ; - typedef typename map_type::pointer_type pointer_type ; + typedef typename map_type::reference_type reference_type; + typedef typename map_type::pointer_type pointer_type; - enum { reference_type_is_lvalue_reference = std::is_lvalue_reference< reference_type >::value }; + enum { + reference_type_is_lvalue_reference = + std::is_lvalue_reference::value + }; KOKKOS_INLINE_FUNCTION constexpr size_t span() const { return m_map.span(); } #ifdef KOKKOS_ENABLE_DEPRECATED_CODE // Deprecated, use 'span()' instead - KOKKOS_INLINE_FUNCTION constexpr size_t capacity() const { return m_map.span(); } + KOKKOS_INLINE_FUNCTION constexpr size_t capacity() const { + return m_map.span(); + } #endif - KOKKOS_INLINE_FUNCTION constexpr bool span_is_contiguous() const { return m_map.span_is_contiguous(); } - KOKKOS_INLINE_FUNCTION constexpr pointer_type data() const { return m_map.data(); } + KOKKOS_INLINE_FUNCTION constexpr bool span_is_contiguous() const { + return m_map.span_is_contiguous(); + } + KOKKOS_INLINE_FUNCTION constexpr pointer_type data() const { + return m_map.data(); + } #ifdef KOKKOS_ENABLE_DEPRECATED_CODE // Deprecated, use 'span_is_contigous()' instead - KOKKOS_INLINE_FUNCTION constexpr bool is_contiguous() const { return m_map.span_is_contiguous(); } + KOKKOS_INLINE_FUNCTION constexpr bool is_contiguous() const { + return m_map.span_is_contiguous(); + } // Deprecated, use 'data()' instead - KOKKOS_INLINE_FUNCTION constexpr pointer_type ptr_on_device() const { return m_map.data(); } + KOKKOS_INLINE_FUNCTION constexpr pointer_type ptr_on_device() const { + return m_map.data(); + } #endif //---------------------------------------- // Allow specializations to query their specialized map #ifdef KOKKOS_ENABLE_DEPRECATED_CODE KOKKOS_INLINE_FUNCTION - const Kokkos::Impl::ViewMapping< traits , typename traits::specialize > & - implementation_map() const { return m_map ; } + const Kokkos::Impl::ViewMapping& + implementation_map() const { + return m_map; + } #endif KOKKOS_INLINE_FUNCTION - const Kokkos::Impl::ViewMapping< traits , typename traits::specialize > & - impl_map() const { return m_map ; } + const Kokkos::Impl::ViewMapping& + impl_map() const { + return m_map; + } //---------------------------------------- -private: - + private: enum { - is_layout_left = std::is_same< typename traits::array_layout - , Kokkos::LayoutLeft >::value , + is_layout_left = + std::is_same::value, - is_layout_right = std::is_same< typename traits::array_layout - , Kokkos::LayoutRight >::value , + is_layout_right = + std::is_same::value, - is_layout_stride = std::is_same< typename traits::array_layout - , Kokkos::LayoutStride >::value , + is_layout_stride = std::is_same::value, - is_default_map = - std::is_same< typename traits::specialize , void >::value && - ( is_layout_left || is_layout_right || is_layout_stride ) + is_default_map = std::is_same::value && + (is_layout_left || is_layout_right || is_layout_stride) }; - template< class Space , bool = Kokkos::Impl::MemorySpaceAccess< Space , typename traits::memory_space >::accessible > struct verify_space - { KOKKOS_FORCEINLINE_FUNCTION static void check() {} }; + template ::accessible> + struct verify_space { + KOKKOS_FORCEINLINE_FUNCTION static void check() {} + }; - template< class Space > struct verify_space - { KOKKOS_FORCEINLINE_FUNCTION static void check() - { Kokkos::abort("Kokkos::DynRankView ERROR: attempt to access inaccessible memory space"); }; + template + struct verify_space { + KOKKOS_FORCEINLINE_FUNCTION static void check() { + Kokkos::abort( + "Kokkos::DynRankView ERROR: attempt to access inaccessible memory " + "space"); }; + }; // Bounds checking macros -#if defined( KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK ) +#if defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK) // rank of the calling operator - included as first argument in ARG -#define KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( ARG ) \ - DynRankView::template verify_space< Kokkos::Impl::ActiveExecutionMemorySpace >::check(); \ - Kokkos::Impl::dyn_rank_view_verify_operator_bounds< typename traits::memory_space > ARG ; +#define KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(ARG) \ + DynRankView::template verify_space< \ + Kokkos::Impl::ActiveExecutionMemorySpace>::check(); \ + Kokkos::Impl::dyn_rank_view_verify_operator_bounds< \ + typename traits::memory_space> \ + ARG; #else -#define KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( ARG ) \ - DynRankView::template verify_space< Kokkos::Impl::ActiveExecutionMemorySpace >::check(); +#define KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(ARG) \ + DynRankView::template verify_space< \ + Kokkos::Impl::ActiveExecutionMemorySpace>::check(); #endif -public: - + public: KOKKOS_INLINE_FUNCTION constexpr unsigned rank() const { return m_rank; } - - //operators () + // operators () // Rank 0 KOKKOS_INLINE_FUNCTION - reference_type operator()() const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (0 , this->rank(), m_track, m_map) ) - return impl_map().reference(); - //return m_map.reference(0,0,0,0,0,0,0); - } + reference_type operator()() const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((0, this->rank(), m_track, m_map)) + return impl_map().reference(); + // return m_map.reference(0,0,0,0,0,0,0); + } // Rank 1 - // This assumes a contiguous underlying memory (i.e. no padding, no striding...) - template< typename iType > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< std::is_same::value && std::is_integral::value, reference_type>::type - operator[](const iType & i0) const - { - //Phalanx is violating this, since they use the operator to access ALL elements in the allocation - //KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (1 , this->rank(), m_track, m_map) ) - return data()[i0]; - } + // This assumes a contiguous underlying memory (i.e. no padding, no + // striding...) + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + std::is_same::value && + std::is_integral::value, + reference_type>::type + operator[](const iType& i0) const { + // Phalanx is violating this, since they use the operator to access ALL + // elements in the allocation KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (1 , + // this->rank(), m_track, m_map) ) + return data()[i0]; + } - // This assumes a contiguous underlying memory (i.e. no padding, no striding... - // AND a Trilinos/Sacado scalar type ) - template< typename iType > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< !std::is_same::value && std::is_integral::value, reference_type>::type - operator[](const iType & i0) const - { -// auto map = impl_map(); - const size_t dim_scalar = m_map.dimension_scalar(); - const size_t bytes = this->span() / dim_scalar; - - typedef Kokkos::View > tmp_view_type; - tmp_view_type rankone_view(this->data(), bytes, dim_scalar); - return rankone_view(i0); - } + // This assumes a contiguous underlying memory (i.e. no padding, no + // striding... AND a Trilinos/Sacado scalar type ) + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + !std::is_same::value && + std::is_integral::value, + reference_type>::type + operator[](const iType& i0) const { + // auto map = impl_map(); + const size_t dim_scalar = m_map.dimension_scalar(); + const size_t bytes = this->span() / dim_scalar; + + typedef Kokkos::View< + DataType*, typename traits::array_layout, typename traits::device_type, + Kokkos::MemoryTraits > + tmp_view_type; + tmp_view_type rankone_view(this->data(), bytes, dim_scalar); + return rankone_view(i0); + } // Rank 1 parenthesis - template< typename iType > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_same::value && std::is_integral::value), reference_type>::type - operator()(const iType & i0 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (1 , this->rank(), m_track, m_map, i0) ) - return m_map.reference(i0); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_same::value && + std::is_integral::value), + reference_type>::type + operator()(const iType& i0) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((1, this->rank(), m_track, m_map, i0)) + return m_map.reference(i0); + } - template< typename iType > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< !(std::is_same::value && std::is_integral::value), reference_type>::type - operator()(const iType & i0 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (1 , this->rank(), m_track, m_map, i0) ) - return m_map.reference(i0,0,0,0,0,0,0); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + !(std::is_same::value && + std::is_integral::value), + reference_type>::type + operator()(const iType& i0) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((1, this->rank(), m_track, m_map, i0)) + return m_map.reference(i0, 0, 0, 0, 0, 0, 0); + } // Rank 2 - template< typename iType0 , typename iType1 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_same::value && std::is_integral::value && std::is_integral::value), reference_type>::type - operator()(const iType0 & i0 , const iType1 & i1 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (2 , this->rank(), m_track, m_map, i0, i1) ) - return m_map.reference(i0,i1); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_same::value && + std::is_integral::value && std::is_integral::value), + reference_type>::type + operator()(const iType0& i0, const iType1& i1) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((2, this->rank(), m_track, m_map, i0, i1)) + return m_map.reference(i0, i1); + } - template< typename iType0 , typename iType1 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< !(std::is_same::value && std::is_integral::value), reference_type>::type - operator()(const iType0 & i0 , const iType1 & i1 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (2 , this->rank(), m_track, m_map, i0, i1) ) - return m_map.reference(i0,i1,0,0,0,0,0); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + !(std::is_same::value && + std::is_integral::value), + reference_type>::type + operator()(const iType0& i0, const iType1& i1) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((2, this->rank(), m_track, m_map, i0, i1)) + return m_map.reference(i0, i1, 0, 0, 0, 0, 0); + } // Rank 3 - template< typename iType0 , typename iType1 , typename iType2 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_same::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type>::type - operator()(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (3 , this->rank(), m_track, m_map, i0, i1, i2) ) - return m_map.reference(i0,i1,i2); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_same::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value), + reference_type>::type + operator()(const iType0& i0, const iType1& i1, const iType2& i2) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (3, this->rank(), m_track, m_map, i0, i1, i2)) + return m_map.reference(i0, i1, i2); + } - template< typename iType0 , typename iType1 , typename iType2 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< !(std::is_same::value && std::is_integral::value), reference_type>::type - operator()(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (3 , this->rank(), m_track, m_map, i0, i1, i2) ) - return m_map.reference(i0,i1,i2,0,0,0,0); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + !(std::is_same::value && + std::is_integral::value), + reference_type>::type + operator()(const iType0& i0, const iType1& i1, const iType2& i2) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (3, this->rank(), m_track, m_map, i0, i1, i2)) + return m_map.reference(i0, i1, i2, 0, 0, 0, 0); + } // Rank 4 - template< typename iType0 , typename iType1 , typename iType2 , typename iType3 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_same::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type>::type - operator()(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (4 , this->rank(), m_track, m_map, i0, i1, i2, i3) ) - return m_map.reference(i0,i1,i2,i3); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_same::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value && std::is_integral::value), + reference_type>::type + operator()(const iType0& i0, const iType1& i1, const iType2& i2, + const iType3& i3) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (4, this->rank(), m_track, m_map, i0, i1, i2, i3)) + return m_map.reference(i0, i1, i2, i3); + } - template< typename iType0 , typename iType1 , typename iType2 , typename iType3 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< !(std::is_same::value && std::is_integral::value), reference_type>::type - operator()(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (4 , this->rank(), m_track, m_map, i0, i1, i2, i3) ) - return m_map.reference(i0,i1,i2,i3,0,0,0); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + !(std::is_same::value && + std::is_integral::value), + reference_type>::type + operator()(const iType0& i0, const iType1& i1, const iType2& i2, + const iType3& i3) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (4, this->rank(), m_track, m_map, i0, i1, i2, i3)) + return m_map.reference(i0, i1, i2, i3, 0, 0, 0); + } // Rank 5 - template< typename iType0 , typename iType1 , typename iType2 , typename iType3, typename iType4 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_same::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type>::type - operator()(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , const iType4 & i4 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (5 , this->rank(), m_track, m_map, i0, i1, i2, i3, i4) ) - return m_map.reference(i0,i1,i2,i3,i4); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_same::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value), + reference_type>::type + operator()(const iType0& i0, const iType1& i1, const iType2& i2, + const iType3& i3, const iType4& i4) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (5, this->rank(), m_track, m_map, i0, i1, i2, i3, i4)) + return m_map.reference(i0, i1, i2, i3, i4); + } - template< typename iType0 , typename iType1 , typename iType2 , typename iType3, typename iType4 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< !(std::is_same::value && std::is_integral::value), reference_type>::type - operator()(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , const iType4 & i4 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (5 , this->rank(), m_track, m_map, i0, i1, i2, i3, i4) ) - return m_map.reference(i0,i1,i2,i3,i4,0,0); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + !(std::is_same::value && + std::is_integral::value), + reference_type>::type + operator()(const iType0& i0, const iType1& i1, const iType2& i2, + const iType3& i3, const iType4& i4) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (5, this->rank(), m_track, m_map, i0, i1, i2, i3, i4)) + return m_map.reference(i0, i1, i2, i3, i4, 0, 0); + } // Rank 6 - template< typename iType0 , typename iType1 , typename iType2 , typename iType3, typename iType4 , typename iType5 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_same::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type>::type - operator()(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , const iType4 & i4 , const iType5 & i5 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (6 , this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5) ) - return m_map.reference(i0,i1,i2,i3,i4,i5); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_same::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value && std::is_integral::value), + reference_type>::type + operator()(const iType0& i0, const iType1& i1, const iType2& i2, + const iType3& i3, const iType4& i4, const iType5& i5) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (6, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5)) + return m_map.reference(i0, i1, i2, i3, i4, i5); + } - template< typename iType0 , typename iType1 , typename iType2 , typename iType3, typename iType4 , typename iType5 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< !(std::is_same::value && std::is_integral::value), reference_type>::type - operator()(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , const iType4 & i4 , const iType5 & i5 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (6 , this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5) ) - return m_map.reference(i0,i1,i2,i3,i4,i5,0); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + !(std::is_same::value && + std::is_integral::value), + reference_type>::type + operator()(const iType0& i0, const iType1& i1, const iType2& i2, + const iType3& i3, const iType4& i4, const iType5& i5) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (6, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5)) + return m_map.reference(i0, i1, i2, i3, i4, i5, 0); + } // Rank 7 - template< typename iType0 , typename iType1 , typename iType2 , typename iType3, typename iType4 , typename iType5 , typename iType6 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type>::type - operator()(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , const iType4 & i4 , const iType5 & i5 , const iType6 & i6 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (7 , this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5, i6) ) - return m_map.reference(i0,i1,i2,i3,i4,i5,i6); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_integral::value && std::is_integral::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value), + reference_type>::type + operator()(const iType0& i0, const iType1& i1, const iType2& i2, + const iType3& i3, const iType4& i4, const iType5& i5, + const iType6& i6) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (7, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5, i6)) + return m_map.reference(i0, i1, i2, i3, i4, i5, i6); + } // Rank 0 KOKKOS_INLINE_FUNCTION - reference_type access() const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (0 , this->rank(), m_track, m_map) ) - return impl_map().reference(); - //return m_map.reference(0,0,0,0,0,0,0); - } + reference_type access() const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((0, this->rank(), m_track, m_map)) + return impl_map().reference(); + // return m_map.reference(0,0,0,0,0,0,0); + } // Rank 1 - // Rank 1 parenthesis - template< typename iType > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_same::value && std::is_integral::value), reference_type>::type - access(const iType & i0 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (1 , this->rank(), m_track, m_map, i0) ) - return m_map.reference(i0); - } + // Rank 1 parenthesis + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_same::value && + std::is_integral::value), + reference_type>::type + access(const iType& i0) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((1, this->rank(), m_track, m_map, i0)) + return m_map.reference(i0); + } - template< typename iType > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< !(std::is_same::value && std::is_integral::value), reference_type>::type - access(const iType & i0 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (1 , this->rank(), m_track, m_map, i0) ) - return m_map.reference(i0,0,0,0,0,0,0); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + !(std::is_same::value && + std::is_integral::value), + reference_type>::type + access(const iType& i0) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((1, this->rank(), m_track, m_map, i0)) + return m_map.reference(i0, 0, 0, 0, 0, 0, 0); + } // Rank 2 - template< typename iType0 , typename iType1 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_same::value && std::is_integral::value && std::is_integral::value), reference_type>::type - access(const iType0 & i0 , const iType1 & i1 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (2 , this->rank(), m_track, m_map, i0, i1) ) - return m_map.reference(i0,i1); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_same::value && + std::is_integral::value && std::is_integral::value), + reference_type>::type + access(const iType0& i0, const iType1& i1) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((2, this->rank(), m_track, m_map, i0, i1)) + return m_map.reference(i0, i1); + } - template< typename iType0 , typename iType1 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< !(std::is_same::value && std::is_integral::value), reference_type>::type - access(const iType0 & i0 , const iType1 & i1 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (2 , this->rank(), m_track, m_map, i0, i1) ) - return m_map.reference(i0,i1,0,0,0,0,0); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + !(std::is_same::value && + std::is_integral::value), + reference_type>::type + access(const iType0& i0, const iType1& i1) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((2, this->rank(), m_track, m_map, i0, i1)) + return m_map.reference(i0, i1, 0, 0, 0, 0, 0); + } // Rank 3 - template< typename iType0 , typename iType1 , typename iType2 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_same::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type>::type - access(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (3 , this->rank(), m_track, m_map, i0, i1, i2) ) - return m_map.reference(i0,i1,i2); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_same::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value), + reference_type>::type + access(const iType0& i0, const iType1& i1, const iType2& i2) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (3, this->rank(), m_track, m_map, i0, i1, i2)) + return m_map.reference(i0, i1, i2); + } - template< typename iType0 , typename iType1 , typename iType2 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< !(std::is_same::value && std::is_integral::value), reference_type>::type - access(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (3 , this->rank(), m_track, m_map, i0, i1, i2) ) - return m_map.reference(i0,i1,i2,0,0,0,0); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + !(std::is_same::value && + std::is_integral::value), + reference_type>::type + access(const iType0& i0, const iType1& i1, const iType2& i2) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (3, this->rank(), m_track, m_map, i0, i1, i2)) + return m_map.reference(i0, i1, i2, 0, 0, 0, 0); + } // Rank 4 - template< typename iType0 , typename iType1 , typename iType2 , typename iType3 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_same::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type>::type - access(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (4 , this->rank(), m_track, m_map, i0, i1, i2, i3) ) - return m_map.reference(i0,i1,i2,i3); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_same::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value && std::is_integral::value), + reference_type>::type + access(const iType0& i0, const iType1& i1, const iType2& i2, + const iType3& i3) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (4, this->rank(), m_track, m_map, i0, i1, i2, i3)) + return m_map.reference(i0, i1, i2, i3); + } - template< typename iType0 , typename iType1 , typename iType2 , typename iType3 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< !(std::is_same::value && std::is_integral::value), reference_type>::type - access(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (4 , this->rank(), m_track, m_map, i0, i1, i2, i3) ) - return m_map.reference(i0,i1,i2,i3,0,0,0); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + !(std::is_same::value && + std::is_integral::value), + reference_type>::type + access(const iType0& i0, const iType1& i1, const iType2& i2, + const iType3& i3) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (4, this->rank(), m_track, m_map, i0, i1, i2, i3)) + return m_map.reference(i0, i1, i2, i3, 0, 0, 0); + } // Rank 5 - template< typename iType0 , typename iType1 , typename iType2 , typename iType3, typename iType4 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_same::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type>::type - access(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , const iType4 & i4 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (5 , this->rank(), m_track, m_map, i0, i1, i2, i3, i4) ) - return m_map.reference(i0,i1,i2,i3,i4); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_same::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value), + reference_type>::type + access(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3, + const iType4& i4) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (5, this->rank(), m_track, m_map, i0, i1, i2, i3, i4)) + return m_map.reference(i0, i1, i2, i3, i4); + } - template< typename iType0 , typename iType1 , typename iType2 , typename iType3, typename iType4 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< !(std::is_same::value && std::is_integral::value), reference_type>::type - access(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , const iType4 & i4 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (5 , this->rank(), m_track, m_map, i0, i1, i2, i3, i4) ) - return m_map.reference(i0,i1,i2,i3,i4,0,0); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + !(std::is_same::value && + std::is_integral::value), + reference_type>::type + access(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3, + const iType4& i4) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (5, this->rank(), m_track, m_map, i0, i1, i2, i3, i4)) + return m_map.reference(i0, i1, i2, i3, i4, 0, 0); + } // Rank 6 - template< typename iType0 , typename iType1 , typename iType2 , typename iType3, typename iType4 , typename iType5 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_same::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type>::type - access(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , const iType4 & i4 , const iType5 & i5 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (6 , this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5) ) - return m_map.reference(i0,i1,i2,i3,i4,i5); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_same::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value && std::is_integral::value), + reference_type>::type + access(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3, + const iType4& i4, const iType5& i5) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (6, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5)) + return m_map.reference(i0, i1, i2, i3, i4, i5); + } - template< typename iType0 , typename iType1 , typename iType2 , typename iType3, typename iType4 , typename iType5 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< !(std::is_same::value && std::is_integral::value), reference_type>::type - access(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , const iType4 & i4 , const iType5 & i5 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (6 , this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5) ) - return m_map.reference(i0,i1,i2,i3,i4,i5,0); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + !(std::is_same::value && + std::is_integral::value), + reference_type>::type + access(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3, + const iType4& i4, const iType5& i5) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (6, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5)) + return m_map.reference(i0, i1, i2, i3, i4, i5, 0); + } // Rank 7 - template< typename iType0 , typename iType1 , typename iType2 , typename iType3, typename iType4 , typename iType5 , typename iType6 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type>::type - access(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , const iType4 & i4 , const iType5 & i5 , const iType6 & i6 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (7 , this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5, i6) ) - return m_map.reference(i0,i1,i2,i3,i4,i5,i6); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_integral::value && std::is_integral::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value), + reference_type>::type + access(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3, + const iType4& i4, const iType5& i5, const iType6& i6) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (7, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5, i6)) + return m_map.reference(i0, i1, i2, i3, i4, i5, i6); + } #undef KOKKOS_IMPL_VIEW_OPERATOR_VERIFY @@ -927,405 +1069,393 @@ public: ~DynRankView() {} KOKKOS_INLINE_FUNCTION - DynRankView() : m_track(), m_map(), m_rank() {} //Default ctor + DynRankView() : m_track(), m_map(), m_rank() {} // Default ctor KOKKOS_INLINE_FUNCTION - DynRankView( const DynRankView & rhs ) : m_track( rhs.m_track ), m_map( rhs.m_map ), m_rank(rhs.m_rank) {} + DynRankView(const DynRankView& rhs) + : m_track(rhs.m_track), m_map(rhs.m_map), m_rank(rhs.m_rank) {} KOKKOS_INLINE_FUNCTION - DynRankView( DynRankView && rhs ) : m_track( rhs.m_track ), m_map( rhs.m_map ), m_rank(rhs.m_rank) {} + DynRankView(DynRankView&& rhs) + : m_track(rhs.m_track), m_map(rhs.m_map), m_rank(rhs.m_rank) {} KOKKOS_INLINE_FUNCTION - DynRankView & operator = ( const DynRankView & rhs ) { m_track = rhs.m_track; m_map = rhs.m_map; m_rank = rhs.m_rank; return *this; } + DynRankView& operator=(const DynRankView& rhs) { + m_track = rhs.m_track; + m_map = rhs.m_map; + m_rank = rhs.m_rank; + return *this; + } KOKKOS_INLINE_FUNCTION - DynRankView & operator = ( DynRankView && rhs ) { m_track = rhs.m_track; m_map = rhs.m_map; m_rank = rhs.m_rank; return *this; } + DynRankView& operator=(DynRankView&& rhs) { + m_track = rhs.m_track; + m_map = rhs.m_map; + m_rank = rhs.m_rank; + return *this; + } //---------------------------------------- // Compatible view copy constructor and assignment // may assign unmanaged from managed. - template< class RT , class ... RP > - KOKKOS_INLINE_FUNCTION - DynRankView( const DynRankView & rhs ) - : m_track( rhs.m_track , traits::is_managed ) - , m_map() - , m_rank(rhs.m_rank) - { - typedef typename DynRankView ::traits SrcTraits ; - typedef Kokkos::Impl::ViewMapping< traits , SrcTraits , typename traits::specialize > Mapping ; - static_assert( Mapping::is_assignable , "Incompatible DynRankView copy construction" ); - Mapping::assign( m_map , rhs.m_map , rhs.m_track ); - } + template + KOKKOS_INLINE_FUNCTION DynRankView(const DynRankView& rhs) + : m_track(rhs.m_track, traits::is_managed), m_map(), m_rank(rhs.m_rank) { + typedef typename DynRankView::traits SrcTraits; + typedef Kokkos::Impl::ViewMapping + Mapping; + static_assert(Mapping::is_assignable, + "Incompatible DynRankView copy construction"); + Mapping::assign(m_map, rhs.m_map, rhs.m_track); + } - template< class RT , class ... RP > - KOKKOS_INLINE_FUNCTION - DynRankView & operator = (const DynRankView & rhs ) - { - typedef typename DynRankView ::traits SrcTraits ; - typedef Kokkos::Impl::ViewMapping< traits , SrcTraits , typename traits::specialize > Mapping ; - static_assert( Mapping::is_assignable , "Incompatible DynRankView copy construction" ); - Mapping::assign( m_map , rhs.m_map , rhs.m_track ); - m_track.assign( rhs.m_track , traits::is_managed ); - m_rank = rhs.rank(); - return *this; - } + template + KOKKOS_INLINE_FUNCTION DynRankView& operator=( + const DynRankView& rhs) { + typedef typename DynRankView::traits SrcTraits; + typedef Kokkos::Impl::ViewMapping + Mapping; + static_assert(Mapping::is_assignable, + "Incompatible DynRankView copy construction"); + Mapping::assign(m_map, rhs.m_map, rhs.m_track); + m_track.assign(rhs.m_track, traits::is_managed); + m_rank = rhs.rank(); + return *this; + } -// Copy/Assign View to DynRankView - template< class RT , class ... RP > - KOKKOS_INLINE_FUNCTION - DynRankView( const View & rhs ) - : m_track() - , m_map() - , m_rank( rhs.Rank ) - { - typedef typename View::traits SrcTraits ; - typedef Kokkos::Impl::ViewMapping< traits , SrcTraits , Kokkos::Impl::ViewToDynRankViewTag > Mapping ; - static_assert( Mapping::is_assignable , "Incompatible View to DynRankView copy construction" ); - Mapping::assign( *this , rhs ); - } + // Copy/Assign View to DynRankView + template + KOKKOS_INLINE_FUNCTION DynRankView(const View& rhs) + : m_track(), m_map(), m_rank(rhs.Rank) { + typedef typename View::traits SrcTraits; + typedef Kokkos::Impl::ViewMapping + Mapping; + static_assert(Mapping::is_assignable, + "Incompatible View to DynRankView copy construction"); + Mapping::assign(*this, rhs); + } - template< class RT , class ... RP > - KOKKOS_INLINE_FUNCTION - DynRankView & operator = ( const View & rhs ) - { - typedef typename View::traits SrcTraits ; - typedef Kokkos::Impl::ViewMapping< traits , SrcTraits , Kokkos::Impl::ViewToDynRankViewTag > Mapping ; - static_assert( Mapping::is_assignable , "Incompatible View to DynRankView copy assignment" ); - Mapping::assign( *this , rhs ); - return *this ; - } + template + KOKKOS_INLINE_FUNCTION DynRankView& operator=(const View& rhs) { + typedef typename View::traits SrcTraits; + typedef Kokkos::Impl::ViewMapping + Mapping; + static_assert(Mapping::is_assignable, + "Incompatible View to DynRankView copy assignment"); + Mapping::assign(*this, rhs); + return *this; + } //---------------------------------------- // Allocation tracking properties KOKKOS_INLINE_FUNCTION - int use_count() const - { return m_track.use_count(); } + int use_count() const { return m_track.use_count(); } - inline - const std::string label() const - { return m_track.template get_label< typename traits::memory_space >(); } + inline const std::string label() const { + return m_track.template get_label(); + } //---------------------------------------- // Allocation according to allocation properties and array layout - // unused arg_layout dimensions must be set to KOKKOS_INVALID_INDEX so that rank deduction can properly take place - template< class ... P > - explicit inline - DynRankView( const Kokkos::Impl::ViewCtorProp< P ... > & arg_prop - , typename std::enable_if< ! Kokkos::Impl::ViewCtorProp< P... >::has_pointer - , typename traits::array_layout - >::type const & arg_layout - ) - : m_track() - , m_map() - , m_rank( Impl::DynRankDimTraits::template computeRank< typename traits::array_layout, P...>(arg_prop, arg_layout) ) - { - // Append layout and spaces if not input - typedef Kokkos::Impl::ViewCtorProp< P ... > alloc_prop_input ; - - // use 'std::integral_constant' for non-types - // to avoid duplicate class error. - typedef Kokkos::Impl::ViewCtorProp - < P ... - , typename std::conditional - < alloc_prop_input::has_label - , std::integral_constant - , typename std::string - >::type - , typename std::conditional - < alloc_prop_input::has_memory_space - , std::integral_constant - , typename traits::device_type::memory_space - >::type - , typename std::conditional - < alloc_prop_input::has_execution_space - , std::integral_constant - , typename traits::device_type::execution_space - >::type - > alloc_prop ; - - static_assert( traits::is_managed - , "View allocation constructor requires managed memory" ); - - if ( alloc_prop::initialize && + // unused arg_layout dimensions must be set to KOKKOS_INVALID_INDEX so that + // rank deduction can properly take place + template + explicit inline DynRankView( + const Kokkos::Impl::ViewCtorProp& arg_prop, + typename std::enable_if::has_pointer, + typename traits::array_layout>::type const& + arg_layout) + : m_track(), + m_map(), + m_rank(Impl::DynRankDimTraits:: + template computeRank( + arg_prop, arg_layout)) { + // Append layout and spaces if not input + typedef Kokkos::Impl::ViewCtorProp alloc_prop_input; + + // use 'std::integral_constant' for non-types + // to avoid duplicate class error. + typedef Kokkos::Impl::ViewCtorProp< + P..., + typename std::conditional, + typename std::string>::type, + typename std::conditional< + alloc_prop_input::has_memory_space, + std::integral_constant, + typename traits::device_type::memory_space>::type, + typename std::conditional< + alloc_prop_input::has_execution_space, + std::integral_constant, + typename traits::device_type::execution_space>::type> + alloc_prop; + + static_assert(traits::is_managed, + "View allocation constructor requires managed memory"); + + if (alloc_prop::initialize && #ifdef KOKKOS_ENABLE_DEPRECATED_CODE - ! alloc_prop::execution_space::is_initialized() + !alloc_prop::execution_space::is_initialized() #else - ! alloc_prop::execution_space::impl_is_initialized() + !alloc_prop::execution_space::impl_is_initialized() #endif - ) { - // If initializing view data then - // the execution space must be initialized. - Kokkos::Impl::throw_runtime_exception("Constructing DynRankView and initializing data with uninitialized execution space"); - } + ) { + // If initializing view data then + // the execution space must be initialized. + Kokkos::Impl::throw_runtime_exception( + "Constructing DynRankView and initializing data with uninitialized " + "execution space"); + } - // Copy the input allocation properties with possibly defaulted properties - alloc_prop prop_copy( arg_prop ); + // Copy the input allocation properties with possibly defaulted properties + alloc_prop prop_copy(arg_prop); //------------------------------------------------------------ -#if defined( KOKKOS_ENABLE_CUDA ) - // If allocating in CudaUVMSpace must fence before and after - // the allocation to protect against possible concurrent access - // on the CPU and the GPU. - // Fence using the trait's executon space (which will be Kokkos::Cuda) - // to avoid incomplete type errors from usng Kokkos::Cuda directly. - if ( std::is_same< Kokkos::CudaUVMSpace , typename traits::device_type::memory_space >::value ) { - typename traits::device_type::memory_space::execution_space().fence(); - } +#if defined(KOKKOS_ENABLE_CUDA) + // If allocating in CudaUVMSpace must fence before and after + // the allocation to protect against possible concurrent access + // on the CPU and the GPU. + // Fence using the trait's executon space (which will be Kokkos::Cuda) + // to avoid incomplete type errors from usng Kokkos::Cuda directly. + if (std::is_same::value) { + typename traits::device_type::memory_space::execution_space().fence(); + } #endif -//------------------------------------------------------------ + //------------------------------------------------------------ - Kokkos::Impl::SharedAllocationRecord<> * - record = m_map.allocate_shared( prop_copy, Impl::DynRankDimTraits::template createLayout(arg_prop, arg_layout) ); + Kokkos::Impl::SharedAllocationRecord<>* record = m_map.allocate_shared( + prop_copy, + Impl::DynRankDimTraits:: + template createLayout(arg_prop, arg_layout)); //------------------------------------------------------------ -#if defined( KOKKOS_ENABLE_CUDA ) - if ( std::is_same< Kokkos::CudaUVMSpace , typename traits::device_type::memory_space >::value ) { - typename traits::device_type::memory_space::execution_space().fence(); - } -#endif -//------------------------------------------------------------ - - // Setup and initialization complete, start tracking - m_track.assign_allocated_record_to_uninitialized( record ); +#if defined(KOKKOS_ENABLE_CUDA) + if (std::is_same::value) { + typename traits::device_type::memory_space::execution_space().fence(); } +#endif + //------------------------------------------------------------ + // Setup and initialization complete, start tracking + m_track.assign_allocated_record_to_uninitialized(record); + } // Wrappers - template< class ... P > - explicit KOKKOS_INLINE_FUNCTION - DynRankView( const Kokkos::Impl::ViewCtorProp< P ... > & arg_prop - , typename std::enable_if< Kokkos::Impl::ViewCtorProp< P... >::has_pointer - , typename traits::array_layout - >::type const & arg_layout - ) - : m_track() // No memory tracking - , m_map( arg_prop , Impl::DynRankDimTraits::template createLayout(arg_prop, arg_layout) ) - , m_rank( Impl::DynRankDimTraits::template computeRank< typename traits::array_layout, P...>(arg_prop, arg_layout) ) - { - static_assert( - std::is_same< pointer_type - , typename Impl::ViewCtorProp< P... >::pointer_type - >::value , - "Constructing DynRankView to wrap user memory must supply matching pointer type" ); - } + template + explicit KOKKOS_INLINE_FUNCTION DynRankView( + const Kokkos::Impl::ViewCtorProp& arg_prop, + typename std::enable_if::has_pointer, + typename traits::array_layout>::type const& + arg_layout) + : m_track() // No memory tracking + , + m_map(arg_prop, + Impl::DynRankDimTraits:: + template createLayout(arg_prop, arg_layout)), + m_rank(Impl::DynRankDimTraits:: + template computeRank( + arg_prop, arg_layout)) { + static_assert( + std::is_same::pointer_type>::value, + "Constructing DynRankView to wrap user memory must supply matching " + "pointer type"); + } //---------------------------------------- - //Constructor(s) + // Constructor(s) // Simple dimension-only layout - template< class ... P > - explicit inline - DynRankView( const Kokkos::Impl::ViewCtorProp< P ... > & arg_prop - , typename std::enable_if< ! Kokkos::Impl::ViewCtorProp< P... >::has_pointer - , size_t - >::type const arg_N0 =KOKKOS_INVALID_INDEX - , const size_t arg_N1 =KOKKOS_INVALID_INDEX - , const size_t arg_N2 =KOKKOS_INVALID_INDEX - , const size_t arg_N3 =KOKKOS_INVALID_INDEX - , const size_t arg_N4 =KOKKOS_INVALID_INDEX - , const size_t arg_N5 =KOKKOS_INVALID_INDEX - , const size_t arg_N6 =KOKKOS_INVALID_INDEX - , const size_t arg_N7 =KOKKOS_INVALID_INDEX - ) - : DynRankView( arg_prop - , typename traits::array_layout - ( arg_N0 , arg_N1 , arg_N2 , arg_N3 , arg_N4 , arg_N5 , arg_N6 , arg_N7 ) - ) - {} - - template< class ... P > - explicit KOKKOS_INLINE_FUNCTION - DynRankView( const Kokkos::Impl::ViewCtorProp< P ... > & arg_prop - , typename std::enable_if< Kokkos::Impl::ViewCtorProp< P... >::has_pointer - , size_t - >::type const arg_N0 =KOKKOS_INVALID_INDEX - , const size_t arg_N1 =KOKKOS_INVALID_INDEX - , const size_t arg_N2 =KOKKOS_INVALID_INDEX - , const size_t arg_N3 =KOKKOS_INVALID_INDEX - , const size_t arg_N4 =KOKKOS_INVALID_INDEX - , const size_t arg_N5 =KOKKOS_INVALID_INDEX - , const size_t arg_N6 =KOKKOS_INVALID_INDEX - , const size_t arg_N7 =KOKKOS_INVALID_INDEX - ) - : DynRankView( arg_prop - , typename traits::array_layout - ( arg_N0 , arg_N1 , arg_N2 , arg_N3 , arg_N4 , arg_N5 , arg_N6 , arg_N7 ) - ) - {} + template + explicit inline DynRankView( + const Kokkos::Impl::ViewCtorProp& arg_prop, + typename std::enable_if::has_pointer, + size_t>::type const arg_N0 = KOKKOS_INVALID_INDEX, + const size_t arg_N1 = KOKKOS_INVALID_INDEX, + const size_t arg_N2 = KOKKOS_INVALID_INDEX, + const size_t arg_N3 = KOKKOS_INVALID_INDEX, + const size_t arg_N4 = KOKKOS_INVALID_INDEX, + const size_t arg_N5 = KOKKOS_INVALID_INDEX, + const size_t arg_N6 = KOKKOS_INVALID_INDEX, + const size_t arg_N7 = KOKKOS_INVALID_INDEX) + : DynRankView(arg_prop, typename traits::array_layout( + arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, + arg_N5, arg_N6, arg_N7)) {} + + template + explicit KOKKOS_INLINE_FUNCTION DynRankView( + const Kokkos::Impl::ViewCtorProp& arg_prop, + typename std::enable_if::has_pointer, + size_t>::type const arg_N0 = KOKKOS_INVALID_INDEX, + const size_t arg_N1 = KOKKOS_INVALID_INDEX, + const size_t arg_N2 = KOKKOS_INVALID_INDEX, + const size_t arg_N3 = KOKKOS_INVALID_INDEX, + const size_t arg_N4 = KOKKOS_INVALID_INDEX, + const size_t arg_N5 = KOKKOS_INVALID_INDEX, + const size_t arg_N6 = KOKKOS_INVALID_INDEX, + const size_t arg_N7 = KOKKOS_INVALID_INDEX) + : DynRankView(arg_prop, typename traits::array_layout( + arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, + arg_N5, arg_N6, arg_N7)) {} // Allocate with label and layout - template< typename Label > - explicit inline - DynRankView( const Label & arg_label - , typename std::enable_if< - Kokkos::Impl::is_view_label