generate a HTML index of test failures (fca2efed) · Commits · github_fork / Clip

.gitignore

+1 −0

Original line number	Diff line number	Diff line
		@@ -21,3 +21,4 @@ install_manifest.txt
		wwwdocs
		test
		config.h
		test-results

CMakeLists.txt

+5 −41

Original line number	Diff line number	Diff line
		cmake_minimum_required(VERSION 3.8)
		project(fviz VERSION "0.5.0")
		set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/platform/cmake")
		enable_testing()


		# Options
		@@ -107,47 +106,12 @@ install(TARGETS fviz-cli DESTINATION bin)

		# Testing
		# -----------------------------------------------------------------------------
		add_test(test-prepare-dir ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/test)
		set_tests_properties(test-prepare-dir PROPERTIES FIXTURES_SETUP text-fixtures)

		add_test(test-prepare-examples ${CMAKE_CURRENT_SOURCE_DIR}/examples/update_tests.sh)
		set_tests_properties(test-prepare-examples PROPERTIES FIXTURES_SETUP text-fixtures)

		file(GLOB unit_test_files "tests/unit/test_*.cc")
		foreach(unit_test_path ${unit_test_files})
		get_filename_component(unit_test_name ${unit_test_path} NAME_WE)
		get_filename_component(unit_test_srcdir ${unit_test_path} DIRECTORY)

		add_executable(${unit_test_name} ${unit_test_path} $<TARGET_OBJECTS:fviz>)
		target_link_libraries(${unit_test_name} ${FVIZ_LIB_LDFLAGS})

		add_test(
		NAME test-unit-${unit_test_name}
		COMMAND ${CMAKE_COMMAND} -E env
		FVIZ_TEST_SRCDIR=${CMAKE_SOURCE_DIR}
		${CMAKE_CURRENT_BINARY_DIR}/${unit_test_name}
		DEPENDS fviz test-prepare)

		set_tests_properties(
		test-unit-${unit_test_name}
		PROPERTIES FIXTURES_REQUIRED text-fixtures)
		endforeach()

		file(GLOB spec_test_files "tests/spec/*.fvz")
		foreach(spec_test_path ${spec_test_files})
		get_filename_component(spec_test_name ${spec_test_path} NAME_WE)
		get_filename_component(spec_test_srcdir ${spec_test_path} DIRECTORY)
		add_custom_target(test
		COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_runner.sh
		DEPENDS fviz-cli)

		add_test(
		NAME test-spec-${spec_test_name}
		WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
		COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_runner.sh ${CMAKE_CURRENT_BINARY_DIR}/fviz ${spec_test_name} ${spec_test_srcdir} ${CMAKE_CURRENT_BINARY_DIR}/test
		DEPENDS fviz-cli test-prepare)

		set_tests_properties(
		test-spec-${spec_test_name}
		PROPERTIES FIXTURES_REQUIRED text-fixtures)
		endforeach()
		add_custom_target(test-examples ${CMAKE_CURRENT_SOURCE_DIR}/examples/update_tests.sh)
		add_dependencies(test test-examples)


		# Examples

tests/test_runner.sh

+214 −71

Original line number	Diff line number	Diff line
		#!/bin/bash
		set -e
		set -ue

		binfile="$1"
		testname="$2"
		srcdir="$3"
		tmpdir="$4"
		format="svg"
		source_path="$(realpath "$(dirname "$0")/..")"
		proc_path="$(realpath "./fviz")"
		test_path="${source_path}/tests"
		result_path="$(realpath ./test-results)"

		infile="${srcdir}/${testname}.fvz"
		reffile="${srcdir}/${testname}.${format}"
		errfile="${srcdir}/${testname}.err"
		outfile="${tmpdir}/${testname}.${format}"
		logfile="${tmpdir}/${testname}.log"
		run() {
		mkdir -p "${result_path}"
		log_init

		num_total=0
		num_pass=0
		num_fail=0

		test_cases=$((cd "${source_path}/tests/spec" && find . -name "*.fvz") \| sort)

		for test_id in ${test_cases[@]}; do
		local test_file="$(basename ${test_id})"
		local test_id="${test_file%.*}"

		num_total=$[ $num_total + 1 ]

		if run_test "${test_id}"; then
		echo "$(printf "\033[1;32m%s\033[0m" "[PASS]") ${test_id}"
		num_pass=$[ $num_pass + 1 ]
		else
		echo "$(printf "\033[1;31m%s\033[0m" "[FAIL]") ${test_id}"
		num_fail=$[ $num_fail + 1 ]
		fi
		done

		if [[ ${num_total} == ${num_pass} ]]; then
		echo
		echo "TEST RESULT: $(printf "\033[1;32m%s\033[0m" "PASS")"
		echo
		echo "${num_pass}/${num_total} test cases ok"
		exit 0
		else
		echo
		echo "Test Result: $(printf "\033[1;31m%s\033[0m" "FAIL")"
		echo
		echo "${num_fail}/${num_total} test cases failed"
		exit 1
		fi
		}

		run_test() {
		local test_id="$1"
		local format="svg"

		local infile="${test_path}/spec/${test_id}.fvz"
		local reffile="${test_path}/spec/${test_id}.${format}"
		local errfile="${test_path}/spec/${test_id}.err"
		local outfile="${result_path}/${test_id}.${format}"
		local logfile="${result_path}/${test_id}.log"

		# clean up old files
		rm -rf "${outfile}" "${logfile}"

		# run fviz
		echo "-------------------------------------------------------------------------------"
		echo "${binfile}" --in "${infile}" --out "${outfile}"
		echo "-------------------------------------------------------------------------------"

		result=""
		if "${binfile}" \
		if (cd ${source_path} && "${proc_path}" \
		--font-defaults off \
		--font-load "tests/testdata/fonts/LiberationSans-Regular.ttf" \
		--in "${infile}" \
		--out "${outfile}" \
		2> "${logfile}"; then
		2> "${logfile}"); then
		result="ok"
		else
		result="fail"
		@@ -36,40 +75,144 @@ fi
		# check error messages
		if [[ -e "${errfile}" ]]; then
		if [[ ${result} != "fail" ]]; then
		echo "ERROR: expected failure but got success"
		cat ${logfile}
		exit 1
		echo "ERROR: expected failure but got success" >> ${logfile}
		return 1
		fi

		if (diff ${logfile} ${errfile} &>/dev/null); then
		exit 0
		return 0
		else
		echo "ERROR: error messages do not match"
		diff ${logfile} ${errfile} \|\| true
		exit 1
		echo "ERROR: error messages do not match" >> ${logfile}
		return 1
		fi
		fi

		# check result
		if [[ ${result} != "ok" ]]; then
		echo "ERROR: execution failed"
		echo "ERROR: execution failed" >> ${logfile}
		cat ${logfile}
		exit 1
		return 0
		fi

		if [[ ! -e ${reffile} \|\| ! -z "${FVIZ_TEST_FORCE}" ]]; then
		if [[ ! -e ${reffile} ]]; then
		cp ${outfile} ${reffile}
		fi

		if (diff ${outfile} ${reffile} &>/dev/null); then
		cat ${logfile}
		exit 0
		return 0
		else
		echo "ERROR: output files do not match"
		echo "OUTPUT:" ${outfile}
		echo "EXPECT:" ${reffile}
		echo "-------------------------------------------------------------------------------"
		#cat ${logfile}
		#diff ${outfile} ${reffile} \|\| true
		exit 1
		log_failure "${test_id}" "${outfile}" "${reffile}" "${logfile}"
		return 1
		fi
		}

		log_append() {
		echo "$1" >> "${result_path}/index.html"
		}

		log_init() {
		(cat > "${result_path}/index.html") <<-EOF
		<title>Test Results</title>
		<style type='text/css'>
		body {
		max-width: 1200px;
		font-family: sans-serif;
		}

		th {
		background: #eee;
		padding: 2pt;
		}

		td, th {
		text-align: left;
		}

		table {
		border: 1px solid #000;
		}

		section {
		margin-bottom: 2em;
		}

		img {
		max-width: 100%;
		}

		h3 {
		margin: 4pt 0;
		}
		</style>

		EOF
		}

		log_failure() {
		local test_id="$1"
		local test_result_path="$2"
		local test_expectation_path="$3"
		local test_logfile_path="$4"

		log_append "<section>"
		log_append " <table width='100%'>"
		log_append " <tr>"
		log_append " <th colspan=2>"
		log_append " <h3>Test Failure: ${test_id}</h3>"
		log_append " </th>"
		log_append " </tr>"
		log_append " <tr>"
		log_append " <th>Result</th>"
		log_append " <th>Expectation</th>"
		log_append " </tr>"
		log_append " <tr>"
		log_append " <td><img src='${test_result_path}' /></td>"
		log_append " <td><img src='${test_expectation_path}' /></td>"
		log_append " </tr>"
		log_append " <tr>"
		log_append " <th colspan=2>Command</th>"
		log_append " </tr>"
		log_append " <tr>"
		log_append " <td><code>$ ${proc_path} --in tests/spec/${test_id}.fvz --out /tmp/t.svg</code></td>"
		log_append " </tr>"
		log_append " <tr>"
		log_append " <th colspan=2>Reference File</th>"
		log_append " </tr>"
		log_append " <tr>"
		log_append " <td><code>$(realpath "${test_expectation_path}")</code></td>"
		log_append " </tr>"
		log_append " <tr>"
		log_append " <th colspan=2>Details</th>"
		log_append " </tr>"
		log_append " <tr>"
		log_append " <pre>$(cat ${test_logfile_path})</pre>"
		log_append " </tr>"
		log_append " <tr>"
		log_append " <td>"
		log_append " <details>"
		log_append " <summary>Logfile</summary>"
		log_append " <pre>$(log_escape < ${test_logfile_path})</pre>"
		log_append " </details>"
		log_append " </td>"
		log_append " </tr>"
		log_append " <tr>"
		log_append " <td>"
		log_append " <details>"
		log_append " <summary>SVG Diff</summary>"
		log_append " <pre>$(diff ${test_result_path} ${test_expectation_path} \| log_escape)</pre>"
		log_append " </details>"
		log_append " </td>"
		log_append " </tr>"
		log_append " </table>"
		log_append "</section>"
		}

		log_escape() {
		sed \
		-e 's/&/\&/g' \
		-e 's/</\</g' \
		-e 's/>/\>/g'
		}

		run

Admin message