diff --git a/cmake/generate-tests b/cmake/generate-tests
index deaacd6c..7edae32f 100755
--- a/cmake/generate-tests
+++ b/cmake/generate-tests
@@ -1,2 +1,2 @@
 cd ${0%/*}
-../third-party/cxxtest/bin/cxxtestgen --runner=ErrorPrinter -o ../tests/runner-autogen.cpp ../tests/*.hpp ../source/*/tests/*.hpp
\ No newline at end of file
+../third-party/cxxtest/bin/cxxtestgen --runner=ErrorPrinter -o "$1"/tests/runner-autogen.cpp ../tests/*.hpp ../source/*/tests/*.hpp
diff --git a/cmake/generate-tests.bat b/cmake/generate-tests.bat
index 269338a2..70960f71 100644
--- a/cmake/generate-tests.bat
+++ b/cmake/generate-tests.bat
@@ -1,2 +1,2 @@
 cd %~dp0
-../third-party/cxxtest/bin/cxxtestgen --runner=ErrorPrinter -o ../tests/runner-autogen.cpp ../tests/*.hpp ../source/*/tests/*.hpp
\ No newline at end of file
+../third-party/cxxtest/bin/cxxtestgen --runner=ErrorPrinter -o "%1"/tests/runner-autogen.cpp ../tests/*.hpp ../source/*/tests/*.hpp
diff --git a/cmake/xlnt.test.cmake b/cmake/xlnt.test.cmake
index 3b97f9de..81e90a6e 100644
--- a/cmake/xlnt.test.cmake
+++ b/cmake/xlnt.test.cmake
@@ -25,7 +25,9 @@ FILE(GLOB TEST_HELPERS_SOURCES tests/helpers/*.cpp)
 
 SET(TEST_HELPERS ${TEST_HELPERS_HEADERS} ${TEST_HELPERS_SOURCES})
 
-SET(RUNNER tests/runner-autogen.cpp)
+file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/tests")
+SET(RUNNER "${CMAKE_CURRENT_BINARY_DIR}/tests/runner-autogen.cpp")
+SET_SOURCE_FILES_PROPERTIES(${RUNNER} PROPERTIES GENERATED TRUE)
 
 add_executable(xlnt.test ${TEST_HELPERS} ${TESTS} ${RUNNER} )
 
@@ -58,9 +60,12 @@ if(${CMAKE_CXX_COMPILER_ID} STREQUAL "MSVC")
     target_link_libraries(xlnt.test Shlwapi)
 endif()
 
-add_custom_target (generate-test-runner
-    COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/generate-tests
+add_custom_command(OUTPUT ${RUNNER}
+    COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/generate-tests ${CMAKE_CURRENT_BINARY_DIR}
+    DEPENDS ${TESTS}
     COMMENT "Generating test runner ${RUNNER}"
 )
 
+add_custom_target(generate-test-runner DEPENDS ${RUNNER})
+
 add_dependencies(xlnt.test generate-test-runner)
diff --git a/tests/runner-autogen.cpp b/tests/runner-autogen.cpp
deleted file mode 100644
index 68b4678e..00000000
--- a/tests/runner-autogen.cpp
+++ /dev/null
@@ -1 +0,0 @@
-#error This file should be generated from tests before compiling