projects / mono.git / blobdiff
? search:
summary | shortlog | log | commit | commitdiff | tree
raw | inline | side by side
Bump corefx
[mono.git] / mcs / build / tests.make
diff --git a/mcs/build/tests.make b/mcs/build/tests.make
index 749b5a0faf4edbd8d29eb176582e397ec64db32c..054de2582a1750be5601bc6fb4bc377464e07e5b 100644 (file)
--- a/mcs/build/tests.make
+++ b/mcs/build/tests.make
@@ -19,13 +19,20 @@ TEST_RUNTIME_WRAPPERS_PATH = $(shell dirname $(RUNTIME))/_tmpinst/bin
 ## Unit test support
 ifndef NO_TEST
 
-ifdef NUNIT_LITE
 test_nunit_lib = nunitlite.dll
-else
-test_nunit_lib = nunit.framework.dll nunit.core.dll nunit.util.dll nunit.mocks.dll
-endif
+xunit_core := xunit.core xunit.abstractions xunit.assert Xunit.NetCore.Extensions
+xunit_deps := System.Runtime
+xunit_src  := $(patsubst %,$(topdir)/../external/xunit-binaries/%,BenchmarkAttribute.cs BenchmarkDiscover.cs) $(topdir)/../mcs/class/test-helpers/PlatformDetection.cs
+xunit_class_deps := 
+
+xunit_libs_ref = $(patsubst %,-r:$(topdir)/../external/xunit-binaries/%.dll,$(xunit_core))
+xunit_libs_ref += $(patsubst %,-r:$(topdir)/class/lib/$(PROFILE)/Facades/%.dll,$(xunit_deps))
+
+xunit_libs_dep = $(xunit_class_deps:%=$(topdir)/class/lib/$(PROFILE)/$(PARENT_PROFILE)%.dll)
+xunit_libs_ref += $(xunit_libs_dep:%=-r:%)
 
 TEST_LIB_MCS_FLAGS = $(patsubst %,-r:$(topdir)/class/lib/$(PROFILE)/%.dll,$(TEST_LIB_REFS))
+XTEST_LIB_MCS_FLAGS = $(patsubst %,-r:$(topdir)/class/lib/$(PROFILE)/%.dll,$(XTEST_LIB_REFS))
 
 test_nunit_dep = $(test_nunit_lib:%=$(topdir)/class/lib/$(PROFILE)/$(PARENT_PROFILE)%)
 test_nunit_ref = $(test_nunit_dep:%=-r:%)
@@ -38,6 +45,7 @@ test_sourcefile = $(ASSEMBLY:$(ASSEMBLY_EXT)=_test.dll.sources)
 endif
 
 test_lib = $(PROFILE)_$(ASSEMBLY:$(ASSEMBLY_EXT)=_test.dll)
+
 test_sourcefile_excludes = $(test_lib).exclude.sources
 
 test_pdb = $(test_lib:.dll=.pdb)
@@ -46,33 +54,42 @@ test_makefrag = $(depsdir)/$(test_lib).makefrag
 test_flags = -r:$(the_assembly) $(test_nunit_ref) $(TEST_MCS_FLAGS) $(TEST_LIB_MCS_FLAGS)
 tests_CLEAN_FILES += $(ASSEMBLY:$(ASSEMBLY_EXT)=_test*.dll) $(ASSEMBLY:$(ASSEMBLY_EXT)=_test*.pdb) $(test_response) $(test_makefrag)
 
+xtest_sourcefile = $(PROFILE)_$(ASSEMBLY:$(ASSEMBLY_EXT)=_xtest.dll.sources)
+
+
+xunit_test_lib = $(PROFILE)_$(ASSEMBLY:$(ASSEMBLY_EXT)=_xunit-test.dll)
+
+xtest_response = $(depsdir)/$(xunit_test_lib).response
+xtest_makefrag = $(depsdir)/$(xunit_test_lib).makefrag
+xtest_flags = -r:$(the_assembly) $(xunit_libs_ref) $(XTEST_MCS_FLAGS) $(XTEST_LIB_MCS_FLAGS)
+
+ifeq ($(wildcard $(xtest_sourcefile)),)
+xtest_sourcefile = $(ASSEMBLY:$(ASSEMBLY_EXT)=_xtest.dll.sources)
+tests_CLEAN_FILES += $(xunit_test_lib) $(xtest_response) $(xtest_makefrag)
+endif
+
 ifndef HAVE_CS_TESTS
 HAVE_CS_TESTS := $(wildcard $(test_sourcefile))
 endif
 
 HAVE_SOURCE_EXCLUDES := $(wildcard $(test_sourcefile_excludes))
 
+HAVE_CS_XTESTS := $(wildcard $(xtest_sourcefile))
+
 endif # !NO_TEST
 
 ifndef NO_TEST
 $(test_nunit_dep): $(topdir)/build/deps/nunit-$(PROFILE).stamp
        @if test -f $@; then :; else rm -f $<; $(MAKE) $<; fi
 
-ifdef NUNIT_LITE
 $(topdir)/build/deps/nunit-$(PROFILE).stamp:
 ifndef PARENT_PROFILE
        cd ${topdir}/tools/nunit-lite && $(MAKE)
 endif
        echo "stamp" >$@
-else
-$(topdir)/build/deps/nunit-$(PROFILE).stamp:
-ifndef PARENT_PROFILE
-       cd ${topdir}/nunit24 && $(MAKE)
-endif
-       echo "stamp" >$@
-endif
 
 tests_CLEAN_FILES += $(topdir)/build/deps/nunit-$(PROFILE).stamp
+
 endif
 
 test_assemblies :=
@@ -87,55 +104,21 @@ test-local: $(test_assemblies)
 run-test-local: run-test-lib
 run-test-ondotnet-local: run-test-ondotnet-lib
 
-TEST_HARNESS_EXCLUDES = -exclude=$(PLATFORM_TEST_HARNESS_EXCLUDES)$(PROFILE_TEST_HARNESS_EXCLUDES)NotWorking,ValueAdd,CAS,InetAccess
+TEST_HARNESS_EXCLUDES = -exclude=$(PLATFORM_TEST_HARNESS_EXCLUDES)$(PROFILE_TEST_HARNESS_EXCLUDES)NotWorking,CAS
 TEST_HARNESS_EXCLUDES_ONDOTNET = /exclude:$(PLATFORM_TEST_HARNESS_EXCLUDES)$(PROFILE_TEST_HARNESS_EXCLUDES)NotDotNet,CAS
 
-ifdef NUNIT_LITE
 NOSHADOW_FLAG =
-NUNIT_XML_FLAG = -format:nunit2 -result:
-OUTPUT_FILE_FLAG=-out
-else
-OUTPUT_FILE_FLAG=-output
-NOSHADOW_FLAG = -noshadow
-NUNIT_XML_FLAG = -xml=
-endif
-
-ifdef NUNIT_LITE
-NOSHADOW_FLAG =
-NUNIT_XML_FLAG = -format:nunit2 -result:
-OUTPUT_FILE_FLAG=-out
-else
-OUTPUT_FILE_FLAG=-output
-NOSHADOW_FLAG = -noshadow
-NUNIT_XML_FLAG = -xml=
-endif
-
-ifdef TEST_HARNESS_VERBOSE
-TEST_HARNESS_OUTPUT = -labels
-TEST_HARNESS_OUTPUT_ONDOTNET = -labels
-TEST_HARNESS_POSTPROC = :
-TEST_HARNESS_POSTPROC_ONDOTNET = :
-else
-TEST_HARNESS_OUTPUT = $(OUTPUT_FILE_FLAG)=TestResult-$(PROFILE).log
-TEST_HARNESS_OUTPUT_ONDOTNET = $(OUTPUT_FILE_FLAG)=TestResult-ondotnet-$(PROFILE).log
-TEST_HARNESS_POSTPROC = (echo ''; cat TestResult-$(PROFILE).log) | sed '1,/^Tests run: /d'; xsltproc $(topdir)/build/nunit-summary.xsl TestResult-$(PROFILE).xml >> TestResult-$(PROFILE).log
-TEST_HARNESS_POSTPROC_ONDOTNET = (echo ''; cat TestResult-ondotnet-$(PROFILE).log) | sed '1,/^Tests run: /d'; xsltproc $(topdir)/build/nunit-summary.xsl TestResult-ondotnet-$(PROFILE).xml >> TestResult-ondotnet-$(PROFILE).log
-endif
 
 ifdef FIXTURE
-ifdef NUNIT_LITE
 FIXTURE_ARG = -test=MonoTests.$(FIXTURE)
-else
-FIXTURE_ARG = -fixture=MonoTests.$(FIXTURE)
-endif
 endif
 
 ifdef TESTNAME
-ifdef NUNIT_LITE
 TESTNAME_ARG = -test=MonoTests.$(TESTNAME)
-else
-TESTNAME_ARG = -run=MonoTests.$(TESTNAME)
 endif
+
+ifdef TEST_HARNESS_VERBOSE
+LABELS_ARG = -labels
 endif
 
 ifdef ALWAYS_AOT
@@ -147,19 +130,30 @@ test-local-aot-compile: $(topdir)/build/deps/nunit-$(PROFILE).stamp
 
 endif # ALWAYS_AOT
 
+NUNITLITE_CONFIG_FILE=$(topdir)/class/lib/$(PROFILE)/$(PARENT_PROFILE)nunit-lite-console.exe.config
+
+patch-nunitlite-appconfig:
+       cp -f $(topdir)/tools/nunit-lite/nunit-lite-console/nunit-lite-console.exe.config.tmpl $(NUNITLITE_CONFIG_FILE)
+ifdef TEST_NUNITLITE_APP_CONFIG_GLOBAL
+       sed -i -e "/__INSERT_CUSTOM_APP_CONFIG_GLOBAL__/r $(TEST_NUNITLITE_APP_CONFIG_GLOBAL)" $(NUNITLITE_CONFIG_FILE)
+endif
+ifdef TEST_NUNITLITE_APP_CONFIG_RUNTIME
+       sed -i -e "/__INSERT_CUSTOM_APP_CONFIG_RUNTIME__/r $(TEST_NUNITLITE_APP_CONFIG_RUNTIME)" $(NUNITLITE_CONFIG_FILE)
+endif
+
 ## FIXME: i18n problem in the 'sed' command below
-run-test-lib: test-local test-local-aot-compile
+run-test-lib: test-local test-local-aot-compile patch-nunitlite-appconfig
        ok=:; \
-       PATH="$(TEST_RUNTIME_WRAPPERS_PATH):$(PATH)" MONO_REGISTRY_PATH="$(HOME)/.mono/registry" MONO_TESTS_IN_PROGRESS="yes" $(TEST_RUNTIME) $(RUNTIME_FLAGS) $(AOT_RUN_FLAGS) $(TEST_HARNESS) $(test_assemblies) $(NOSHADOW_FLAG) $(TEST_HARNESS_FLAGS) $(LOCAL_TEST_HARNESS_FLAGS) $(TEST_HARNESS_EXCLUDES) $(TEST_HARNESS_OUTPUT) $(NUNIT_XML_FLAG)TestResult-$(PROFILE).xml $(FIXTURE_ARG) $(TESTNAME_ARG)|| ok=false; \
+       PATH="$(TEST_RUNTIME_WRAPPERS_PATH):$(PATH)" MONO_REGISTRY_PATH="$(HOME)/.mono/registry" MONO_TESTS_IN_PROGRESS="yes" $(TEST_RUNTIME) $(RUNTIME_FLAGS) $(AOT_RUN_FLAGS) $(TEST_HARNESS) $(test_assemblies) $(NOSHADOW_FLAG) $(TEST_HARNESS_FLAGS) $(LOCAL_TEST_HARNESS_FLAGS) $(TEST_HARNESS_EXCLUDES) $(LABELS_ARG) -format:nunit2 -result:TestResult-$(PROFILE).xml $(FIXTURE_ARG) $(TESTNAME_ARG)|| ok=false; \
        if [ ! -f "TestResult-$(PROFILE).xml" ]; then echo "<?xml version='1.0' encoding='utf-8'?><test-results failures='1' total='1' not-run='0' name='bcl-tests' date='$$(date +%F)' time='$$(date +%T)'><test-suite name='$(strip $(test_assemblies))' success='False' time='0'><results><test-case name='crash' executed='True' success='False' time='0'><failure><message>The test runner didn't produce a test result XML, probably due to a crash of the runtime. Check the log for more details.</message><stack-trace></stack-trace></failure></test-case></results></test-suite></test-results>" > TestResult-$(PROFILE).xml; fi; \
-       $(TEST_HARNESS_POSTPROC) ; $$ok
+       $$ok
 
 ## Instructs compiler to compile to target .net execution, it can be usefull in rare cases when runtime detection is not possible
 run-test-ondotnet-lib: LOCAL_TEST_COMPILER_ONDOTNET_FLAGS:=-d:RUN_ONDOTNET
 run-test-ondotnet-lib: test-local
        ok=:; \
-       $(TEST_HARNESS) $(test_assemblies) $(NOSHADOW_FLAG) $(TEST_HARNESS_FLAGS) $(LOCAL_TEST_HARNESS_ONDOTNET_FLAGS) $(TEST_HARNESS_EXCLUDES_ONDOTNET) $(TEST_HARNESS_OUTPUT_ONDOTNET) $(NUNIT_XML_FLAG)=TestResult-ondotnet-$(PROFILE).xml $(FIXTURE_ARG) $(TESTNAME_ARG) || ok=false; \
-       $(TEST_HARNESS_POSTPROC_ONDOTNET) ; $$ok
+       $(TEST_HARNESS) $(test_assemblies) $(NOSHADOW_FLAG) $(TEST_HARNESS_FLAGS) $(LOCAL_TEST_HARNESS_ONDOTNET_FLAGS) $(TEST_HARNESS_EXCLUDES_ONDOTNET) $(LABELS_ARG) -format:nunit2 -result:TestResult-ondotnet-$(PROFILE).xml $(FIXTURE_ARG) $(TESTNAME_ARG) || ok=false; \
+       $$ok
 
 
 endif # test_assemblies
@@ -192,5 +186,57 @@ $(test_makefrag): $(test_response)
 
 -include $(test_makefrag)
 
+build-test-lib: $(test_lib)
+       @echo Building testing lib
+
+endif
+
+
+ifdef HAVE_CS_XTESTS
+
+XTEST_HARNESS_PATH = $(topdir)/../external/xunit-binaries
+XTEST_HARNESS = $(XTEST_HARNESS_PATH)/xunit.console.exe
+XTEST_HARNESS_FLAGS := -noappdomain -noshadow -parallel none -nunit TestResult-$(PROFILE)-xunit.xml
+XTEST_TRAIT := -notrait category=failing -notrait category=nonnetcoreapptests -notrait Benchmark=true -notrait category=outerloop
+
+ifdef FIXTURE
+XTEST_HARNESS_FLAGS += -class $(FIXTURE)
+endif
+
+ifdef TESTNAME
+XTEST_HARNESS_FLAGS += -method $(TESTNAME)
 endif
 
+check: run-xunit-test-local
+run-xunit-test: run-xunit-test-local
+xunit-test-local: $(xunit_test_lib)
+run-xunit-test-local: run-xunit-test-lib
+
+# cp -rf is a HACK for xunit runner to require xunit.execution.desktop.dll file in local folder on .net only
+run-xunit-test-lib: xunit-test-local
+       @cp -rf $(XTEST_HARNESS_PATH)/xunit.execution.desktop.dll xunit.execution.desktop.dll
+       ok=:; \
+       PATH="$(TEST_RUNTIME_WRAPPERS_PATH):$(PATH)" $(TEST_RUNTIME) $(RUNTIME_FLAGS) $(AOT_RUN_FLAGS) $(XTEST_HARNESS) $(xunit_test_lib) $(XTEST_HARNESS_FLAGS) $(XTEST_TRAIT) || ok=false; \
+       $$ok
+       @rm -f xunit.execution.desktop.dll
+
+$(xunit_test_lib): $(the_assembly) $(xtest_response) $(xunit_libs_dep) $(xunit_src)
+       $(TEST_COMPILE) $(LIBRARY_FLAGS) $(XTEST_LIB_FLAGS) -target:library -out:$@ $(xtest_flags) @$(xtest_response) $(xunit_src)
+
+xtest_response_preprocessed = $(xtest_response)_preprocessed
+
+# This handles .excludes/.sources pairs, as well as resolving the
+# includes that occur in .sources files
+$(xtest_response): $(xtest_sourcefile)
+       $(SHELL) $(topdir)/build/gensources.sh $@ '$(xtest_sourcefile)' '$(xtest_sourcefile_excludes)'
+
+$(xtest_makefrag): $(xtest_response)
+       @echo Creating $@ ...
+       @sed 's,^,$(xunit_test_lib): ,' $< >$@
+
+-include $(xtest_makefrag)
+
+endif
+
+
+.PHONY: patch-nunitlite-appconfig
my mono mirror. check out Moved to http://github.com/mono/mono