mcs/build/tests.make

   1 # -*- makefile -*-
   2 #
   3 # Rules for building unit tests.
   4 #
   5 # Includers of this file must define the following values:
   6 #
   7 #   ASSEMBLY
   8 #   ASSEMBLY_EXT
   9 #   the_assembly
  10
  11 tests_CLEAN_FILES :=
  12
  13 ifndef TEST_COMPILE
  14 TEST_COMPILE = $(subst $(test_remove),,$(CSCOMPILE))
  15 endif
  16
  17 TEST_RUNTIME_WRAPPERS_PATH = $(shell dirname $(RUNTIME))/_tmpinst/bin
  18
  19 ## Unit test support
  20 ifndef NO_TEST
  21
  22 test_nunit_lib = nunitlite.dll
  23 xunit_core := xunit.core xunit.abstractions xunit.assert Xunit.NetCore.Extensions
  24 xunit_deps := System.Runtime
  25 xunit_src  := $(patsubst %,$(topdir)/../external/xunit-binaries/%,BenchmarkAttribute.cs BenchmarkDiscover.cs)
  26 xunit_class_deps :=
  27
  28 xunit_libs_ref = $(patsubst %,-r:$(topdir)/../external/xunit-binaries/%.dll,$(xunit_core))
  29 xunit_libs_ref += $(patsubst %,-r:$(topdir)/class/lib/$(PROFILE)/Facades/%.dll,$(xunit_deps))
  30
  31 xunit_libs_dep = $(xunit_class_deps:%=$(topdir)/class/lib/$(PROFILE)/$(PARENT_PROFILE)%.dll)
  32 xunit_libs_ref += $(xunit_libs_dep:%=-r:%)
  33
  34 TEST_LIB_MCS_FLAGS = $(patsubst %,-r:$(topdir)/class/lib/$(PROFILE)/%.dll,$(TEST_LIB_REFS))
  35 XTEST_LIB_MCS_FLAGS = $(patsubst %,-r:$(topdir)/class/lib/$(PROFILE)/%.dll,$(XTEST_LIB_REFS))
  36
  37 test_nunit_dep = $(test_nunit_lib:%=$(topdir)/class/lib/$(PROFILE)/$(PARENT_PROFILE)%)
  38 test_nunit_ref = $(test_nunit_dep:%=-r:%)
  39 tests_CLEAN_FILES += TestResult*.xml
  40
  41 test_sourcefile = $(PROFILE)_$(ASSEMBLY:$(ASSEMBLY_EXT)=_test.dll.sources)
  42
  43 ifeq ($(wildcard $(test_sourcefile)),)
  44 test_sourcefile = $(ASSEMBLY:$(ASSEMBLY_EXT)=_test.dll.sources)
  45 endif
  46
  47 test_lib = $(PROFILE)_$(ASSEMBLY:$(ASSEMBLY_EXT)=_test.dll)
  48
  49 test_sourcefile_excludes = $(test_lib).exclude.sources
  50
  51 test_pdb = $(test_lib:.dll=.pdb)
  52 test_response = $(depsdir)/$(test_lib).response
  53 test_makefrag = $(depsdir)/$(test_lib).makefrag
  54 test_flags = -r:$(the_assembly) $(test_nunit_ref) $(TEST_MCS_FLAGS) $(TEST_LIB_MCS_FLAGS)
  55 tests_CLEAN_FILES += $(ASSEMBLY:$(ASSEMBLY_EXT)=_test*.dll) $(ASSEMBLY:$(ASSEMBLY_EXT)=_test*.pdb) $(test_response) $(test_makefrag)
  56
  57 xtest_sourcefile = $(PROFILE)_$(ASSEMBLY:$(ASSEMBLY_EXT)=_xtest.dll.sources)
  58
  59
  60 xunit_test_lib = $(PROFILE)_$(ASSEMBLY:$(ASSEMBLY_EXT)=_xunit-test.dll)
  61
  62 xtest_response = $(depsdir)/$(xunit_test_lib).response
  63 xtest_makefrag = $(depsdir)/$(xunit_test_lib).makefrag
  64 xtest_flags = -r:$(the_assembly) $(xunit_libs_ref) $(XTEST_MCS_FLAGS) $(XTEST_LIB_MCS_FLAGS)
  65
  66 ifeq ($(wildcard $(xtest_sourcefile)),)
  67 xtest_sourcefile = $(ASSEMBLY:$(ASSEMBLY_EXT)=_xtest.dll.sources)
  68 tests_CLEAN_FILES += $(xunit_test_lib) $(xtest_response) $(xtest_makefrag)
  69 endif
  70
  71 ifndef HAVE_CS_TESTS
  72 HAVE_CS_TESTS := $(wildcard $(test_sourcefile))
  73 endif
  74
  75 HAVE_SOURCE_EXCLUDES := $(wildcard $(test_sourcefile_excludes))
  76
  77 HAVE_CS_XTESTS := $(wildcard $(xtest_sourcefile))
  78
  79 endif # !NO_TEST
  80
  81 ifndef NO_TEST
  82 $(test_nunit_dep): $(topdir)/build/deps/nunit-$(PROFILE).stamp
  83         @if test -f $@; then :; else rm -f $<; $(MAKE) $<; fi
  84
  85 $(topdir)/build/deps/nunit-$(PROFILE).stamp:
  86 ifndef PARENT_PROFILE
  87         cd ${topdir}/tools/nunit-lite && $(MAKE)
  88 endif
  89         echo "stamp" >$@
  90
  91 tests_CLEAN_FILES += $(topdir)/build/deps/nunit-$(PROFILE).stamp
  92
  93 endif
  94
  95 test_assemblies :=
  96
  97 test_lib_dir = $(topdir)/class/lib/$(PROFILE)/tests
  98
  99 test_lib_output = $(topdir)/class/lib/$(PROFILE)/tests/$(test_lib)
 100
 101 ifdef HAVE_CS_TESTS
 102 test_assemblies += $(test_lib_output)
 103 endif
 104
 105 ifdef test_assemblies
 106 check: run-test
 107 test-local: $(test_assemblies)
 108 run-test-local: run-test-lib
 109 run-test-ondotnet-local: run-test-ondotnet-lib
 110
 111 TEST_HARNESS_EXCLUDES = -exclude=$(PLATFORM_TEST_HARNESS_EXCLUDES)$(PROFILE_TEST_HARNESS_EXCLUDES)NotWorking,CAS
 112 TEST_HARNESS_EXCLUDES_ONDOTNET = /exclude:$(PLATFORM_TEST_HARNESS_EXCLUDES)$(PROFILE_TEST_HARNESS_EXCLUDES)NotDotNet,CAS
 113
 114 NOSHADOW_FLAG =
 115
 116 ifdef FIXTURE
 117 FIXTURE_ARG = -test=MonoTests.$(FIXTURE)
 118 endif
 119
 120 ifdef TESTNAME
 121 TESTNAME_ARG = -test=MonoTests.$(TESTNAME)
 122 endif
 123
 124 ifdef TEST_HARNESS_VERBOSE
 125 LABELS_ARG = -labels
 126 endif
 127
 128 ifdef ALWAYS_AOT
 129 test-local-aot-compile: $(topdir)/build/deps/nunit-$(PROFILE).stamp
 130         PATH="$(TEST_RUNTIME_WRAPPERS_PATH):$(PATH)" MONO_REGISTRY_PATH="$(HOME)/.mono/registry" MONO_TESTS_IN_PROGRESS="yes" $(TEST_RUNTIME) $(RUNTIME_FLAGS) $(AOT_BUILD_FLAGS) $(test_assemblies)
 131
 132 else
 133 test-local-aot-compile: $(topdir)/build/deps/nunit-$(PROFILE).stamp
 134
 135 endif # ALWAYS_AOT
 136
 137 NUNITLITE_CONFIG_FILE=$(topdir)/class/lib/$(PROFILE)/$(PARENT_PROFILE)nunit-lite-console.exe.config
 138
 139 patch-nunitlite-appconfig:
 140         cp -f $(topdir)/tools/nunit-lite/nunit-lite-console/nunit-lite-console.exe.config.tmpl $(NUNITLITE_CONFIG_FILE)
 141 ifdef TEST_NUNITLITE_APP_CONFIG_GLOBAL
 142         sed -i -e "/__INSERT_CUSTOM_APP_CONFIG_GLOBAL__/r $(TEST_NUNITLITE_APP_CONFIG_GLOBAL)" $(NUNITLITE_CONFIG_FILE)
 143 endif
 144 ifdef TEST_NUNITLITE_APP_CONFIG_RUNTIME
 145         sed -i -e "/__INSERT_CUSTOM_APP_CONFIG_RUNTIME__/r $(TEST_NUNITLITE_APP_CONFIG_RUNTIME)" $(NUNITLITE_CONFIG_FILE)
 146 endif
 147
 148 ifdef PLATFORM_AOT_SUFFIX
 149
 150 MKBUNDLE_TEST_BIN = $(TEST_HARNESS).static
 151 MKBUNDLE_EXE = $(topdir)/class/lib/$(PROFILE)/mkbundle.exe
 152 BUNDLED_ASSEMBLIES := $(sort $(patsubst .//%,%,$(filter-out %.exe.static %.dll.dll %.exe.dll %bare% %plaincore% %secxml% %Facades% %ilasm%,$(filter %.dll %.exe,$(wildcard $(topdir)/class/lib/$(PROFILE)/*)))))
 153
 154 $(MKBUNDLE_EXE): $(topdir)/tools/mkbundle/mkbundle.cs
 155         make -C $(topdir)/tools/mkbundle
 156
 157 mkbundle-all-tests: $(MKBUNDLE_EXE)
 158         $(Q_AOT) $(MAKE) -C $(topdir)/class do-test
 159         $(Q_AOT) $(MAKE) $(MKBUNDLE_TEST_BIN) # recursive make re-computes variables for BUNDLED_ASSEMBLIES
 160
 161 $(MKBUNDLE_TEST_BIN): $(BUNDLED_ASSEMBLIES) $(TEST_HARNESS)
 162         $(Q_AOT) MONO_PATH="$(dir $<)" PKG_CONFIG_PATH="$(topdir)/../data" $(RUNTIME) $(RUNTIME_FLAGS) $(MKBUNDLE_EXE) -L $(topdir)/class/lib/$(PROFILE) -v --deps $(TEST_HARNESS) $(BUNDLED_ASSEMBLIES) -o $(MKBUNDLE_TEST_BIN) --aot-mode $(AOT_MODE) --aot-runtime $(RUNTIME) --aot-args $(AOT_BUILD_ATTRS) --in-tree $(topdir)/..
 163
 164 endif # PLATFORM_AOT_SUFFIX
 165
 166 ifneq ($(wildcard $(MKBUNDLE_TEST_BIN)),)
 167 TEST_HARNESS_EXEC = $(MKBUNDLE_TEST_BIN)
 168 else
 169 TEST_HARNESS_EXEC = $(TEST_RUNTIME) $(RUNTIME_FLAGS) $(AOT_RUN_FLAGS) $(TEST_HARNESS)
 170 endif
 171
 172 ## FIXME: i18n problem in the 'sed' command below
 173 run-test-lib: test-local test-local-aot-compile patch-nunitlite-appconfig $(MKBUNDLE_TEST_BIN)
 174         ok=:; \
 175         PATH="$(TEST_RUNTIME_WRAPPERS_PATH):$(PATH)" MONO_REGISTRY_PATH="$(HOME)/.mono/registry" MONO_TESTS_IN_PROGRESS="yes" $(TEST_HARNESS_EXEC) $(test_assemblies) $(NOSHADOW_FLAG) $(TEST_HARNESS_FLAGS) $(LOCAL_TEST_HARNESS_FLAGS) $(TEST_HARNESS_EXCLUDES) $(LABELS_ARG) -format:nunit2 -result:TestResult-$(PROFILE).xml $(FIXTURE_ARG) $(TESTNAME_ARG)|| ok=false; \
 176         if [ ! -f "TestResult-$(PROFILE).xml" ]; then echo "<?xml version='1.0' encoding='utf-8'?><test-results failures='1' total='1' not-run='0' name='bcl-tests' date='$$(date +%F)' time='$$(date +%T)'><test-suite name='$(strip $(test_assemblies))' success='False' time='0'><results><test-case name='crash' executed='True' success='False' time='0'><failure><message>The test runner didn't produce a test result XML, probably due to a crash of the runtime. Check the log for more details.</message><stack-trace></stack-trace></failure></test-case></results></test-suite></test-results>" > TestResult-$(PROFILE).xml; fi; \
 177         $$ok
 178
 179 ## Instructs compiler to compile to target .net execution, it can be usefull in rare cases when runtime detection is not possible
 180 run-test-ondotnet-lib: LOCAL_TEST_COMPILER_ONDOTNET_FLAGS:=-d:RUN_ONDOTNET
 181 run-test-ondotnet-lib: test-local
 182         ok=:; \
 183         $(TEST_HARNESS) $(test_assemblies) $(NOSHADOW_FLAG) $(TEST_HARNESS_FLAGS) $(LOCAL_TEST_HARNESS_ONDOTNET_FLAGS) $(TEST_HARNESS_EXCLUDES_ONDOTNET) $(LABELS_ARG) -format:nunit2 -result:TestResult-ondotnet-$(PROFILE).xml $(FIXTURE_ARG) $(TESTNAME_ARG) || ok=false; \
 184         $$ok
 185
 186
 187 endif # test_assemblies
 188
 189 TEST_FILES =
 190
 191 ifdef HAVE_CS_TESTS
 192 TEST_FILES += `sed -e '/^$$/d' -e 's,^../,,' -et -e 's,^,Test/,' $(test_sourcefile)`
 193 endif
 194
 195 ifdef HAVE_CS_TESTS
 196
 197 $(test_lib_dir):
 198         mkdir -p $@
 199
 200 $(test_lib_output): $(the_assembly) $(test_response) $(test_nunit_dep) $(test_lib_dir)
 201         $(TEST_COMPILE) $(LIBRARY_FLAGS) -target:library -out:$@ $(test_flags) $(LOCAL_TEST_COMPILER_ONDOTNET_FLAGS) @$(test_response)
 202
 203 test_response_preprocessed = $(test_response)_preprocessed
 204
 205 # This handles .excludes/.sources pairs, as well as resolving the
 206 # includes that occur in .sources files
 207 $(test_response_preprocessed): $(test_sourcefile)
 208         $(SHELL) $(topdir)/build/gensources.sh $@ '$(test_sourcefile)' '$(test_sourcefile_excludes)'
 209
 210 $(test_response): $(test_response_preprocessed)
 211 #       @echo Creating $@ ...
 212         @sed -e '/^$$/d' -e 's,^,Test/,' $(test_response_preprocessed) | $(PLATFORM_CHANGE_SEPARATOR_CMD) >$@
 213
 214 $(test_makefrag): $(test_response)
 215 #       @echo Creating $@ ...
 216         @sed 's,^,$(test_lib): ,' $< >$@
 217
 218 -include $(test_makefrag)
 219
 220 build-test-lib: $(test_lib_output)
 221         @echo Building testing lib
 222
 223 endif
 224
 225
 226 ifdef HAVE_CS_XTESTS
 227
 228 XTEST_HARNESS_PATH = $(topdir)/../external/xunit-binaries
 229 XTEST_HARNESS = $(XTEST_HARNESS_PATH)/xunit.console.exe
 230 XTEST_HARNESS_FLAGS := -noappdomain -noshadow -parallel none -nunit TestResult-$(PROFILE)-xunit.xml
 231 XTEST_TRAIT := -notrait category=failing -notrait category=nonnetcoreapptests -notrait Benchmark=true -notrait category=outerloop
 232
 233 ifdef FIXTURE
 234 XTEST_HARNESS_FLAGS += -class $(FIXTURE)
 235 endif
 236
 237 ifdef TESTNAME
 238 XTEST_HARNESS_FLAGS += -method $(TESTNAME)
 239 endif
 240
 241 check: run-xunit-test-local
 242 run-xunit-test: run-xunit-test-local
 243 xunit-test-local: $(xunit_test_lib)
 244 run-xunit-test-local: run-xunit-test-lib
 245
 246 # ln -s is a HACK for xunit runner to require xunit.execution.desktop.dll file in local folder on .net only
 247 run-xunit-test-lib: xunit-test-local
 248         @ln -fs $(XTEST_HARNESS_PATH)/xunit.execution.desktop.dll xunit.execution.desktop.dll
 249         ok=:; \
 250         PATH="$(TEST_RUNTIME_WRAPPERS_PATH):$(PATH)" $(TEST_RUNTIME) $(RUNTIME_FLAGS) $(AOT_RUN_FLAGS) $(XTEST_HARNESS) $(xunit_test_lib) $(XTEST_HARNESS_FLAGS) $(XTEST_TRAIT) || ok=false; \
 251         $$ok
 252         @rm -f xunit.execution.desktop.dll
 253
 254 $(xunit_test_lib): $(the_assembly) $(xtest_response) $(xunit_libs_dep) $(xunit_src)
 255         $(TEST_COMPILE) $(LIBRARY_FLAGS) $(XTEST_LIB_FLAGS) -target:library -out:$@ $(xtest_flags) @$(xtest_response) $(xunit_src)
 256
 257 xtest_response_preprocessed = $(xtest_response)_preprocessed
 258
 259 # This handles .excludes/.sources pairs, as well as resolving the
 260 # includes that occur in .sources files
 261 $(xtest_response): $(xtest_sourcefile)
 262         $(SHELL) $(topdir)/build/gensources.sh $@ '$(xtest_sourcefile)' '$(xtest_sourcefile_excludes)'
 263
 264 $(xtest_makefrag): $(xtest_response)
 265         @echo Creating $@ ...
 266         @sed 's,^,$(xunit_test_lib): ,' $< >$@
 267
 268 -include $(xtest_makefrag)
 269
 270 endif
 271
 272
 273 .PHONY: patch-nunitlite-appconfig