summaryrefslogtreecommitdiff
path: root/test/Makefile.in
blob: 0bb35581d182db7b4305f6c460f2230991c698d7 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
### @configure_input@

# Copyright (C) 2010-2021 Free Software Foundation, Inc.

# This file is part of GNU Emacs.

# GNU Emacs is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# GNU Emacs is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.

### Commentary:

## Some targets:
## check: re-run all tests, writing to .log files.
## check-maybe: run all tests which are outdated with their .log file
## or the source files they are testing.
## filename.log: run tests from filename.el(c) if .log file needs updating
## filename: re-run tests from filename.el(c), with no logging

### Code:

SHELL = @SHELL@

srcdir = @srcdir@
abs_top_srcdir = @abs_top_srcdir@
top_builddir = @top_builddir@
VPATH = $(srcdir)

FIND_DELETE = @FIND_DELETE@
MKDIR_P = @MKDIR_P@
CC = @CC@
CFLAGS = @CFLAGS@
PROFILING_CFLAGS = @PROFILING_CFLAGS@
WARN_CFLAGS = @WARN_CFLAGS@
WERROR_CFLAGS = @WERROR_CFLAGS@
CPPFLAGS = @CPPFLAGS@
SO = @MODULES_SUFFIX@

SEPCHAR = @SEPCHAR@

HAVE_NATIVE_COMP = @HAVE_NATIVE_COMP@

REPLACE_FREE = @REPLACE_FREE@

-include ${top_builddir}/src/verbose.mk

# Load any GNU ELPA dependencies that are present, for optional tests.
GNU_ELPA_DIRECTORY ?= $(srcdir)/../../elpa
# Keep elpa_dependencies dependency-ordered.
elpa_dependencies = \
	url-http-ntlm/url-http-ntlm.el \
	web-server/web-server.el
elpa_els = $(addprefix $(GNU_ELPA_DIRECTORY)/packages/,$(elpa_dependencies))
elpa_opts = $(foreach el,$(elpa_els),$(and $(wildcard $(el)),-L $(dir $(el)) -l $(el)))

# We never change directory before running Emacs, so a relative file
# name is fine, and makes life easier.  If we need to change
# directory, we can use emacs --chdir.
EMACS = ../src/emacs

EMACS_EXTRAOPT =

# Command line flags for Emacs.
# Apparently MSYS bash would convert "-L :" to "-L ;" anyway,
# but we might as well be explicit.
EMACSOPT = --no-init-file --no-site-file --no-site-lisp -L "$(SEPCHAR)$(srcdir)" $(elpa_opts) $(EMACS_EXTRAOPT)

# Prevent any settings in the user environment causing problems.
unexport EMACSDATA EMACSDOC EMACSPATH GREP_OPTIONS XDG_CONFIG_HOME

# To run tests under a debugger, set this to eg: "gdb --args".
GDB =

# Whether a timeout shall be given, writing possibly a core dump.
ifneq (${EMACS_TEST_TIMEOUT},)
TEST_TIMEOUT = timeout -s ABRT ${EMACS_TEST_TIMEOUT}
endif

# Set this to 'yes' to run the tests in an interactive instance.
TEST_INTERACTIVE ?= no

ifeq ($(TEST_INTERACTIVE),yes)
TEST_RUN_ERT = --eval '(ert (quote ${SELECTOR_ACTUAL}))'
else
TEST_RUN_ERT = --batch --eval '(ert-run-tests-batch-and-exit (quote ${SELECTOR_ACTUAL}))' ${WRITE_LOG}
endif

# Whether to run tests from .el files in preference to .elc, we do
# this by default since it gives nicer stacktraces.
# If you just want a pass/fail, setting this to no is much faster.
export TEST_LOAD_EL ?= \
    $(if $(findstring $(MAKECMDGOALS), all check check-maybe),no,yes)

# Additional settings for ert.
ert_opts += $(elpa_opts)

# Maximum length of lines in ert backtraces; nil for no limit.
# (if empty, use the default ert-batch-backtrace-right-margin).
TEST_BACKTRACE_LINE_LENGTH =

ifneq (${TEST_BACKTRACE_LINE_LENGTH},)
ert_opts += --eval '(setq ert-batch-backtrace-right-margin ${TEST_BACKTRACE_LINE_LENGTH})'
endif

ifeq (@HAVE_MODULES@, yes)
MODULES_EMACSOPT := --module-assertions
else
MODULES_EMACSOPT :=
endif

# The actual Emacs command run in the targets below.
# Prevent any setting of EMACSLOADPATH in user environment causing problems,
# and prevent locals to influence the text of the errors we expect to receive.
emacs = LANG=C EMACSLOADPATH= 		   \
 EMACS_TEST_DIRECTORY=$(abspath $(srcdir)) \
 $(GDB) $(TEST_TIMEOUT) "$(EMACS)" $(MODULES_EMACSOPT) $(EMACSOPT)

# Set HOME to a nonexistent directory to prevent tests from accessing
# it accidentally (e.g., popping up a gnupg dialog if ~/.authinfo.gpg
# exists, or writing to ~/.bzr.log when running bzr commands).
# NOTE if the '/nonexistent' name is changed `normal-top-level' in
# startup.el must be updated too.
TEST_HOME = /nonexistent

test_module_dir := src/emacs-module-resources

.PHONY: all check

all: check

ifeq ($(HAVE_NATIVE_COMP),yes)
SELECTOR_DEFAULT = (not (or (tag :expensive-test) (tag :unstable)))
SELECTOR_EXPENSIVE = (not (tag :unstable))
SELECTOR_ALL = t
else
SELECTOR_DEFAULT = (not (or (tag :expensive-test) (tag :unstable) (tag :nativecomp)))
SELECTOR_EXPENSIVE = (not (or (tag :unstable) (tag :nativecomp)))
SELECTOR_ALL = (not (tag :nativecomp))
endif
ifdef SELECTOR
SELECTOR_ACTUAL=$(SELECTOR)
else ifndef MAKECMDGOALS
SELECTOR_ACTUAL=$(SELECTOR_DEFAULT)
else ifeq ($(MAKECMDGOALS),all)
SELECTOR_ACTUAL=$(SELECTOR_DEFAULT)
else ifeq ($(MAKECMDGOALS),check)
SELECTOR_ACTUAL=$(SELECTOR_DEFAULT)
else ifeq ($(MAKECMDGOALS),check-maybe)
SELECTOR_ACTUAL=$(SELECTOR_DEFAULT)
else
SELECTOR_ACTUAL=$(SELECTOR_EXPENSIVE)
endif

## Byte-compile all test files to test for errors.
%.elc: %.el
	$(AM_V_ELC)$(emacs) --batch -f batch-byte-compile $<

## Save logs, and show logs for failed tests.
WRITE_LOG = > $@ 2>&1 || { STAT=$$?; cat $@; exit $$STAT; }
## On Hydra or Emba, always show logs for certain problematic tests.
ifdef EMACS_HYDRA_CI
lisp/net/tramp-tests.log \
: WRITE_LOG = 2>&1 | tee $@
endif
ifdef EMACS_EMBA_CI
lisp/filenotify-tests.log lisp/net/tramp-tests.log src/emacs-module-tests.log \
: WRITE_LOG = 2>&1 | tee $@
endif

ifeq ($(TEST_LOAD_EL), yes)
testloadfile = $*.el
else
testloadfile = $*
endif

%.log: %.elc
	$(AM_V_at)${MKDIR_P} $(dir $@)
	$(AM_V_GEN)HOME=$(TEST_HOME) $(emacs) \
	  -l ert ${ert_opts} -l $(testloadfile) \
	  $(TEST_RUN_ERT)

ifeq (@HAVE_MODULES@, yes)
maybe_exclude_module_tests :=
else
maybe_exclude_module_tests := -name emacs-module-tests.el -prune -o
endif

## Optional list of .el files to exclude from testing.
## Intended for use in automated testing where one or more files
## has some problem and needs to be excluded.
## To avoid writing full name, can use eg %foo-tests.el.
EXCLUDE_TESTS =

## To speed up parallel builds, put these slow test files (which can
## take longer than all the rest combined) at the start of the list.
SLOW_TESTS = ${srcdir}/lisp/net/tramp-tests.el

ELFILES := $(sort $(shell find ${srcdir} -name manual -prune -o \
		-name data -prune -o \
		-name "*resources" -prune -o \
		${maybe_exclude_module_tests} \
		-name "*.el" ! -name ".*" -print))

$(foreach slow,${SLOW_TESTS},$(eval ELFILES:= ${slow} $(filter-out ${slow},${ELFILES})))

$(foreach exclude,${EXCLUDE_TESTS},$(eval ELFILES:= $(filter-out ${exclude},${ELFILES})))

## .log files may be in a different directory for out of source builds
LOGFILES := $(patsubst %.el,%.log, \
		$(patsubst $(srcdir)/%,%,$(ELFILES)))
TESTS := $(LOGFILES:.log=)

## If we have to interrupt a hanging test, preserve the log so we can
## see what the problem was.
.PRECIOUS: %.log

## Stop make deleting these as intermediate files.
.SECONDARY: ${ELFILES:.el=.elc} $(test_module_dir)/*.o

.PHONY: ${TESTS}

define test_template
  ## A test FOO-tests depends on the source file with the similar
  ## name, unless FOO itself contains the string '-tests/'.
  ## The similar name is FOO.c if FOO begins with '{lib-,}src/', FOO.el
  ## otherwise.  Although this heuristic does not identify all the
  ## dependencies, it is better than nothing.
  ifeq (,$(patsubst %-tests,,$(1))$(findstring -tests/,$(1)))
    $(1).log: $(patsubst %-tests,$(srcdir)/../%,$(1))$(if \
					$(patsubst src/%,,$(patsubst lib-src/%,,$(1))),.el,.c)
    $(notdir $(1).log): $(1).log
  endif

  ## Short aliases that always re-run the tests, with no logging.
  ## Define both with and without the directory name for ease of use.
  .PHONY: $(1) $(notdir $(1))
  $(1):
	@test ! -f $(1).log || mv $(1).log $(1).log~
	@$(MAKE) $(1).log WRITE_LOG=
  $(notdir $(1)): $(1)
endef

$(foreach test,${TESTS},$(eval $(call test_template,${test})))

## Get the tests for only a specific directory.
SUBDIRS = $(sort $(shell cd ${srcdir} && find lib-src lisp misc src -type d \
                 ! \( -path "*resources*" -o -path "*auto-save-list" \) -print))
SUBDIR_TARGETS =

define subdir_template
  SUBDIR_TARGETS += check-$(subst /,-,$(1))
  .PHONY: check-$(subst /,-,$(1))
  check-$(subst /,-,$(1)):
	@${MAKE} check LOGFILES="$(patsubst %.el,%.log, \
		$(patsubst $(srcdir)/%,%,$(wildcard ${srcdir}/$(1)/*.el)))"
endef

$(foreach subdir, $(SUBDIRS), $(eval $(call subdir_template,$(subdir))))

ifeq (@HAVE_MODULES@, yes)
# -fPIC is a no-op on Windows, but causes a compiler warning
ifeq ($(SO),.dll)
FPIC_CFLAGS =
else
FPIC_CFLAGS = -fPIC
endif

GMP_H = @GMP_H@
LIBGMP = @LIBGMP@

MODULE_CFLAGS = -I../src -I$(srcdir)/../src -I../lib -I$(srcdir)/../lib \
  $(FPIC_CFLAGS) $(PROFILING_CFLAGS) \
  $(WARN_CFLAGS) $(WERROR_CFLAGS) $(CFLAGS)

test_module = $(test_module_dir)/mod-test${SO}
src/emacs-module-tests.log src/emacs-module-tests.elc: $(test_module)

FREE_SOURCE_0 =
FREE_SOURCE_1 = $(srcdir)/../lib/free.c

# In the compilation command, we can't use any object or archive file
# as source because those are not compiled with -fPIC.  Therefore we
# use only source files.
$(test_module): $(test_module:${SO}=.c) ../src/emacs-module.h
	$(AM_V_at)${MKDIR_P} $(dir $@)
	$(AM_V_CCLD)$(CC) -shared $(CPPFLAGS) $(MODULE_CFLAGS) $(LDFLAGS) \
	  -o $@ $< $(LIBGMP) \
	  $(and $(GMP_H),$(srcdir)/../lib/mini-gmp-gnulib.c) \
	  $(FREE_SOURCE_$(REPLACE_FREE)) \
	  $(srcdir)/../lib/timespec.c $(srcdir)/../lib/gettime.c
endif

src/emacs-tests.log: ../lib-src/seccomp-filter.c

## Check that there is no 'automated' subdirectory, which would
## indicate an incomplete merge from an older version of Emacs where
## the tests were arranged differently.
.PHONY: check-no-automated-subdir
check-no-automated-subdir:
	${AM_V_at}test ! -d $(srcdir)/automated

## Rerun all default tests.
check: mostlyclean check-no-automated-subdir
	@${MAKE} check-doit SELECTOR="${SELECTOR_ACTUAL}"

## Rerun all default and expensive tests.
.PHONY: check-expensive
check-expensive: mostlyclean check-no-automated-subdir
	@${MAKE} check-doit SELECTOR="${SELECTOR_EXPENSIVE}"

## Run all tests, regardless of tag.
.PHONY: check-all
check-all: mostlyclean check-no-automated-subdir
	@${MAKE} check-doit SELECTOR="${SELECTOR_ALL}"

## Re-run all tests which are outdated. A test is outdated if its
## logfile is out-of-date with either the test file, or the source
## files that the tests depend on.  See test_template.
.PHONY: check-maybe
check-maybe: check-no-automated-subdir
	@${MAKE} check-doit SELECTOR="${SELECTOR_ACTUAL}"

## Run the tests.
.PHONY: check-doit
## We can't put LOGFILES as prerequisites, because that would stop the
## summarizing step from running when there is an error.
check-doit:
ifeq ($(TEST_INTERACTIVE), yes)
	HOME=$(TEST_HOME) $(emacs) \
	  -l ert ${ert_opts} \
	  $(patsubst %,-l %,$(if $(findstring $(TEST_LOAD_EL),yes),$ELFILES,$(ELFILES:.el=))) \
	  $(TEST_RUN_ERT)
else
	-@${MAKE} -k ${LOGFILES}
	@$(emacs) --batch -l ert --eval \
	"(ert-summarize-tests-batch-and-exit ${SUMMARIZE_TESTS})" ${LOGFILES}
endif

.PHONY: mostlyclean clean bootstrap-clean distclean maintainer-clean

mostlyclean:
	-@for f in ${LOGFILES}; do test ! -f $$f || mv $$f $$f~; done
	rm -f ./*.tmp

clean:
	find . '(' -name '*.log' -o -name '*.log~' ')' $(FIND_DELETE)
	find . '(' -name '*.xml' -a ! -path '*resources*' ')' $(FIND_DELETE)
	rm -f ${srcdir}/lisp/gnus/mml-sec-resources/random_seed
	rm -f $(test_module_dir)/*.o $(test_module_dir)/*.so \
	  $(test_module_dir)/*.dll

bootstrap-clean: clean
	find $(srcdir) -name '*.elc' $(FIND_DELETE)

distclean: clean
	rm -f Makefile

maintainer-clean: distclean bootstrap-clean

.PHONY: check-declare

check-declare:
	$(emacs) --batch -l check-declare \
	  --eval '(check-declare-directory "$(srcdir)")'

.PHONY: subdirs subdir-targets generate-test-jobs

subdirs:
	@: $(info $(SUBDIRS))

subdir-targets:
	@: $(info $(SUBDIR_TARGETS))

generate-test-jobs:
	@$(MAKE) -C infra generate-test-jobs SUBDIRS="$(SUBDIRS)"