test/verifyexclude/VerifyExcludeTest.gmk - master - githubmirror/jtreg

test/verifyexclude/VerifyExcludeTest.gmk
296 строк11 KB
Ludvig Janiuk
7904015: --verify-exclude fails to abort the test run when discovering failures
26 май 2025, 10:20
26 май 2025, 10:20666453d
#
# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License version 2 only, as
# published by the Free Software Foundation.  Oracle designates this
# particular file as subject to the "Classpath" exception as provided
# by Oracle in the LICENSE file that accompanied this code.
#
# This code is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
# version 2 for more details (a copy is included in the LICENSE file that
# accompanied this code).
#
# You should have received a copy of the GNU General Public License version
# 2 along with this work; if not, write to the Free Software Foundation,
# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
# or visit www.oracle.com if you need additional information or have any
# questions.
#

#----------------------------------------------------------------------

$(BUILDTESTDIR)/verifyexclude.format.1.ok: \
                $(JTREG_IMAGEDIR)/lib/jtreg.jar \
		$(JTREG_IMAGEDIR)/lib/javatest.jar
	$(RM) $(@:%.ok=%/work) $(@:%.ok=%/report)
	$(MKDIR) $(@:%.ok=%)
	$(JDKHOME)/bin/java \
		-Djavatest.regtest.showCmd=true \
		-jar $(JTREG_IMAGEJARDIR)/jtreg.jar \
		-w:$(@:%.ok=%/work) \
		-r:$(@:%.ok=%/report) \
		-exclude:$(TESTDIR)/verifyexclude/ProblemList/format.1.txt \
		--verify-exclude \
		$(TESTDIR)/verifyexclude \
		> $(@:%.ok=%/log) \
		&& (cat $(@:%.ok=%/log) ; exit 1) \
		|| (exit 0)
	$(GREP) -s "Must follow:" $(@:%.ok=%/log)
	$(GREP) -s "The fully qualified test must exist." $(@:%.ok=%/log) && exit 1 || exit 0
	$(GREP) -s "Test results" $(@:%.ok=%/log) && \
		(echo $@ failed, did not abort on exclude errors) && exit 1 || (exit 0)
	echo $@ passed at `date` > $@

$(BUILDTESTDIR)/verifyexclude.format.2.ok: \
                $(JTREG_IMAGEDIR)/lib/jtreg.jar \
		$(JTREG_IMAGEDIR)/lib/javatest.jar
	$(RM) $(@:%.ok=%/work) $(@:%.ok=%/report)
	$(MKDIR) $(@:%.ok=%)
	$(JDKHOME)/bin/java \
		-Djavatest.regtest.showCmd=true \
		-jar $(JTREG_IMAGEJARDIR)/jtreg.jar \
		-w:$(@:%.ok=%/work) \
		-r:$(@:%.ok=%/report) \
		-exclude:$(TESTDIR)/verifyexclude/ProblemList/format.2.txt \
		--verify-exclude \
		$(TESTDIR)/verifyexclude \
		> $(@:%.ok=%/log) \
	    && (cat $(@:%.ok=%/log) ; exit 1) \
		|| (exit 0)
	$(GREP) -s "Must follow:" $(@:%.ok=%/log)
	$(GREP) -s "The fully qualified test must exist." $(@:%.ok=%/log) && exit 1 || exit 0
	$(GREP) -s "Test results" $(@:%.ok=%/log) && \
		(echo $@ failed, did not abort on exclude errors) && exit 1 || (exit 0)
	echo $@ passed at `date` > $@

$(BUILDTESTDIR)/verifyexclude.exist.ok: \
                $(JTREG_IMAGEDIR)/lib/jtreg.jar \
		$(JTREG_IMAGEDIR)/lib/javatest.jar
	$(RM) $(@:%.ok=%/work) $(@:%.ok=%/report)
	$(MKDIR) $(@:%.ok=%)
	$(JDKHOME)/bin/java \
		-Djavatest.regtest.showCmd=true \
		-jar $(JTREG_IMAGEJARDIR)/jtreg.jar \
		-w:$(@:%.ok=%/work) \
		-r:$(@:%.ok=%/report) \
		-exclude:$(TESTDIR)/verifyexclude/ProblemList/exist.txt \
		--verify-exclude \
		$(TESTDIR)/verifyexclude \
		> $(@:%.ok=%/log) \
	    && (cat $(@:%.ok=%/log) ; exit 1) \
		|| (exit 0)
	$(GREP) -s "Must follow:" $(@:%.ok=%/log) && exit 1 || exit 0
	$(GREP) -s "The fully qualified test must exist." $(@:%.ok=%/log)
	$(GREP) -s "Test results" $(@:%.ok=%/log) && \
		(echo $@ failed, did not abort on exclude errors) && exit 1 || (exit 0)
	echo $@ passed at `date` > $@

$(BUILDTESTDIR)/verifyexclude.id.ok: \
                $(JTREG_IMAGEDIR)/lib/jtreg.jar \
		$(JTREG_IMAGEDIR)/lib/javatest.jar
	$(RM) $(@:%.ok=%/work) $(@:%.ok=%/report)
	$(MKDIR) $(@:%.ok=%)
	$(JDKHOME)/bin/java \
		-Djavatest.regtest.showCmd=true \
		-jar $(JTREG_IMAGEJARDIR)/jtreg.jar \
		-w:$(@:%.ok=%/work) \
		-r:$(@:%.ok=%/report) \
		-exclude:$(TESTDIR)/verifyexclude/ProblemList/id.txt \
		--verify-exclude \
		$(TESTDIR)/verifyexclude \
		> $(@:%.ok=%/log) \
		&& (cat $(@:%.ok=%/log) ; exit 1) \
		|| (exit 0)
	$(GREP) -s "The fully qualified test must exist." $(@:%.ok=%/log)
	$(GREP) -s "Test results" $(@:%.ok=%/log) && \
		(echo $@ failed, did not abort on exclude errors) && exit 1 || (exit 0)
	echo $@ passed at `date` > $@

$(BUILDTESTDIR)/verifyexclude.format.whitespace.ok: \
                $(JTREG_IMAGEDIR)/lib/jtreg.jar \
		$(JTREG_IMAGEDIR)/lib/javatest.jar
	$(RM) $(@:%.ok=%/work) $(@:%.ok=%/report)
	$(MKDIR) $(@:%.ok=%)
	$(JDKHOME)/bin/java \
		-Djavatest.regtest.showCmd=true \
		-jar $(JTREG_IMAGEJARDIR)/jtreg.jar \
		-w:$(@:%.ok=%/work) \
		-r:$(@:%.ok=%/report) \
		-exclude:$(TESTDIR)/verifyexclude/ProblemList/format.whitespace.txt \
		--verify-exclude \
		$(TESTDIR)/verifyexclude \
		> $(@:%.ok=%/log) \
	    || (cat $(@:%.ok=%/log) ; exit 1)
	echo $@ passed at `date` > $@

$(BUILDTESTDIR)/verifyexclude.good.ok: \
                $(JTREG_IMAGEDIR)/lib/jtreg.jar \
		$(JTREG_IMAGEDIR)/lib/javatest.jar
	$(RM) $(@:%.ok=%/work) $(@:%.ok=%/report)
	$(MKDIR) $(@:%.ok=%)
	$(JDKHOME)/bin/java \
		-Djavatest.regtest.showCmd=true \
		-jar $(JTREG_IMAGEJARDIR)/jtreg.jar \
		-w:$(@:%.ok=%/work) \
		-r:$(@:%.ok=%/report) \
		-exclude:$(TESTDIR)/verifyexclude/ProblemList/good.txt \
		--verify-exclude \
		$(TESTDIR)/verifyexclude \
		> $(@:%.ok=%/log) \
	    || (cat $(@:%.ok=%/log) ; exit 1)
	echo $@ passed at `date` > $@

$(BUILDTESTDIR)/verifyexclude.duplicate.ok: \
                $(JTREG_IMAGEDIR)/lib/jtreg.jar \
		$(JTREG_IMAGEDIR)/lib/javatest.jar
	$(RM) $(@:%.ok=%/work) $(@:%.ok=%/report)
	$(MKDIR) $(@:%.ok=%)
	$(JDKHOME)/bin/java \
		-Djavatest.regtest.showCmd=true \
		-jar $(JTREG_IMAGEJARDIR)/jtreg.jar \
		-w:$(@:%.ok=%/work) \
		-r:$(@:%.ok=%/report) \
		-exclude:$(TESTDIR)/verifyexclude/ProblemList/duplicate.txt \
		--verify-exclude \
		$(TESTDIR)/verifyexclude \
		> $(@:%.ok=%/log) \
	    && (cat $(@:%.ok=%/log) ; exit 1) \
		|| (exit 0)
	$(GREP) -s "Exclude file cannot contain duplicate entries." $(@:%.ok=%/log)
	$(GREP) -s "Test results" $(@:%.ok=%/log) && \
		(echo $@ failed, did not abort on exclude errors) && exit 1 || (exit 0)
	echo $@ passed at `date` > $@

$(BUILDTESTDIR)/verifyexclude.match.format.1.ok: \
                $(JTREG_IMAGEDIR)/lib/jtreg.jar \
		$(JTREG_IMAGEDIR)/lib/javatest.jar
	$(RM) $(@:%.ok=%/work) $(@:%.ok=%/report)
	$(MKDIR) $(@:%.ok=%)
	$(JDKHOME)/bin/java \
		-Djavatest.regtest.showCmd=true \
		-jar $(JTREG_IMAGEJARDIR)/jtreg.jar \
		-w:$(@:%.ok=%/work) \
		-r:$(@:%.ok=%/report) \
		-match:$(TESTDIR)/verifyexclude/ProblemList/format.1.txt \
		--verify-exclude \
		$(TESTDIR)/verifyexclude \
		> $(@:%.ok=%/log) \
	    && (cat $(@:%.ok=%/log) ; exit 1) \
		|| (exit 0)
	$(GREP) -s "Must follow:" $(@:%.ok=%/log)
	$(GREP) -s "The fully qualified test must exist." $(@:%.ok=%/log) && exit 1 || exit 0
	$(GREP) -s "Test results" $(@:%.ok=%/log) && \
		(echo $@ failed, did not abort on exclude errors) && exit 1 || (exit 0)
	echo $@ passed at `date` > $@

$(BUILDTESTDIR)/verifyexclude.shortform.ok: \
                $(JTREG_IMAGEDIR)/lib/jtreg.jar \
		$(JTREG_IMAGEDIR)/lib/javatest.jar
	$(RM) $(@:%.ok=%/work) $(@:%.ok=%/report)
	$(MKDIR) $(@:%.ok=%)
	$(JDKHOME)/bin/java \
		-Djavatest.regtest.showCmd=true \
		-jar $(JTREG_IMAGEJARDIR)/jtreg.jar \
		-w:$(@:%.ok=%/work) \
		-r:$(@:%.ok=%/report) \
		-exclude:$(TESTDIR)/verifyexclude/ProblemList/format.1.txt \
		-i \
		$(TESTDIR)/verifyexclude \
		> $(@:%.ok=%/log) \
	    && (cat $(@:%.ok=%/log) ; exit 1) \
		|| (exit 0)
	$(GREP) -s "Must follow:" $(@:%.ok=%/log)
	$(GREP) -s "Test results" $(@:%.ok=%/log) && \
		(echo $@ failed, did not abort on exclude errors) && exit 1 || (exit 0)
	echo $@ passed at `date` > $@

$(BUILDTESTDIR)/verifyexclude.good.invoke.direct.ok: \
                $(JTREG_IMAGEDIR)/lib/jtreg.jar \
		$(JTREG_IMAGEDIR)/lib/javatest.jar
	$(RM) $(@:%.ok=%/work) $(@:%.ok=%/report)
	$(MKDIR) $(@:%.ok=%)
	$(JDKHOME)/bin/java \
		-Djavatest.regtest.showCmd=true \
		-jar $(JTREG_IMAGEJARDIR)/jtreg.jar \
		-w:$(@:%.ok=%/work) \
		-r:$(@:%.ok=%/report) \
		-exclude:$(TESTDIR)/verifyexclude/ProblemList/good.txt \
		--verify-exclude \
		$(TESTDIR)/verifyexclude/p/Pass.java \
		> $(@:%.ok=%/log) \
	    || (cat $(@:%.ok=%/log) ; exit 1)
	echo $@ passed at `date` > $@

$(BUILDTESTDIR)/verifyexclude.good.invoke.group.ok: \
                $(JTREG_IMAGEDIR)/lib/jtreg.jar \
		$(JTREG_IMAGEDIR)/lib/javatest.jar
	$(RM) $(@:%.ok=%/work) $(@:%.ok=%/report)
	$(MKDIR) $(@:%.ok=%)
	$(JDKHOME)/bin/java \
		-Djavatest.regtest.showCmd=true \
		-jar $(JTREG_IMAGEJARDIR)/jtreg.jar \
		-w:$(@:%.ok=%/work) \
		-r:$(@:%.ok=%/report) \
		-exclude:$(TESTDIR)/verifyexclude/ProblemList/good.txt \
		--verify-exclude \
		$(TESTDIR)/verifyexclude:group1 \
		> $(@:%.ok=%/log) \
	    || (cat $(@:%.ok=%/log) ; exit 1)
	echo $@ passed at `date` > $@

$(BUILDTESTDIR)/verifyexclude.good.verbose.ok: \
                $(JTREG_IMAGEDIR)/lib/jtreg.jar \
		$(JTREG_IMAGEDIR)/lib/javatest.jar
	$(RM) $(@:%.ok=%/work) $(@:%.ok=%/report)
	$(MKDIR) $(@:%.ok=%)
	$(JDKHOME)/bin/java \
		-Djavatest.regtest.showCmd=true \
		-jar $(JTREG_IMAGEJARDIR)/jtreg.jar \
		-w:$(@:%.ok=%/work) \
		-r:$(@:%.ok=%/report) \
		-v \
		-exclude:$(TESTDIR)/verifyexclude/ProblemList/good.txt \
		--verify-exclude \
		$(TESTDIR)/verifyexclude \
		> $(@:%.ok=%/log) \
	    || (cat $(@:%.ok=%/log) ; exit 1)
	$(GREP) -s "Testsuite:" $(@:%.ok=%/log)
	echo $@ passed at `date` > $@

$(BUILDTESTDIR)/verifyexclude.good.verbose.all.ok: \
                $(JTREG_IMAGEDIR)/lib/jtreg.jar \
		$(JTREG_IMAGEDIR)/lib/javatest.jar
	$(RM) $(@:%.ok=%/work) $(@:%.ok=%/report)
	$(MKDIR) $(@:%.ok=%)
	$(JDKHOME)/bin/java \
		-Djavatest.regtest.showCmd=true \
		-jar $(JTREG_IMAGEJARDIR)/jtreg.jar \
		-w:$(@:%.ok=%/work) \
		-r:$(@:%.ok=%/report) \
		-va \
		-exclude:$(TESTDIR)/verifyexclude/ProblemList/good.txt \
		--verify-exclude \
		$(TESTDIR)/verifyexclude \
		> $(@:%.ok=%/log) \
	    || (cat $(@:%.ok=%/log) ; exit 1)
	$(GREP) -s "All recognized tests:" $(@:%.ok=%/log)
	echo $@ passed at `date` > $@

TESTS.jtreg += \
	$(BUILDTESTDIR)/verifyexclude.duplicate.ok \
	$(BUILDTESTDIR)/verifyexclude.format.1.ok \
	$(BUILDTESTDIR)/verifyexclude.format.2.ok \
	$(BUILDTESTDIR)/verifyexclude.good.ok \
	$(BUILDTESTDIR)/verifyexclude.id.ok \
	$(BUILDTESTDIR)/verifyexclude.exist.ok \
	$(BUILDTESTDIR)/verifyexclude.match.format.1.ok \
	$(BUILDTESTDIR)/verifyexclude.shortform.ok \
	$(BUILDTESTDIR)/verifyexclude.good.invoke.direct.ok \
	$(BUILDTESTDIR)/verifyexclude.good.verbose.all.ok \
	$(BUILDTESTDIR)/verifyexclude.good.verbose.ok