From: Jon Griffiths Date: Thu, 1 Sep 2016 09:24:36 +0000 (+1200) Subject: Makefile: First try at rewriting the ccan Makefile. X-Git-Url: https://git.ozlabs.org/?p=ccan;a=commitdiff_plain;h=2ee3b7c5586c283412ace072eed9d70a86368efd;hp=f94261729ff13eb4e1f3341167c0e54afaa6a73a Makefile: First try at rewriting the ccan Makefile. This change contains a simpler Makefile replacement with only 62 lines of directives, 10 rules, and a 13 line support script for dependencies. The build dependencies have been minimised and in some cases, corrected. FEATURES: * All targets can be built from a clean tree in one invocation. * Parallel builds (tested with -j32 on 8 cores). * Auto discovery of modules via _info files. * Hopefully complete dependencies via a simplified generator. * CFLAGS are respected and appended to compile flags. * LINTFLAGS can be set to add check options (e.g. LINTFLAGS=-v). * 'make clean' doesn't build anything before cleaning now. * 'make quiet=1' builds quietly. 'make check quiet=1 -j N' produces summary output like the former summary target. * Non-phony test targets; tests are rebuilt only when dirty. Targets are: check, fastcheck and fullcheck, the latter runs in non-summary mode. * 'make .[check|fastcheck|fullcheck]' runs tests for single modules. TODO: * Support Makefile-web and any other scattered targets NOTES: * The changes to dependency generation expose a circular dependency between asort and order which is not fixed here. * Tests always run their dependent tests. With -j support and minimised rebuilds via tighter dependencies, its not worth avoiding. * Some targets have been dropped as uneeded (e.g. distclean, tools). Signed-off-by: Jon Griffiths --- diff --git a/.gitignore b/.gitignore index 680d1296..0ef27518 100644 --- a/.gitignore +++ b/.gitignore @@ -1,21 +1,17 @@ TAGS -.depends .valgrind_suppressions *.d *.o -libccan.a +*.ok config.h +config.h.tmp *~ tools/ccan_depends tools/doc_extract tools/namespacize tools/run_tests tools/ccanlint/ccanlint -tools/ccanlint/generated-testlist tools/modfiles -inter-depends -test-depends -lib-depends tools/_infotojson/infotojson tools/ccanlint/test/run-file_analysis tools/configurator/configurator diff --git a/Makefile b/Makefile index 127b8757..ff0bad4e 100644 --- a/Makefile +++ b/Makefile @@ -1,97 +1,94 @@ -# Hacky makefile to compile everything and run the tests in some kind -# of sane order. - -# Main targets: -# -# check: run tests on all ccan modules (use 'make check V=--verbose' for more) -# Includes building libccan.a. -# libccan.a: A library with all the ccan modules in it. -# tools: build useful tools in tools/ dir. -# Especially tools/ccanlint/ccanlint and tools/namespacize. -# distclean: destroy everything back to pristine state - -# Where make scores puts the results -SCOREDIR=scores/$(shell whoami)/$(shell uname -s)-$(shell uname -m)-$(CC)-$(shell git describe --always --dirty) -CCANLINT=tools/ccanlint/ccanlint --deps-fail-ignore -CCANLINT_FAST=$(CCANLINT) -x tests_pass_valgrind -x tests_compile_coverage - -default: all_info libccan.a - -ALL_DEPENDS=$(patsubst %, ccan/%/.depends, $(MODS)) - -# By default, we skip modules with external deps (or plaform specific) -MODS_EXCLUDE:=altstack generator jmap jset nfs ogg_to_pcm tal/talloc wwviaudio -# This randomly fails, and reliably fails under Jenkins :( -MODS_FLAKY:=altstack -MODS_RELIABLE=$(filter-out $(MODS_FLAKY),$(MODS)) - -include Makefile-ccan - -fastcheck: $(MODS_RELIABLE:%=summary-fastcheck/%) - -check: $(MODS_RELIABLE:%=summary-check/%) - -distclean: clean - rm -f $(ALL_DEPENDS) - -scores: $(SCOREDIR)/SUMMARY - -$(SCOREDIR)/SUMMARY: $(MODS:%=$(SCOREDIR)/%.score) - git describe --always > $@ - uname -a >> $@ - $(CC) -v >> $@ - cat $^ | grep 'Total score:' >> $@ - -$(SCOREDIR)/%.score: ccan/%/_info tools/ccanlint/ccanlint $(OBJFILES) - mkdir -p `dirname $@` - $(CCANLINT) -v -s ccan/$* > $@ || true - -$(ALL_DEPENDS): %/.depends: %/_info tools/ccan_depends - tools/ccan_depends $* > $@ || ( rm -f $@; exit 1 ) - -# Actual dependencies are created in inter-depends -check/%: tools/ccanlint/ccanlint - $(CCANLINT) ccan/$* - -fastcheck/%: tools/ccanlint/ccanlint - $(CCANLINT_FAST) ccan/$* - -# Doesn't test dependencies, doesn't print verbose fail results. -summary-check/%: tools/ccanlint/ccanlint $(OBJFILES) - $(CCANLINT) -s ccan/$* - -summary-fastcheck/%: tools/ccanlint/ccanlint $(OBJFILES) - $(CCANLINT_FAST) -s ccan/$* - -ccan/%/info: ccan/%/_info config.h - $(CC) $(CCAN_CFLAGS) -I. -o $@ -x c $< - -all_info: $(MODS:%=ccan/%/info) - -clean: tools-clean - rm -f `find * -name '*.o'` `find * -name '.depends'` `find * -name '*.a'` `find * -name info` `find * -name '*.d'` `find ccan -name '*-Makefile'` - rm -f config.h - rm -f inter-depends lib-depends test-depends - -# Creates a dependency from the tests to the object files which it needs. -inter-depends: $(ALL_DEPENDS) Makefile - for f in $(ALL_DEPENDS); do echo check-$$(basename $$(dirname $$f) ): $$(for dir in $$(cat $$f) $$(dirname $$f); do [ "$$(echo $$dir/*.c)" = "$$dir/*.c" ] || echo ccan/"$$(basename $$dir)".o; done); done > $@ - -# Creates dependencies between tests, so if foo depends on bar, bar is tested -# first -test-depends: $(ALL_DEPENDS) Makefile - for f in $(ALL_DEPENDS); do echo check/`basename \`dirname $$f\``: `sed -n 's,ccan/\(.*\),check/\1,p' < $$f`; done > $@ - -TAGS: FORCE - find * -name '*.[ch]' | xargs etags - -FORCE: - -# Ensure we don't end up with empty file if configurator fails! -config.h: tools/configurator/configurator Makefile Makefile-ccan - tools/configurator/configurator $(CC) $(CCAN_CFLAGS) > $@.tmp && mv $@.tmp $@ - -include tools/Makefile --include inter-depends --include test-depends --include Makefile-web +# Makefile for CCAN + +# 'make quiet=1' builds silently +QUIETEN.1 := @ +PRE := $(QUIETEN.$(quiet)) + +all:: + +# Our flags for building +WARN_CFLAGS := -Wall -Wstrict-prototypes -Wold-style-definition -Wundef \ + -Wmissing-prototypes -Wmissing-declarations -Wpointer-arith -Wwrite-strings +DEP_CFLAGS = -MMD -MP -MF$(@:%=%.d) -MT$@ +CCAN_CFLAGS := -g3 -ggdb $(WARN_CFLAGS) -DCCAN_STR_DEBUG=1 -I. $(CFLAGS) + +# Anything with an _info file is a module ... +INFO_SRCS := $(wildcard ccan/*/_info ccan/*/*/_info) +ALL_INFOS := $(INFO_SRCS:%_info=%info) +ALL_MODULES := $(ALL_INFOS:%/info=%) + +# ... Except stuff that needs external dependencies, which we exclude +EXCLUDE := altstack generator jmap jset nfs ogg_to_pcm tal/talloc wwviaudio +MODULES:= $(filter-out $(EXCLUDE:%=ccan/%), $(ALL_MODULES)) + +# Sources are C files in each module, objects the resulting .o files +SRCS := $(wildcard $(MODULES:%=%/*.c)) +OBJS := $(SRCS:%.c=%.o) +DEPS := $(OBJS:%=%.d) + +# We build all object files using our CCAN_CFLAGS, after config.h +%.o : %.c config.h + $(PRE)$(CC) $(CCAN_CFLAGS) $(DEP_CFLAGS) -c $< -o $@ + +# _info files are compiled into executables and don't need dependencies +%info : %_info config.h + $(PRE)$(CC) $(CCAN_CFLAGS) -I. -o $@ -x c $< + +# config.h is built by configurator which has no ccan dependencies +CONFIGURATOR := tools/configurator/configurator +$(CONFIGURATOR): $(CONFIGURATOR).c + $(PRE)$(CC) $(CCAN_CFLAGS) $(DEP_CFLAGS) $< -o $@ +config.h: $(CONFIGURATOR) Makefile + $(PRE)$(CONFIGURATOR) $(CC) $(CCAN_CFLAGS) >$@.tmp && mv $@.tmp $@ + +# Tools +TOOLS := tools/ccan_depends tools/doc_extract tools/namespacize tools/modfiles +TOOLS_SRCS := $(filter-out $(TOOLS:%=%.c), $(wildcard tools/*.c)) +TOOLS_DEPS := $(TOOLS_SRCS:%.c=%.d) $(TOOLS:%=%.d) +TOOLS_CCAN_MODULES := err foreach hash htable list noerr opt rbuf \ + read_write_all str take tal tal/grab_file tal/link tal/path tal/str time +TOOLS_CCAN_SRCS := $(wildcard $(TOOLS_CCAN_MODULES:%=ccan/%/*.c)) +TOOLS_OBJS := $(TOOLS_SRCS:%.c=%.o) $(TOOLS_CCAN_SRCS:%.c=%.o) +tools/% : tools/%.c $(TOOLS_OBJS) + $(PRE)$(CC) $(CCAN_CFLAGS) $(DEP_CFLAGS) $< $(TOOLS_OBJS) -lm -o $@ + +# ccanlint +LINT := tools/ccanlint/ccanlint +LINT_OPTS.ok := -s +LINT_OPTS.fast.ok := -s -x tests_pass_valgrind -x tests_compile_coverage +LINT_SRCS := $(filter-out $(LINT).c, $(wildcard tools/ccanlint/*.c tools/ccanlint/tests/*.c)) +LINT_DEPS := $(LINT_SRCS:%.c=%.d) $(LINT).d +LINT_CCAN_MODULES := asort autodata dgraph ilog lbalance ptr_valid strmap +LINT_CCAN_SRCS := $(wildcard $(LINT_CCAN_MODULES:%=ccan/%/*.c)) +LINT_OBJS := $(LINT_SRCS:%.c=%.o) $(LINT_CCAN_SRCS:%.c=%.o) $(TOOLS_OBJS) +$(LINT): $(LINT).c $(LINT_OBJS) + $(PRE)$(CC) $(CCAN_CFLAGS) $(DEP_CFLAGS) $(LINT).c $(LINT_OBJS) -lm -o $@ + +# We generate dependencies for tests into a .d file +%/.d: %/info tools/gen_deps.sh tools/ccan_depends + $(PRE)tools/gen_deps.sh $* > $@ || rm -f $@ +TEST_DEPS := $(MODULES:%=%/.d) + +# We produce .ok files when the tests succeed +%.ok: $(LINT) + $(PRE)$(LINT) $(LINT_OPTS$(notdir $@)) --deps-fail-ignore $(LINTFLAGS) $(dir $*) && touch $@ + +check: $(MODULES:%=%/.ok) +fastcheck: $(MODULES:%=%/.fast.ok) +fullcheck: $(MODULES:%=%/.full.ok) + +ifeq ($(strip $(filter clean config.h, $(MAKECMDGOALS))),) +-include $(DEPS) $(LINT_DEPS) $(TOOLS_DEPS) $(TEST_DEPS) +endif + +# Default target: object files, info files and tools +all:: $(OBJS) $(ALL_INFOS) $(CONFIGURATOR) $(LINT) $(TOOLS) + +.PHONY: clean TAGS +clean: + $(PRE)find . -name "*.d" -o -name "*.o" -o -name "*.ok" | xargs -n 256 rm -f + $(PRE)rm -f $(CONFIGURATOR) $(LINT) $(TOOLS) TAGS config.h config.h.d $(ALL_INFOS) + +# 'make TAGS' builds etags +TAGS: + $(PRE)find * -name '*.[ch]' | xargs etags diff --git a/Makefile-ccan b/Makefile-ccan deleted file mode 100644 index 5c8dea2e..00000000 --- a/Makefile-ccan +++ /dev/null @@ -1,160 +0,0 @@ -# Example makefile which makes a "libccan.a" of everything under ccan/. -# For simple projects you could just do: -# SRCFILES += $(wildcard ccan/*/*.c) - -#CCAN_CFLAGS=-g -O3 -Wall -Wstrict-prototypes -Wold-style-definition -Wmissing-prototypes -Wmissing-declarations -Wpointer-arith -Wwrite-strings -Wundef -DCCAN_STR_DEBUG=1 -CCAN_CFLAGS=-g3 -ggdb -Wall -Wstrict-prototypes -Wold-style-definition -Wmissing-prototypes -Wmissing-declarations -Wpointer-arith -Wwrite-strings -Wundef -DCCAN_STR_DEBUG=1 -CFLAGS = $(CCAN_CFLAGS) -I. $(DEPGEN) - -MODS := a_star \ - aga \ - agar \ - alignof \ - altstack \ - antithread \ - antithread/alloc \ - argcheck \ - array_size \ - asearch \ - asort \ - asprintf \ - autodata \ - avl \ - base64 \ - bdelta \ - bitmap \ - block_pool \ - breakpoint \ - btree \ - build_assert \ - bytestring \ - cast \ - ccan_tokenizer \ - cdump \ - charset \ - check_type \ - ciniparser \ - compiler \ - container_of \ - cppmagic \ - cpuid \ - crc \ - crcsync \ - crypto/ripemd160 \ - crypto/sha256 \ - crypto/sha512 \ - crypto/shachain \ - crypto/siphash24 \ - daemonize \ - daemon_with_notify \ - darray \ - deque \ - dgraph \ - endian \ - eratosthenes \ - err \ - failtest \ - foreach \ - generator \ - grab_file \ - hash \ - heap \ - htable \ - idtree \ - ilog \ - invbloom \ - io \ - isaac \ - iscsi \ - jacobson_karels \ - jmap \ - jset \ - json \ - lbalance \ - likely \ - list \ - lpq \ - lqueue \ - lstack \ - md4 \ - mem \ - minmax \ - net \ - nfs \ - noerr \ - ntdb \ - objset \ - ogg_to_pcm \ - opt \ - order \ - permutation \ - pipecmd \ - pr_log \ - ptrint \ - ptr_valid \ - pushpull \ - rbtree \ - rbuf \ - read_write_all \ - rfc822 \ - rszshm \ - short_types \ - siphash \ - sparse_bsearch \ - str \ - str/hex \ - strgrp \ - stringbuilder \ - stringmap \ - strmap \ - strset \ - structeq \ - take \ - tal \ - tal/grab_file \ - tal/link \ - tal/path \ - tal/stack \ - tal/str \ - tal/talloc \ - talloc \ - tally \ - tap \ - tcon \ - time \ - timer \ - tlist \ - tlist2 \ - ttxml \ - typesafe_cb \ - version \ - wwviaudio \ - xstring - -# Anything with C files needs building; dir leaves / on, sort uniquifies -MODS_WITH_SRC = $(patsubst ccan/%/, %, $(sort $(foreach m, $(MODS), $(dir $(wildcard ccan/$m/*.c))))) - -default: libccan.a - -# Automatic dependency generation: makes ccan/*/*.d files. -DEPGEN=-MMD --include $(foreach m, $(MODS), ccan/$(m)/*.d) - -DIRS=$(patsubst %, ccan/%, $(filter-out $(MODS_EXCLUDE), $(MODS_WITH_SRC))) - -# Generate everyone's separate Makefiles. --include $(foreach dir, $(DIRS), $(dir)-Makefile) - -ccan/%-Makefile: - @echo $@: $(wildcard ccan/$*/*.[ch]) ccan/$*/_info > $@ - @echo ccan/$*.o: $(patsubst %.c, %.o, $(wildcard ccan/$*/*.c)) >> $@ - -# We compile all the ccan/foo/*.o files together into ccan/foo.o -OBJFILES=$(DIRS:=.o) - -# We create all the .o files and link them together. -$(OBJFILES): %.o: - $(LD) -r -o $@ $^ - -libccan.a: $(OBJFILES) - $(AR) r $@ $(OBJFILES) diff --git a/tools/Makefile b/tools/Makefile deleted file mode 100644 index 6e4833a3..00000000 --- a/tools/Makefile +++ /dev/null @@ -1,50 +0,0 @@ -ALL_TOOLS = tools/configurator/configurator tools/ccan_depends tools/doc_extract tools/namespacize tools/ccanlint/ccanlint tools/modfiles -LDLIBS = -lrt -DEP_OBJS = ccan/err/err.o \ - ccan/foreach/foreach.o \ - ccan/hash/hash.o \ - ccan/htable/htable.o \ - ccan/list/list.o \ - ccan/noerr/noerr.o \ - ccan/opt/opt.o \ - ccan/opt/helpers.o \ - ccan/opt/parse.o \ - ccan/opt/usage.o \ - ccan/rbuf/rbuf.o \ - ccan/read_write_all/read_write_all.o \ - ccan/str/debug.o \ - ccan/str/str.o \ - ccan/take/take.o \ - ccan/tal/tal.o \ - ccan/tal/grab_file/grab_file.o \ - ccan/tal/link/link.o \ - ccan/tal/path/path.o \ - ccan/tal/str/str.o \ - ccan/time/time.o \ - tools/read_config_header.o \ - tools/ccan_dir.o \ - tools/compile.o \ - tools/depends.o \ - tools/tools.o - -.PHONY: tools -tools: $(ALL_TOOLS) - -tools/ccan_depends.o: config.h - -tools/ccan_depends: tools/ccan_depends.o $(DEP_OBJS) - -tools/doc_extract: tools/doc_extract.o tools/doc_extract-core.o $(DEP_OBJS) - -tools/namespacize: tools/namespacize.o $(DEP_OBJS) - -tools/namespacize.o tools/depends.o: tools/tools.h - -tools/configurator/configurator: tools/configurator/configurator.c - -tools/modfiles: tools/modfiles.o tools/manifest.o $(DEP_OBJS) - -tools-clean: ccanlint-clean - rm -f $(ALL_TOOLS) - -include tools/ccanlint/Makefile diff --git a/tools/ccanlint/Makefile b/tools/ccanlint/Makefile deleted file mode 100644 index 5a266796..00000000 --- a/tools/ccanlint/Makefile +++ /dev/null @@ -1,50 +0,0 @@ -TEST_CFILES := $(wildcard tools/ccanlint/tests/*.c) -TEST_OBJS := $(TEST_CFILES:.c=.o) - -CORE_OBJS := \ - ccan/asort/asort.o \ - ccan/autodata/autodata.o \ - ccan/dgraph/dgraph.o \ - ccan/foreach/foreach.o \ - ccan/hash/hash.o \ - ccan/htable/htable.o \ - ccan/ilog/ilog.o \ - ccan/lbalance/lbalance.o \ - ccan/list/list.o \ - ccan/noerr/noerr.o \ - ccan/opt/helpers.o \ - ccan/opt/opt.o \ - ccan/opt/parse.o \ - ccan/opt/usage.o \ - ccan/ptr_valid/ptr_valid.o \ - ccan/rbuf/rbuf.o \ - ccan/read_write_all/read_write_all.o \ - ccan/str/str.o ccan/str/debug.o \ - ccan/strmap/strmap.o \ - ccan/take/take.o \ - ccan/tal/tal.o \ - ccan/tal/grab_file/grab_file.o \ - ccan/tal/link/link.o \ - ccan/tal/path/path.o \ - ccan/tal/str/str.o \ - ccan/time/time.o \ - tools/ccanlint/async.o \ - tools/ccanlint/ccanlint.o \ - tools/ccanlint/file_analysis.o \ - tools/ccanlint/licenses.o \ - tools/ccan_dir.o \ - tools/compile.o \ - tools/depends.o \ - tools/doc_extract-core.o \ - tools/manifest.o \ - tools/read_config_header.o \ - tools/tools.o - -OBJS := $(CORE_OBJS) $(TEST_OBJS) - -$(CORE_OBJS): config.h - -tools/ccanlint/ccanlint: $(OBJS) - -ccanlint-clean: - rm -f tools/ccanlint/ccanlint diff --git a/tools/gen_deps.sh b/tools/gen_deps.sh new file mode 100755 index 00000000..19cd3d63 --- /dev/null +++ b/tools/gen_deps.sh @@ -0,0 +1,27 @@ +#! /bin/sh + +# Compute the test dependencies for a ccan module. Usage: +# tools/gen_deps.sh ccan/path/to/module + +path=$1 +module=`echo $path | sed 's/^ccan\///g'` + +# The test depends on the test sources ... +test_srcs=`ls $path/test/*.[ch] 2>/dev/null | tr '\n' ' '` + +# ... and the object files of our module (rather than the sources, so +# that we pick up the resursive dependencies for the objects) +module_objs=`ls $path/*.c 2>/dev/null | sed 's/.c$/.o/g' | tr '\n' ' '` + +# ... and on the modules this test uses having passed their tests +deps=$(echo `$path/info testdepends` `$path/info depends` | tr ' ' '\n' | \ + sort | uniq | sed -e 's/$/\/.ok/g' -e '/^\/.ok$/d' | tr '\n' ' ') + +# Print the test targets and target aliases +echo "${module}_ok_deps := $test_srcs $module_objs $deps" +echo "$path/.ok: \$(${module}_ok_deps)" +echo "$path/.fast.ok: \$(${module}_ok_deps:%.ok=%.fast.ok)" +echo "$path/.full.ok: \$(${module}_ok_deps:%.ok=%.full.ok)" +echo "${module}.check: $path/.ok" +echo "${module}.fastcheck: $path/.fast.ok" +echo "${module}.fullcheck: $path/.full.ok"