diff options
| -rw-r--r-- | .cocciconfig | 3 | ||||
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | Documentation/coccinelle.txt | 148 | ||||
| -rwxr-xr-x | scripts/coccicheck | 96 | ||||
| -rw-r--r-- | scripts/coccinelle/free/devm_free.cocci | 26 | ||||
| -rw-r--r-- | scripts/coccinelle/free/ifnullfree.cocci | 4 | ||||
| -rw-r--r-- | scripts/coccinelle/free/kfree.cocci | 18 | ||||
| -rw-r--r-- | scripts/coccinelle/free/kfreeaddr.cocci | 6 | ||||
| -rw-r--r-- | scripts/coccinelle/iterators/device_node_continue.cocci | 3 | ||||
| -rw-r--r-- | scripts/coccinelle/misc/noderef.cocci | 18 | ||||
| -rwxr-xr-x | scripts/package/builddeb | 11 |
11 files changed, 313 insertions, 21 deletions
diff --git a/.cocciconfig b/.cocciconfig new file mode 100644 index 000000000000..43967c6b2015 --- /dev/null +++ b/.cocciconfig @@ -0,0 +1,3 @@ +[spatch] + options = --timeout 200 + options = --use-gitgrep diff --git a/.gitignore b/.gitignore index 2be25f771bd8..c2ed4ecb0acd 100644 --- a/.gitignore +++ b/.gitignore @@ -67,6 +67,7 @@ Module.symvers # !.gitignore !.mailmap +!.cocciconfig # # Generated include files diff --git a/Documentation/coccinelle.txt b/Documentation/coccinelle.txt index 7f773d51fdd9..01fb1dae3163 100644 --- a/Documentation/coccinelle.txt +++ b/Documentation/coccinelle.txt @@ -38,6 +38,15 @@ as a regular user, and install it with sudo make install + Supplemental documentation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For supplemental documentation refer to the wiki: + +https://bottest.wiki.kernel.org/coccicheck + +The wiki documentation always refers to the linux-next version of the script. + Using Coccinelle on the Linux kernel ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -94,11 +103,26 @@ To enable verbose messages set the V= variable, for example: make coccicheck MODE=report V=1 + Coccinelle parallelization +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + By default, coccicheck tries to run as parallel as possible. To change the parallelism, set the J= variable. For example, to run across 4 CPUs: make coccicheck MODE=report J=4 +As of Coccinelle 1.0.2 Coccinelle uses Ocaml parmap for parallelization, +if support for this is detected you will benefit from parmap parallelization. + +When parmap is enabled coccicheck will enable dynamic load balancing by using +'--chunksize 1' argument, this ensures we keep feeding threads with work +one by one, so that we avoid the situation where most work gets done by only +a few threads. With dynamic load balancing, if a thread finishes early we keep +feeding it more work. + +When parmap is enabled, if an error occurs in Coccinelle, this error +value is propagated back, the return value of the 'make coccicheck' +captures this return value. Using Coccinelle with a single semantic patch ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -142,15 +166,118 @@ semantic patch as shown in the previous section. The "report" mode is the default. You can select another one with the MODE variable explained above. + Debugging Coccinelle SmPL patches +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Using coccicheck is best as it provides in the spatch command line +include options matching the options used when we compile the kernel. +You can learn what these options are by using V=1, you could then +manually run Coccinelle with debug options added. + +Alternatively you can debug running Coccinelle against SmPL patches +by asking for stderr to be redirected to stderr, by default stderr +is redirected to /dev/null, if you'd like to capture stderr you +can specify the DEBUG_FILE="file.txt" option to coccicheck. For +instance: + + rm -f cocci.err + make coccicheck COCCI=scripts/coccinelle/free/kfree.cocci MODE=report DEBUG_FILE=cocci.err + cat cocci.err + +You can use SPFLAGS to add debugging flags, for instance you may want to +add both --profile --show-trying to SPFLAGS when debugging. For instance +you may want to use: + + rm -f err.log + export COCCI=scripts/coccinelle/misc/irqf_oneshot.cocci + make coccicheck DEBUG_FILE="err.log" MODE=report SPFLAGS="--profile --show-trying" M=./drivers/mfd/arizona-irq.c + +err.log will now have the profiling information, while stdout will +provide some progress information as Coccinelle moves forward with +work. + +DEBUG_FILE support is only supported when using coccinelle >= 1.2. + + .cocciconfig support +~~~~~~~~~~~~~~~~~~~~~~ + +Coccinelle supports reading .cocciconfig for default Coccinelle options that +should be used every time spatch is spawned, the order of precedence for +variables for .cocciconfig is as follows: + + o Your current user's home directory is processed first + o Your directory from which spatch is called is processed next + o The directory provided with the --dir option is processed last, if used + +Since coccicheck runs through make, it naturally runs from the kernel +proper dir, as such the second rule above would be implied for picking up a +.cocciconfig when using 'make coccicheck'. + +'make coccicheck' also supports using M= targets.If you do not supply +any M= target, it is assumed you want to target the entire kernel. +The kernel coccicheck script has: + + if [ "$KBUILD_EXTMOD" = "" ] ; then + OPTIONS="--dir $srctree $COCCIINCLUDE" + else + OPTIONS="--dir $KBUILD_EXTMOD $COCCIINCLUDE" + fi + +KBUILD_EXTMOD is set when an explicit target with M= is used. For both cases +the spatch --dir argument is used, as such third rule applies when whether M= +is used or not, and when M= is used the target directory can have its own +.cocciconfig file. When M= is not passed as an argument to coccicheck the +target directory is the same as the directory from where spatch was called. + +If not using the kernel's coccicheck target, keep the above precedence +order logic of .cocciconfig reading. If using the kernel's coccicheck target, +override any of the kernel's .coccicheck's settings using SPFLAGS. + +We help Coccinelle when used against Linux with a set of sensible defaults +options for Linux with our own Linux .cocciconfig. This hints to coccinelle +git can be used for 'git grep' queries over coccigrep. A timeout of 200 +seconds should suffice for now. + +The options picked up by coccinelle when reading a .cocciconfig do not appear +as arguments to spatch processes running on your system, to confirm what +options will be used by Coccinelle run: + + spatch --print-options-only + +You can override with your own preferred index option by using SPFLAGS. Take +note that when there are conflicting options Coccinelle takes precedence for +the last options passed. Using .cocciconfig is possible to use idutils, however +given the order of precedence followed by Coccinelle, since the kernel now +carries its own .cocciconfig, you will need to use SPFLAGS to use idutils if +desired. See below section "Additional flags" for more details on how to use +idutils. + Additional flags ~~~~~~~~~~~~~~~~~~ Additional flags can be passed to spatch through the SPFLAGS -variable. +variable. This works as Coccinelle respects the last flags +given to it when options are in conflict. make SPFLAGS=--use-glimpse coccicheck + +Coccinelle supports idutils as well but requires coccinelle >= 1.0.6. +When no ID file is specified coccinelle assumes your ID database file +is in the file .id-utils.index on the top level of the kernel, coccinelle +carries a script scripts/idutils_index.sh which creates the database with + + mkid -i C --output .id-utils.index + +If you have another database filename you can also just symlink with this +name. + make SPFLAGS=--use-idutils coccicheck +Alternatively you can specify the database filename explicitly, for +instance: + + make SPFLAGS="--use-idutils /full-path/to/ID" coccicheck + See spatch --help to learn more about spatch options. Note that the '--use-glimpse' and '--use-idutils' options @@ -159,6 +286,25 @@ thus active by default. However, by indexing the code with one of these tools, and according to the cocci file used, spatch could proceed the entire code base more quickly. + SmPL patch specific options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +SmPL patches can have their own requirements for options passed +to Coccinelle. SmPL patch specific options can be provided by +providing them at the top of the SmPL patch, for instance: + +// Options: --no-includes --include-headers + + SmPL patch Coccinelle requirements +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +As Coccinelle features get added some more advanced SmPL patches +may require newer versions of Coccinelle. If an SmPL patch requires +at least a version of Coccinelle, this can be specified as follows, +as an example if requiring at least Coccinelle >= 1.0.5: + +// Requires: 1.0.5 + Proposing new semantic patches ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/scripts/coccicheck b/scripts/coccicheck index dd85a455b2ba..c92c1528a54d 100755 --- a/scripts/coccicheck +++ b/scripts/coccicheck @@ -1,14 +1,24 @@ #!/bin/bash - +# Linux kernel coccicheck +# +# Read Documentation/coccinelle.txt # # This script requires at least spatch # version 1.0.0-rc11. -# +DIR="$(dirname $(readlink -f $0))/.." SPATCH="`which ${SPATCH:=spatch}`" -trap kill_running SIGTERM SIGINT -declare -a SPATCH_PID +if [ ! -x "$SPATCH" ]; then + echo 'spatch is part of the Coccinelle project and is available at http://coccinelle.lip6.fr/' + exit 1 +fi + +SPATCH_VERSION=$($SPATCH --version | head -1 | awk '{print $3}') +SPATCH_VERSION_NUM=$(echo $SPATCH_VERSION | ${DIR}/scripts/ld-version.sh) + +USE_JOBS="no" +$SPATCH --help | grep "\-\-jobs" > /dev/null && USE_JOBS="yes" # The verbosity may be set by the environmental parameter V= # as for example with 'make V=1 coccicheck' @@ -25,7 +35,28 @@ else NPROC="$J" fi -FLAGS="$SPFLAGS --very-quiet" +FLAGS="--very-quiet" + +# You can use SPFLAGS to append extra arguments to coccicheck or override any +# heuristics done in this file as Coccinelle accepts the last options when +# options conflict. +# +# A good example for use of SPFLAGS is if you want to debug your cocci script, +# you can for instance use the following: +# +# $ export COCCI=scripts/coccinelle/misc/irqf_oneshot.cocci +# $ make coccicheck MODE=report DEBUG_FILE="all.err" SPFLAGS="--profile --show-trying" M=./drivers/mfd/arizona-irq.c +# +# "--show-trying" should show you what rule is being processed as it goes to +# stdout, you do not need a debug file for that. The profile output will be +# be sent to stdout, if you provide a DEBUG_FILE the profiling data can be +# inspected there. +# +# --profile will not output if --very-quiet is used, so avoid it. +echo $SPFLAGS | egrep -e "--profile|--show-trying" 2>&1 > /dev/null +if [ $? -eq 0 ]; then + FLAGS="--quiet" +fi # spatch only allows include directories with the syntax "-I include" # while gcc also allows "-Iinclude" and "-include include" @@ -51,9 +82,14 @@ if [ "$KBUILD_EXTMOD" != "" ] ; then OPTIONS="--patch $srctree $OPTIONS" fi -if [ ! -x "$SPATCH" ]; then - echo 'spatch is part of the Coccinelle project and is available at http://coccinelle.lip6.fr/' - exit 1 +# You can override by using SPFLAGS +if [ "$USE_JOBS" = "no" ]; then + trap kill_running SIGTERM SIGINT + declare -a SPATCH_PID +elif [ "$NPROC" != "1" ]; then + # Using 0 should work as well, refer to _SC_NPROCESSORS_ONLN use on + # https://github.com/rdicosmo/parmap/blob/master/setcore_stubs.c + OPTIONS="$OPTIONS --jobs $NPROC --chunksize 1" fi if [ "$MODE" = "" ] ; then @@ -72,7 +108,7 @@ if [ "$MODE" = "chain" ] ; then echo 'All available modes will be tried (in that order): patch, report, context, org' fi elif [ "$MODE" = "report" -o "$MODE" = "org" ] ; then - FLAGS="$FLAGS --no-show-diff" + FLAGS="--no-show-diff $FLAGS" fi if [ "$ONLINE" = "0" ] ; then @@ -82,7 +118,26 @@ if [ "$ONLINE" = "0" ] ; then echo '' fi -run_cmd() { +run_cmd_parmap() { + if [ $VERBOSE -ne 0 ] ; then + echo "Running ($NPROC in parallel): $@" + fi + if [ "$DEBUG_FILE" != "/dev/null" -a "$DEBUG_FILE" != "" ]; then + if [ -f $DEBUG_FILE ]; then + echo "Debug file $DEBUG_FILE exists, bailing" + exit + fi + else + DEBUG_FILE="/dev/null" + fi + $@ 2>$DEBUG_FILE + if [[ $? -ne 0 ]]; then + echo "coccicheck failed" + exit $? + fi +} + +run_cmd_old() { local i if [ $VERBOSE -ne 0 ] ; then echo "Running ($NPROC in parallel): $@" @@ -97,6 +152,14 @@ run_cmd() { wait } +run_cmd() { + if [ "$USE_JOBS" = "yes" ]; then + run_cmd_parmap $@ + else + run_cmd_old $@ + fi +} + kill_running() { for i in $(seq 0 $(( NPROC - 1 )) ); do if [ $VERBOSE -eq 2 ] ; then @@ -106,10 +169,23 @@ kill_running() { done } +# You can override heuristics with SPFLAGS, these must always go last +OPTIONS="$OPTIONS $SPFLAGS" + coccinelle () { COCCI="$1" OPT=`grep "Option" $COCCI | cut -d':' -f2` + REQ=`grep "Requires" $COCCI | cut -d':' -f2 | sed "s| ||"` + REQ_NUM=$(echo $REQ | ${DIR}/scripts/ld-version.sh) + if [ "$REQ_NUM" != "0" ] ; then + if [ "$SPATCH_VERSION_NUM" -lt "$REQ_NUM" ] ; then + echo "Skipping coccinele SmPL patch: $COCCI" + echo "You have coccinelle: $SPATCH_VERSION" + echo "This SmPL patch requires: $REQ" + return + fi + fi # The option '--parse-cocci' can be used to syntactically check the SmPL files. # diff --git a/scripts/coccinelle/free/devm_free.cocci b/scripts/coccinelle/free/devm_free.cocci index 3d9349012bb3..c990d2c7ee16 100644 --- a/scripts/coccinelle/free/devm_free.cocci +++ b/scripts/coccinelle/free/devm_free.cocci @@ -29,8 +29,24 @@ expression x; @@ ( + x = devm_kmalloc(...) +| + x = devm_kvasprintf(...) +| + x = devm_kasprintf(...) +| x = devm_kzalloc(...) | + x = devm_kmalloc_array(...) +| + x = devm_kcalloc(...) +| + x = devm_kstrdup(...) +| + x = devm_kmemdup(...) +| + x = devm_get_free_pages(...) +| x = devm_request_irq(...) | x = devm_ioremap(...) @@ -48,6 +64,16 @@ position p; ( * kfree@p(x) | +* kzfree@p(x) +| +* __krealloc@p(x, ...) +| +* krealloc@p(x, ...) +| +* free_pages@p(x, ...) +| +* free_page@p(x) +| * free_irq@p(x) | * iounmap@p(x) diff --git a/scripts/coccinelle/free/ifnullfree.cocci b/scripts/coccinelle/free/ifnullfree.cocci index 52bd235286fa..14a4cd98e83b 100644 --- a/scripts/coccinelle/free/ifnullfree.cocci +++ b/scripts/coccinelle/free/ifnullfree.cocci @@ -20,6 +20,8 @@ expression E; ( kfree(E); | + kzfree(E); +| debugfs_remove(E); | debugfs_remove_recursive(E); @@ -39,7 +41,7 @@ position p; @@ * if (E != NULL) -* \(kfree@p\|debugfs_remove@p\|debugfs_remove_recursive@p\| +* \(kfree@p\|kzfree@p\|debugfs_remove@p\|debugfs_remove_recursive@p\| * usb_free_urb@p\|kmem_cache_destroy@p\|mempool_destroy@p\| * dma_pool_destroy@p\)(E); diff --git a/scripts/coccinelle/free/kfree.cocci b/scripts/coccinelle/free/kfree.cocci index 577b78056990..ac438da4fd7b 100644 --- a/scripts/coccinelle/free/kfree.cocci +++ b/scripts/coccinelle/free/kfree.cocci @@ -20,7 +20,11 @@ expression E; position p1; @@ -kfree@p1(E) +( +* kfree@p1(E) +| +* kzfree@p1(E) +) @print expression@ constant char [] c; @@ -60,7 +64,11 @@ position ok; @@ while (1) { ... - kfree@ok(E) +( +* kfree@ok(E) +| +* kzfree@ok(E) +) ... when != break; when != goto l; when forall @@ -74,7 +82,11 @@ statement S; position free.p1!=loop.ok,p2!={print.p,sz.p}; @@ -kfree@p1(E,...) +( +* kfree@p1(E,...) +| +* kzfree@p1(E,...) +) ... ( iter(...,subE,...) S // no use diff --git a/scripts/coccinelle/free/kfreeaddr.cocci b/scripts/coccinelle/free/kfreeaddr.cocci index ce8aacc314cb..d46063b1db8b 100644 --- a/scripts/coccinelle/free/kfreeaddr.cocci +++ b/scripts/coccinelle/free/kfreeaddr.cocci @@ -16,7 +16,11 @@ identifier f; position p; @@ +( * kfree@p(&e->f) +| +* kzfree@p(&e->f) +) @script:python depends on org@ p << r.p; @@ -28,5 +32,5 @@ cocci.print_main("kfree",p) p << r.p; @@ -msg = "ERROR: kfree of structure field" +msg = "ERROR: invalid free of structure field" coccilib.report.print_report(p[0],msg) diff --git a/scripts/coccinelle/iterators/device_node_continue.cocci b/scripts/coccinelle/iterators/device_node_continue.cocci index 38ab744a4037..a36c16db171b 100644 --- a/scripts/coccinelle/iterators/device_node_continue.cocci +++ b/scripts/coccinelle/iterators/device_node_continue.cocci @@ -5,8 +5,11 @@ // Copyright: (C) 2015 Julia Lawall, Inria. GPLv2. // URL: http://coccinelle.lip6.fr/ // Options: --no-includes --include-headers +// Requires: 1.0.4 // Keywords: for_each_child_of_node, etc. +// This uses a conjunction, which requires at least coccinelle >= 1.0.4 + virtual patch virtual context virtual org diff --git a/scripts/coccinelle/misc/noderef.cocci b/scripts/coccinelle/misc/noderef.cocci index 80a831c91161..007f0de0c715 100644 --- a/scripts/coccinelle/misc/noderef.cocci +++ b/scripts/coccinelle/misc/noderef.cocci @@ -16,6 +16,7 @@ virtual patch @depends on patch@ expression *x; expression f; +expression i; type T; @@ @@ -30,15 +31,26 @@ f(...,(T)(x),...,sizeof( + *x ),...) | -f(...,sizeof(x),...,(T)( +f(...,sizeof( +- x ++ *x + ),...,(T)(x),...) +| +f(...,(T)(x),...,i*sizeof( - x + *x ),...) +| +f(...,i*sizeof( +- x ++ *x + ),...,(T)(x),...) ) @r depends on !patch@ expression *x; expression f; +expression i; position p; type T; @@ @@ -49,6 +61,10 @@ type T; *f(...,(T)(x),...,sizeof@p(x),...) | *f(...,sizeof@p(x),...,(T)(x),...) +| +*f(...,(T)(x),...,i*sizeof@p(x),...) +| +*f(...,i*sizeof@p(x),...,(T)(x),...) ) @script:python depends on org@ diff --git a/scripts/package/builddeb b/scripts/package/builddeb index 4d4418a8d54d..e1c09e2f9be7 100755 --- a/scripts/package/builddeb +++ b/scripts/package/builddeb @@ -26,6 +26,8 @@ create_package() { # Fix ownership and permissions chown -R root:root "$pdir" chmod -R go-w "$pdir" + # in case we are in a restrictive umask environment like 0077 + chmod -R a+rX "$pdir" # Create the package dpkg-gencontrol $forcearch -Vkernel:debarch="${debarch}" -p$pname -P"$pdir" @@ -238,7 +240,8 @@ maintainer="$name <$email>" # Try to determine distribution if [ -n "$KDEB_CHANGELOG_DIST" ]; then distribution=$KDEB_CHANGELOG_DIST -elif distribution=$(lsb_release -cs 2>/dev/null) && [ -n "$distribution" ]; then +# In some cases lsb_release returns the codename as n/a, which breaks dpkg-parsechangelog +elif distribution=$(lsb_release -cs 2>/dev/null) && [ -n "$distribution" ] && [ "$distribution" != "n/a" ]; then : # nothing to do in this case else distribution="unstable" @@ -322,12 +325,12 @@ fi # Build kernel header package (cd $srctree; find . -name Makefile\* -o -name Kconfig\* -o -name \*.pl) > "$objtree/debian/hdrsrcfiles" -if grep -q '^CONFIG_STACK_VALIDATION=y' $KCONFIG_CONFIG ; then - (cd $srctree; find tools/objtool -type f -executable) >> "$objtree/debian/hdrsrcfiles" -fi (cd $srctree; find arch/*/include include scripts -type f) >> "$objtree/debian/hdrsrcfiles" (cd $srctree; find arch/$SRCARCH -name module.lds -o -name Kbuild.platforms -o -name Platform) >> "$objtree/debian/hdrsrcfiles" (cd $srctree; find $(find arch/$SRCARCH -name include -o -name scripts -type d) -type f) >> "$objtree/debian/hdrsrcfiles" +if grep -q '^CONFIG_STACK_VALIDATION=y' $KCONFIG_CONFIG ; then + (cd $objtree; find tools/objtool -type f -executable) >> "$objtree/debian/hdrobjfiles" +fi (cd $objtree; find arch/$SRCARCH/include Module.symvers include scripts -type f) >> "$objtree/debian/hdrobjfiles" (cd $objtree; find scripts/gcc-plugins -name \*.so -o -name gcc-common.h) >> "$objtree/debian/hdrobjfiles" destdir=$kernel_headers_dir/usr/src/linux-headers-$version |