likely() and unlikely()

Quote from : http://kernelnewbies.org/FAQ/LikelyUnlikely

1. bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx);
2. if (unlikely(!bvl)) {
3.   mempool_free(bio, bio_pool);
4.   bio = NULL;
5.   goto out;
6. }

In fact, these functions are hints for the compiler that allows it to correctly optimize the branch, by knowing which is the likeliest one. The definitions of these macros, found in include/linux/compiler.h are the following :

1. #define likely(x)       __builtin_expect(!!(x), 1)
2. #define unlikely(x)     __builtin_expect(!!(x), 0)

The GCC documentation explains the role of __builtin_expect() :

1.  -- Built-in Function: long __builtin_expect (long EXP, long C)
2.      You may use `__builtin_expect' to provide the compiler with branch
3.      prediction information.  In general, you should prefer to use
4.      actual profile feedback for this (`-fprofile-arcs'), as
5.      programmers are notoriously bad at predicting how their programs
6.      actually perform.  However, there are applications in which this
7.      data is hard to collect.
9.      The return value is the value of EXP, which should be an integral
10.      expression.  The value of C must be a compile-time constant.  The
11.      semantics of the built-in are that it is expected that EXP == C.
12.      For example:
14.           if (__builtin_expect (x, 0))
15.             foo ();
17.      would indicate that we do not expect to call `foo', since we
18.      expect `x' to be zero.  Since you are limited to integral
19.      expressions for EXP, you should use constructions such as
21.           if (__builtin_expect (ptr != NULL, 1))
22.             error ();
24.      when testing pointer or floating-point values.

How does it optimize things ?

It optimizes things by ordering the generated assembly code correctly, to optimize the usage of the processor pipeline. To do so, they arrange the code so that the likeliest branch is executed without performing any jmp instruction (which has the bad effect of flushing the processor pipeline).

To see how it works, let's compile the following simple C user space program with gcc -O2 :

1. #define likely(x)    __builtin_expect(!!(x), 1)
2. #define unlikely(x)  __builtin_expect(!!(x), 0)
4. int main(char *argv[], int argc)
5. {
6.    int a;
8.    /* Get the value from somewhere GCC can't optimize */
9.    a = atoi (argv[1]);
11.    if (unlikely (a == 2))
12.       a++;
13.    else
14.       a--;
16.    printf ("%d/n", a);
18.    return 0;
19. }

Now, disassemble the resulting binary using objdump -S (comments added by me) :

1. 080483b0 <main>:
2.  // Prologue
3.  80483b0:       55                      push   %ebp
4.  80483b1:       89 e5                   mov    %esp,%ebp
5.  80483b3:       50                      push   %eax
6.  80483b4:       50                      push   %eax
7.  80483b5:       83 e4 f0                and    $0xfffffff0,%esp
8.  //             Call atoi()
9.  80483b8:       8b 45 08                mov    0x8(%ebp),%eax
10.  80483bb:       83 ec 1c                sub    $0x1c,%esp
11.  80483be:       8b 48 04                mov    0x4(%eax),%ecx
12.  80483c1:       51                      push   %ecx
13.  80483c2:       e8 1d ff ff ff          call   80482e4 <atoi@plt>
14.  80483c7:       83 c4 10                add    $0x10,%esp
15.  //             Test the value
16.  80483ca:       83 f8 02                cmp    $0x2,%eax
17.  //             --------------------------------------------------------
18.  //             If 'a' equal to 2 (which is unlikely), then jump,
19.  //             otherwise continue directly, without jump, so that it
20.  //             doesn't flush the pipeline.
21.  //             --------------------------------------------------------
22.  80483cd:       74 12                   je     80483e1 <main+0x31>
23.  80483cf:       48                      dec    %eax
24.  //             Call printf
25.  80483d0:       52                      push   %edx
26.  80483d1:       52                      push   %edx
27.  80483d2:       50                      push   %eax
28.  80483d3:       68 c8 84 04 08          push   $0x80484c8
29.  80483d8:       e8 f7 fe ff ff          call   80482d4 <printf@plt>
30.  //             Return 0 and go out.
31.  80483dd:       31 c0                   xor    %eax,%eax
32.  80483df:       c9                      leave
33.  80483e0:       c3                      ret

Now, in the previous program, replace the unlikely() by a likely(), recompile it, and disassemble it again (again, comments added by me) :

1. 080483b0 <main>:
2.  //             Prologue
3.  80483b0:       55                      push   %ebp
4.  80483b1:       89 e5                   mov    %esp,%ebp
5.  80483b3:       50                      push   %eax
6.  80483b4:       50                      push   %eax
7.  80483b5:       83 e4 f0                and    $0xfffffff0,%esp
8.  //             Call atoi()
9.  80483b8:       8b 45 08                mov    0x8(%ebp),%eax
10.  80483bb:       83 ec 1c                sub    $0x1c,%esp
11.  80483be:       8b 48 04                mov    0x4(%eax),%ecx
12.  80483c1:       51                      push   %ecx
13.  80483c2:       e8 1d ff ff ff          call   80482e4 <atoi@plt>
14.  80483c7:       83 c4 10                add    $0x10,%esp
15.  //             --------------------------------------------------
16.  //             If 'a' equal 2 (which is likely), we will continue
17.  //             without branching, so without flusing the pipeline. The
18.  //             jump only occurs when a != 2, which is unlikely.
19.  //             ---------------------------------------------------
20.  80483ca:       83 f8 02                cmp    $0x2,%eax
21.  80483cd:       75 13                   jne    80483e2 <main+0x32>
22.  //             Here the a++ incrementation has been optimized by gcc
23.  80483cf:       b0 03                   mov    $0x3,%al
24.  //             Call printf()
25.  80483d1:       52                      push   %edx
26.  80483d2:       52                      push   %edx
27.  80483d3:       50                      push   %eax
28.  80483d4:       68 c8 84 04 08          push   $0x80484c8
29.  80483d9:       e8 f6 fe ff ff          call   80482d4 <printf@plt>
30.  //             Return 0 and go out.
31.  80483de:       31 c0                   xor    %eax,%eax
32.  80483e0:       c9                      leave
33.  80483e1:       c3                      ret

How should I use it ?

You should use it only in cases when the likeliest branch is very very very likely, or when the unlikeliest branch is very very very unlikely.