bionic: pthread: use private futexes by default for mutexes and condvars

Private futexes are a recent kernel addition: faster futexes that cannot be
shared between processes. This patch uses them by default, unless the PROCESS_SHARED
attribute flag is used when creating a mutex and/or conditional variable.

Also introduces pthread_condattr_init/destroy/setpshared/getpshared.

Change-Id: I3a0e2116f467072b046524cb5babc00e41057a53

libc/arch-arm/bionic/atomics_arm.S

@@ -43,6 +43,19 @@
 #define FUTEX_WAIT 0
 #define FUTEX_WAKE 1
 
+/* Private futexes belong to a single address space and cannot be
+ * shared among processes. They are however significantly faster to
+ * operate than standard futexes.
+ */
+.global __futex_wait_private
+.type __futex_wait_private, %function
+.global __futex_wake_private
+.type __futex_wake_private, %function
+
+#define FUTEX_PRIVATE_FLAG  128
+#define FUTEX_WAIT_PRIVATE  (FUTEX_WAIT|FUTEX_PRIVATE_FLAG)
+#define FUTEX_WAKE_PRIVATE  (FUTEX_WAKE|FUTEX_PRIVATE_FLAG)
+
 #if 1
    .equ     kernel_cmpxchg, 0xFFFF0FC0
    .equ     kernel_atomic_base, 0xFFFF0FFF
@@ -185,6 +198,28 @@ __futex_wake:
     ldmia   sp!, {r4, r7}
     bx      lr
 
+__futex_wait_private:
+    .fnstart
+    stmdb   sp!, {r4, r7}
+    .save   {r4, r7}
+    mov     r3, r2
+    mov     r2, r1
+    mov     r1, #FUTEX_WAIT_PRIVATE
+    ldr     r7, =__NR_futex
+    swi     #0
+    ldmia   sp!, {r4, r7}
+    bx      lr
+    .fnend
+
+__futex_wake_private:
+    stmdb   sp!, {r4, r7}
+    mov     r2, r1
+    mov     r1, #FUTEX_WAKE_PRIVATE
+    ldr     r7, =__NR_futex
+    swi     #0
+    ldmia   sp!, {r4, r7}
+    bx      lr
+
 #else
 
 __futex_wait:
@@ -200,4 +235,17 @@ __futex_wake:
     swi     #__NR_futex
     bx      lr
 
+__futex_wait_private:
+    mov     r3, r2
+    mov     r2, r1
+    mov     r1, #FUTEX_WAIT_PRIVATE
+    swi     #__NR_futex
+    bx      lr
+
+__futex_wake_private:
+    mov     r2, r1
+    mov     r1, #FUTEX_WAKE_PRIVATE
+    swi     #__NR_futex
+    bx      lr
+
 #endif

libc/arch-sh/bionic/atomics_sh.c

@@ -98,3 +98,21 @@ int __futex_wake(volatile void *ftx, int count)
 {
     return futex(ftx, FUTEX_WAKE, count, NULL, NULL, 0);
 }
+
+/* Private futexes belong to a single address space and cannot be
+ * shared among processes. They are however significantly faster to
+ * operate than standard futexes.
+ */
+#define FUTEX_PRIVATE_FLAG  128
+#define FUTEX_WAIT_PRIVATE  (FUTEX_WAIT|FUTEX_PRIVATE_FLAG)
+#define FUTEX_WAKE_PRIVATE  (FUTEX_WAKE|FUTEX_PRIVATE_FLAG)
+
+int __futex_wait_private(volatile void *ftx, int val, const struct timespec *timeout)
+{
+    return futex(ftx, FUTEX_WAIT_PRIVATE, val, (void *)timeout, NULL, 0);
+}
+
+int __futex_wake_private(volatile void *ftx, int count)
+{
+    return futex(ftx, FUTEX_WAKE_PRIVATE, count, NULL, NULL, 0);
+}

libc/arch-x86/bionic/atomics_x86.c

@@ -60,6 +60,43 @@ int __futex_wake(volatile void *ftx, int count)
     return ret;
 }
 
+/* Private futexes belong to a single address space and cannot be
+ * shared among processes. They are however significantly faster to
+ * operate than standard futexes.
+ */
+#define FUTEX_PRIVATE_FLAG  128
+#define FUTEX_WAIT_PRIVATE  (FUTEX_WAIT|FUTEX_PRIVATE_FLAG)
+#define FUTEX_WAKE_PRIVATE  (FUTEX_WAKE|FUTEX_PRIVATE_FLAG)
+
+int __futex_wait_private(volatile void *ftx, int val)
+{
+    int ret;
+    asm volatile (
+        "int $0x80;"
+        : "=a" (ret)
+        : "0" (FUTEX_SYSCALL),
+          "b" (ftx),
+          "c" (FUTEX_WAIT_PRIVATE),
+          "d" (val),
+          "S" (0)
+    );
+    return ret;
+}
+
+int __futex_wake_private(volatile void *ftx, int count)
+{
+    int ret;
+    asm volatile (
+        "int $0x80;"
+        : "=a" (ret)
+        : "0" (FUTEX_SYSCALL),
+          "b" (ftx),
+          "c" (FUTEX_WAKE_PRIVATE),
+          "d" (count)
+    );
+    return ret;
+}
+
 int __atomic_cmpxchg(int old, int new, volatile int* addr) {
     int xchg;
     asm volatile (

libc/docs/CHANGES.TXT

@@ -80,6 +80,16 @@ Differences between current and Android 2.1:
   an unknown domain name. Due to an initialization bug, a random DNS search
   list was generated for each thread is net.dns.search is not defined.
 
+- <pthread.h>: Add pthread_condattr_init/destroy/setpshared/getpshared.
+  Also modify mutex and condvar implementation to use private futexes by
+  default, unless PROCESS_SHARED is specified in the init attributes.
+
+  Private futexes are limited to a single address space and can't be shared
+  among processes. However they are much faster to wake/wait for. This should
+  speed up mutex and condvar operations.
+
+  NOTE: PROCESS_SHARED mutexes are still NOT robust (see note below).
+
 -------------------------------------------------------------------------------
 Differences between Android 2.1 and 2.0.1:
 

libc/include/pthread.h

@@ -165,6 +165,11 @@ int pthread_mutex_unlock(pthread_mutex_t *mutex);
 int pthread_mutex_trylock(pthread_mutex_t *mutex);
 int pthread_mutex_timedlock(pthread_mutex_t *mutex, struct timespec*  ts);
 
+int pthread_condattr_init(pthread_condattr_t *attr);
+int pthread_condattr_setpshared(pthread_condattr_t *attr, int pshared);
+int pthread_condattr_getpshared(pthread_condattr_t *attr, int *pshared);
+int pthread_condattr_destroy(pthread_condattr_t *attr);
+
 int pthread_cond_init(pthread_cond_t *cond,
                       const pthread_condattr_t *attr);
 int pthread_cond_destroy(pthread_cond_t *cond);