17#ifndef __STARPU_UTIL_H__
18#define __STARPU_UTIL_H__
32#ifdef STARPU_SIMGRID_MC
33#include <simgrid/modelchecker.h>
52#if defined __GNUC__ && defined __GNUC_MINOR__
53# define STARPU_GNUC_PREREQ(maj, min) \
54 ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
56# define STARPU_GNUC_PREREQ(maj, min) 0
64# define STARPU_UNLIKELY(expr) (__builtin_expect(!!(expr),0))
66# define STARPU_UNLIKELY(expr) (expr)
74# define STARPU_LIKELY(expr) (__builtin_expect(!!(expr),1))
76# define STARPU_LIKELY(expr) (expr)
83# define STARPU_ATTRIBUTE_UNUSED __attribute__((unused))
85# define STARPU_ATTRIBUTE_UNUSED
92# define STARPU_ATTRIBUTE_NORETURN __attribute__((noreturn))
94# define STARPU_ATTRIBUTE_NORETURN
101# define STARPU_ATTRIBUTE_INTERNAL __attribute__ ((visibility ("internal")))
103# define STARPU_ATTRIBUTE_INTERNAL
110# define STARPU_ATTRIBUTE_MALLOC __attribute__((malloc))
112# define STARPU_ATTRIBUTE_MALLOC
119# define STARPU_ATTRIBUTE_WARN_UNUSED_RESULT __attribute__((warn_unused_result))
121# define STARPU_ATTRIBUTE_WARN_UNUSED_RESULT
128# define STARPU_ATTRIBUTE_PURE __attribute__((pure))
130# define STARPU_ATTRIBUTE_PURE
137# define STARPU_ATTRIBUTE_ALIGNED(size) __attribute__((aligned(size)))
139# define STARPU_ATTRIBUTE_ALIGNED(size)
143# define STARPU_ATTRIBUTE_FORMAT(type, string, first) __attribute__((format(type, string, first)))
145# define STARPU_ATTRIBUTE_FORMAT(type, string, first)
150#if defined(c_plusplus) || defined(__cplusplus)
151# define STARPU_INLINE inline
152#elif defined(_MSC_VER) || defined(__HP_cc)
153# define STARPU_INLINE __inline
155# define STARPU_INLINE __inline__
158#if STARPU_GNUC_PREREQ(4, 3)
159# define STARPU_ATTRIBUTE_CALLOC_SIZE(num,size) __attribute__((alloc_size(num,size)))
160# define STARPU_ATTRIBUTE_ALLOC_SIZE(size) __attribute__((alloc_size(size)))
162# define STARPU_ATTRIBUTE_CALLOC_SIZE(num,size)
163# define STARPU_ATTRIBUTE_ALLOC_SIZE(size)
166#if STARPU_GNUC_PREREQ(3, 1) && !defined(BUILDING_STARPU) && !defined(STARPU_USE_DEPRECATED_API) && !defined(STARPU_USE_DEPRECATED_ONE_ZERO_API)
167#define STARPU_DEPRECATED __attribute__((__deprecated__))
169#define STARPU_DEPRECATED
172#if STARPU_GNUC_PREREQ(3,3)
173#define STARPU_WARN_UNUSED_RESULT __attribute__((__warn_unused_result__))
175#define STARPU_WARN_UNUSED_RESULT
178#define STARPU_BACKTRACE_LENGTH 32
180# define STARPU_DUMP_BACKTRACE() do { \
181 void *__ptrs[STARPU_BACKTRACE_LENGTH]; \
182 int __n = backtrace(__ptrs, STARPU_BACKTRACE_LENGTH); \
183 backtrace_symbols_fd(__ptrs, __n, 2); \
186# define STARPU_DUMP_BACKTRACE() do { } while (0)
189#ifdef STARPU_SIMGRID_MC
190#define STARPU_SIMGRID_ASSERT(x) MC_assert(!!(x))
192#define STARPU_SIMGRID_ASSERT(x)
199#ifdef STARPU_NO_ASSERT
200#define STARPU_ASSERT(x) do { if (0) { (void) (x); } } while(0)
202# if defined(__CUDACC__) || defined(STARPU_HAVE_WINDOWS)
203# define STARPU_ASSERT(x) do { if (STARPU_UNLIKELY(!(x))) { STARPU_DUMP_BACKTRACE(); STARPU_SIMGRID_ASSERT(x); *(int*)NULL = 0; } } while(0)
205# define STARPU_ASSERT(x) do { if (STARPU_UNLIKELY(!(x))) { STARPU_DUMP_BACKTRACE(); STARPU_SIMGRID_ASSERT(x); assert(x); } } while (0)
209#ifdef STARPU_NO_ASSERT
210#define STARPU_ASSERT_ACCESSIBLE(x) do { if (0) { (void) (x); } } while(0)
212#define STARPU_ASSERT_ACCESSIBLE(ptr) do { volatile char __c STARPU_ATTRIBUTE_UNUSED = *(char*) (ptr); } while(0)
220#ifdef STARPU_NO_ASSERT
221#define STARPU_ASSERT_MSG(x, msg, ...) do { if (0) { (void) (x); (void) msg; } } while(0)
223# if defined(__CUDACC__) || defined(STARPU_HAVE_WINDOWS)
224# define STARPU_ASSERT_MSG(x, msg, ...) do { if (STARPU_UNLIKELY(!(x))) { STARPU_DUMP_BACKTRACE(); fprintf(stderr, "\n[starpu][%s][assert failure] " msg "\n\n", __starpu_func__, ## __VA_ARGS__); STARPU_SIMGRID_ASSERT(x); *(int*)NULL = 0; }} while(0)
226# define STARPU_ASSERT_MSG(x, msg, ...) do { if (STARPU_UNLIKELY(!(x))) { STARPU_DUMP_BACKTRACE(); fprintf(stderr, "\n[starpu][%s][assert failure] " msg "\n\n", __starpu_func__, ## __VA_ARGS__); STARPU_SIMGRID_ASSERT(x); assert(x); } } while(0)
231# ifdef __clang_analyzer__
232# define _starpu_abort() exit(42)
234# define _starpu_abort() *(volatile int*)NULL = 0
237# define _starpu_abort() abort()
243#define STARPU_ABORT() do { \
244 STARPU_DUMP_BACKTRACE(); \
245 fprintf(stderr, "[starpu][abort][%s()@%s:%d]\n", __starpu_func__, __FILE__, __LINE__); \
254#define STARPU_ABORT_MSG(msg, ...) do { \
255 STARPU_DUMP_BACKTRACE(); \
256 fprintf(stderr, "[starpu][abort][%s()@%s:%d] " msg "\n", __starpu_func__, __FILE__, __LINE__, ## __VA_ARGS__); \
261 #undef STARPU_HAVE_STRERROR_R
264#if defined(STARPU_HAVE_STRERROR_R)
265#if (! defined(__GLIBC__) || !__GLIBC__) || ((_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && (! defined(_GNU_SOURCE)))
267# define starpu_strerror_r(errnum, buf, buflen) \
270 int _ret = strerror_r((errnum), (buf), (buflen)); \
271 STARPU_ASSERT(_ret == 0); \
276# define starpu_strerror_r(errnum, buf, buflen) \
279 char * const _user_buf = (buf); \
280 const size_t _user_buflen = (buflen); \
282 STARPU_ASSERT((buf) != NULL); \
283 char * _tmp_buf = strerror_r((errnum), _user_buf, _user_buflen); \
284 if (_tmp_buf != _user_buf) \
286 if (_user_buflen > 0) \
288 strncpy(_user_buf, _tmp_buf, _user_buflen-1); \
289 _user_buf[_user_buflen-1] = '\0'; \
301#if defined(STARPU_HAVE_STRERROR_R)
302# define STARPU_CHECK_RETURN_VALUE(err, message, ...) {if (STARPU_UNLIKELY(err != 0)) { \
303 char xmessage[256]; starpu_strerror_r(-err, xmessage, 256); \
304 fprintf(stderr, "[starpu] Unexpected value: <%d:%s> returned for " message "\n", err, xmessage, ## __VA_ARGS__); \
307# define STARPU_CHECK_RETURN_VALUE(err, message, ...) {if (STARPU_UNLIKELY(err != 0)) { \
308 fprintf(stderr, "[starpu] Unexpected value: <%d> returned for " message "\n", err, ## __VA_ARGS__); \
316#if defined(STARPU_HAVE_STRERROR_R)
317# define STARPU_CHECK_RETURN_VALUE_IS(err, value, message, ...) {if (STARPU_UNLIKELY(err != value)) { \
318 char xmessage[256]; starpu_strerror_r(-err, xmessage, 256); \
319 fprintf(stderr, "[starpu] Unexpected value: <%d!=%d:%s> returned for " message "\n", err, value, xmessage, ## __VA_ARGS__); \
322# define STARPU_CHECK_RETURN_VALUE_IS(err, value, message, ...) {if (STARPU_UNLIKELY(err != value)) { \
323 fprintf(stderr, "[starpu] Unexpected value: <%d != %d> returned for " message "\n", err, value, ## __VA_ARGS__); \
333#if defined(__i386__) || defined(__x86_64__)
334static __starpu_inline
unsigned _starpu_cmpxchg(
unsigned *ptr,
unsigned old,
unsigned next)
336 __asm__ __volatile__(
"lock cmpxchgl %2,%1":
"+a" (old),
"+m" (*ptr) :
"q" (next) :
"memory");
339#define STARPU_HAVE_CMPXCHG
340static __starpu_inline
unsigned _starpu_xchg(
unsigned *ptr,
unsigned next)
343 __asm__ __volatile__(
"xchgl %1,%0":
"+m" (*ptr),
"+q" (next) : :
"memory");
346#define STARPU_HAVE_XCHG
348static __starpu_inline uint32_t _starpu_cmpxchg32(uint32_t *ptr, uint32_t old, uint32_t next)
350 __asm__ __volatile__(
"lock cmpxchgl %2,%1":
"+a" (old),
"+m" (*ptr) :
"q" (next) :
"memory");
353#define STARPU_HAVE_CMPXCHG32
354static __starpu_inline uint32_t _starpu_xchg32(uint32_t *ptr, uint32_t next)
357 __asm__ __volatile__(
"xchgl %1,%0":
"+m" (*ptr),
"+q" (next) : :
"memory");
360#define STARPU_HAVE_XCHG32
363static __starpu_inline
unsigned long _starpu_cmpxchgl(
unsigned long *ptr,
unsigned long old,
unsigned long next)
365 __asm__ __volatile__(
"lock cmpxchgl %2,%1":
"+a" (old),
"+m" (*ptr) :
"q" (next) :
"memory");
368#define STARPU_HAVE_CMPXCHGL
369static __starpu_inline
unsigned long _starpu_xchgl(
unsigned long *ptr,
unsigned long next)
372 __asm__ __volatile__(
"xchgl %1,%0":
"+m" (*ptr),
"+q" (next) : :
"memory");
375#define STARPU_HAVE_XCHGL
378#if defined(__x86_64__)
379static __starpu_inline
unsigned long _starpu_cmpxchgl(
unsigned long *ptr,
unsigned long old,
unsigned long next)
381 __asm__ __volatile__(
"lock cmpxchgq %2,%1":
"+a" (old),
"+m" (*ptr) :
"q" (next) :
"memory");
384#define STARPU_HAVE_CMPXCHGL
385static __starpu_inline
unsigned long _starpu_xchgl(
unsigned long *ptr,
unsigned long next)
388 __asm__ __volatile__(
"xchgq %1,%0":
"+m" (*ptr),
"+q" (next) : :
"memory");
391#define STARPU_HAVE_XCHGL
395static __starpu_inline uint64_t _starpu_cmpxchg64(uint64_t *ptr, uint64_t old, uint64_t next)
397 uint32_t next_hi = next >> 32;
398 uint32_t next_lo = next & 0xfffffffful;
399 __asm__ __volatile__(
"lock cmpxchg8b %1":
"+A" (old),
"+m" (*ptr) :
"c" (next_hi),
"b" (next_lo) :
"memory");
402#define STARPU_HAVE_CMPXCHG64
405#if defined(__x86_64__)
406static __starpu_inline uint64_t _starpu_cmpxchg64(uint64_t *ptr, uint64_t old, uint64_t next)
408 __asm__ __volatile__(
"lock cmpxchgq %2,%1":
"+a" (old),
"+m" (*ptr) :
"q" (next) :
"memory");
411#define STARPU_HAVE_CMPXCHG64
412static __starpu_inline uint64_t _starpu_xchg64(uint64_t *ptr, uint64_t next)
415 __asm__ __volatile__(
"xchgq %1,%0":
"+m" (*ptr),
"+q" (next) : :
"memory");
418#define STARPU_HAVE_XCHG64
423#define STARPU_ATOMIC_SOMETHING(name,expr) \
424static __starpu_inline unsigned starpu_atomic_##name(unsigned *ptr, unsigned value) \
426 unsigned old, next; \
431 if (_starpu_cmpxchg(ptr, old, next) == old) \
436#define STARPU_ATOMIC_SOMETHINGL(name,expr) \
437static __starpu_inline unsigned long starpu_atomic_##name##l(unsigned long *ptr, unsigned long value) \
439 unsigned long old, next; \
444 if (_starpu_cmpxchgl(ptr, old, next) == old) \
449#define STARPU_ATOMIC_SOMETHING64(name,expr) \
450static __starpu_inline uint64_t starpu_atomic_##name##64(uint64_t *ptr, uint64_t value) \
452 uint64_t old, next; \
457 if (_starpu_cmpxchg64(ptr, old, next) == old) \
464#ifdef STARPU_HAVE_SYNC_FETCH_AND_ADD
465#define STARPU_ATOMIC_ADD(ptr, value) (__sync_fetch_and_add ((ptr), (value)) + (value))
466#define STARPU_ATOMIC_ADDL(ptr, value) (__sync_fetch_and_add ((ptr), (value)) + (value))
467#define STARPU_ATOMIC_ADD64(ptr, value) (__sync_fetch_and_add ((ptr), (value)) + (value))
469#if defined(STARPU_HAVE_CMPXCHG)
470STARPU_ATOMIC_SOMETHING(add, old + value)
471#define STARPU_ATOMIC_ADD(ptr, value) starpu_atomic_add(ptr, value)
473#if defined(STARPU_HAVE_CMPXCHGL)
474STARPU_ATOMIC_SOMETHINGL(add, old + value)
475#define STARPU_ATOMIC_ADDL(ptr, value) starpu_atomic_addl(ptr, value)
477#if defined(STARPU_HAVE_CMPXCHG64)
478STARPU_ATOMIC_SOMETHING64(add, old + value)
479#define STARPU_ATOMIC_ADD64(ptr, value) starpu_atomic_add64(ptr, value)
483#ifdef STARPU_HAVE_SYNC_FETCH_AND_OR
484#define STARPU_ATOMIC_OR(ptr, value) (__sync_fetch_and_or ((ptr), (value)))
485#define STARPU_ATOMIC_ORL(ptr, value) (__sync_fetch_and_or ((ptr), (value)))
486#define STARPU_ATOMIC_OR64(ptr, value) (__sync_fetch_and_or ((ptr), (value)))
488#if defined(STARPU_HAVE_CMPXCHG)
489STARPU_ATOMIC_SOMETHING(or, old | value)
490#define STARPU_ATOMIC_OR(ptr, value) starpu_atomic_or(ptr, value)
492#if defined(STARPU_HAVE_CMPXCHGL)
493STARPU_ATOMIC_SOMETHINGL(or, old | value)
494#define STARPU_ATOMIC_ORL(ptr, value) starpu_atomic_orl(ptr, value)
496#if defined(STARPU_HAVE_CMPXCHG64)
497STARPU_ATOMIC_SOMETHING64(or, old | value)
498#define STARPU_ATOMIC_OR64(ptr, value) starpu_atomic_or64(ptr, value)
503#ifdef STARPU_HAVE_SYNC_BOOL_COMPARE_AND_SWAP
504#define STARPU_BOOL_COMPARE_AND_SWAP(ptr, old, value) (__sync_bool_compare_and_swap ((ptr), (old), (value)))
505#define STARPU_BOOL_COMPARE_AND_SWAP32(ptr, old, value) STARPU_BOOL_COMPARE_AND_SWAP(ptr, old, value)
506#define STARPU_BOOL_COMPARE_AND_SWAP64(ptr, old, value) STARPU_BOOL_COMPARE_AND_SWAP(ptr, old, value)
508#ifdef STARPU_HAVE_CMPXCHG
509#define STARPU_BOOL_COMPARE_AND_SWAP(ptr, old, value) (_starpu_cmpxchg((ptr), (old), (value)) == (old))
511#ifdef STARPU_HAVE_CMPXCHG32
512#define STARPU_BOOL_COMPARE_AND_SWAP32(ptr, old, value) (_starpu_cmpxchg32((ptr), (old), (value)) == (old))
514#ifdef STARPU_HAVE_CMPXCHG64
515#define STARPU_BOOL_COMPARE_AND_SWAP64(ptr, old, value) (_starpu_cmpxchg64((ptr), (old), (value)) == (old))
519#if UINTPTR_MAX == UINT64_MAX
520#define STARPU_BOOL_COMPARE_AND_SWAP_PTR(ptr, old, value) STARPU_BOOL_COMPARE_AND_SWAP64(ptr, old, value)
522#define STARPU_BOOL_COMPARE_AND_SWAP_PTR(ptr, old, value) STARPU_BOOL_COMPARE_AND_SWAP32(ptr, old, value)
526#ifdef STARPU_HAVE_SYNC_VAL_COMPARE_AND_SWAP
527#define STARPU_VAL_COMPARE_AND_SWAP(ptr, old, value) (__sync_val_compare_and_swap ((ptr), (old), (value)))
528#define STARPU_VAL_COMPARE_AND_SWAP32(ptr, old, value) STARPU_VAL_COMPARE_AND_SWAP(ptr, old, value)
529#define STARPU_VAL_COMPARE_AND_SWAP64(ptr, old, value) STARPU_VAL_COMPARE_AND_SWAP(ptr, old, value)
531#ifdef STARPU_HAVE_CMPXCHG
532#define STARPU_VAL_COMPARE_AND_SWAP(ptr, old, value) (_starpu_cmpxchg((ptr), (old), (value)))
534#ifdef STARPU_HAVE_CMPXCHG32
535#define STARPU_VAL_COMPARE_AND_SWAP32(ptr, old, value) (_starpu_cmpxchg32((ptr), (old), (value)))
537#ifdef STARPU_HAVE_CMPXCHG64
538#define STARPU_VAL_COMPARE_AND_SWAP64(ptr, old, value) (_starpu_cmpxchg64((ptr), (old), (value)))
542#if UINTPTR_MAX == UINT64_MAX
543#define STARPU_VAL_COMPARE_AND_SWAP_PTR(ptr, old, value) STARPU_VAL_COMPARE_AND_SWAP64(ptr, old, value)
545#define STARPU_VAL_COMPARE_AND_SWAP_PTR(ptr, old, value) STARPU_VAL_COMPARE_AND_SWAP32(ptr, old, value)
548#ifdef STARPU_HAVE_ATOMIC_EXCHANGE_N
549#define STARPU_VAL_EXCHANGE(ptr, value) (__atomic_exchange_n((ptr), (value), __ATOMIC_SEQ_CST))
550#define STARPU_VAL_EXCHANGEL(ptr, value) STARPU_VAL_EXCHANGE((ptr) (value))
551#define STARPU_VAL_EXCHANGE32(ptr, value) STARPU_VAL_EXCHANGE((ptr) (value))
552#define STARPU_VAL_EXCHANGE64(ptr, value) STARPU_VAL_EXCHANGE((ptr) (value))
554#ifdef STARPU_HAVE_XCHG
555#define STARPU_VAL_EXCHANGE(ptr, value) (_starpu_xchg((ptr), (value)))
557#ifdef STARPU_HAVE_XCHGL
558#define STARPU_VAL_EXCHANGEL(ptr, value) (_starpu_xchgl((ptr), (value)))
560#ifdef STARPU_HAVE_XCHG32
561#define STARPU_VAL_EXCHANGE32(ptr, value) (_starpu_xchg32((ptr), (value)))
563#ifdef STARPU_HAVE_XCHG64
564#define STARPU_VAL_EXCHANGE64(ptr, value) (_starpu_xchg64((ptr), (value)))
569#ifdef STARPU_HAVE_SYNC_LOCK_TEST_AND_SET
570#define STARPU_TEST_AND_SET(ptr, value) (__sync_lock_test_and_set ((ptr), (value)))
571#define STARPU_RELEASE(ptr) (__sync_lock_release ((ptr)))
572#elif defined(STARPU_HAVE_XCHG)
573#define STARPU_TEST_AND_SET(ptr, value) (_starpu_xchg((ptr), (value)))
574#define STARPU_RELEASE(ptr) (_starpu_xchg((ptr), 0))
577#ifdef STARPU_HAVE_SYNC_SYNCHRONIZE
578#define STARPU_SYNCHRONIZE() __sync_synchronize()
579#elif defined(__i386__)
580#define STARPU_SYNCHRONIZE() __asm__ __volatile__("lock; addl $0,0(%%esp)" ::: "memory")
581#elif defined(__KNC__) || defined(__KNF__)
582#define STARPU_SYNCHRONIZE() __asm__ __volatile__("lock; addl $0,0(%%rsp)" ::: "memory")
583#elif defined(__x86_64__)
584#define STARPU_SYNCHRONIZE() __asm__ __volatile__("mfence" ::: "memory")
585#elif defined(__ppc__) || defined(__ppc64__)
586#define STARPU_SYNCHRONIZE() __asm__ __volatile__("sync" ::: "memory")
592#if defined(__x86_64__)
593#define STARPU_RMB() __asm__ __volatile__("lfence" ::: "memory")
594#elif defined(__aarch64__)
595#define STARPU_RMB() __asm__ __volatile__("dsb ld" ::: "memory")
597#define STARPU_RMB() STARPU_SYNCHRONIZE()
603#if defined(__x86_64__)
604#define STARPU_WMB() __asm__ __volatile__("sfence" ::: "memory")
605#elif defined(__aarch64__)
606#define STARPU_WMB() __asm__ __volatile__("dsb st" ::: "memory")
608#define STARPU_WMB() STARPU_SYNCHRONIZE()
611#if defined(__i386__) || defined(__x86_64__)
612#define STARPU_CACHELINE_SIZE 64
613#elif defined(__ppc__) || defined(__ppc64__) || defined(__ia64__)
614#define STARPU_CACHELINE_SIZE 128
615#elif defined(__s390__) || defined(__s390x__)
616#define STARPU_CACHELINE_SIZE 256
619#define STARPU_CACHELINE_SIZE 1024
624#include <sys/types.h>
630#if !defined(_MSC_VER) || defined(BUILDING_STARPU)
633#if !defined(STARPU_HAVE_STRUCT_TIMESPEC) || (defined(_MSC_VER) && _MSC_VER < 1900)
635#ifndef STARPU_TIMESPEC_DEFINED
636#define STARPU_TIMESPEC_DEFINED 1
645#if defined(__MINGW32__) || defined(__CYGWIN__)