StarPU Handbook

Data Structures | Macros | Enumerations | Functions | Variables
Workers’ Properties

## Data Structures
struct | starpu_sched_ctx_iterator
struct | starpu_worker_collection

## Macros
#define | starpu_worker_get_id_check()
#define | STARPU_MAXNODES
#define | STARPU_MAXCPUS
#define | STARPU_MAXNUMANODES
#define | STARPU_NMAXWORKERS

## Enumerations
enum | starpu_node_kind { STARPU_UNUSED , STARPU_CPU_RAM , STARPU_CUDA_RAM , STARPU_OPENCL_RAM , STARPU_DISK_RAM , STARPU_MIC_RAM , STARPU_MPI_MS_RAM }
enum | starpu_worker_archtype { STARPU_CPU_WORKER , STARPU_CUDA_WORKER , STARPU_OPENCL_WORKER , STARPU_MIC_WORKER , STARPU_MPI_MS_WORKER , STARPU_ANY_WORKER }
enum | starpu_worker_collection_type { STARPU_WORKER_TREE , STARPU_WORKER_LIST }

## Functions
unsigned | starpu_worker_get_count (void)
unsigned | starpu_cpu_worker_get_count (void)
unsigned | starpu_cuda_worker_get_count (void)
unsigned | starpu_opencl_worker_get_count (void)
unsigned | starpu_mic_worker_get_count (void)
unsigned | starpu_mpi_ms_worker_get_count (void)
unsigned | starpu_mic_device_get_count (void)
int | starpu_worker_get_id (void)
unsigned | _starpu_worker_get_id_check (const char *f, int l)
int | starpu_worker_get_bindid (int workerid)
void | starpu_sched_find_all_worker_combinations (void)
enum starpu_worker_archtype | starpu_worker_get_type (int id)
int | starpu_worker_get_count_by_type (enum starpu_worker_archtype type)
unsigned | starpu_worker_get_ids_by_type (enum starpu_worker_archtype type, int *workerids, unsigned maxsize)
int | starpu_worker_get_by_type (enum starpu_worker_archtype type, int num)
int | starpu_worker_get_by_devid (enum starpu_worker_archtype type, int devid)
void | starpu_worker_get_name (int id, char *dst, size_t maxlen)
void | starpu_worker_display_names (FILE *output, enum starpu_worker_archtype type)
int | starpu_worker_get_devid (int id)
int | starpu_worker_get_mp_nodeid (int id)
struct starpu_tree * | starpu_workers_get_tree (void)
unsigned | starpu_worker_get_sched_ctx_list (int worker, unsigned **sched_ctx)
unsigned | starpu_worker_is_blocked_in_parallel (int workerid)
unsigned | starpu_worker_is_slave_somewhere (int workerid)
char * | starpu_worker_get_type_as_string (enum starpu_worker_archtype type)
int | starpu_bindid_get_workerids (int bindid, int **workerids)
int | starpu_worker_get_devids (enum starpu_worker_archtype type, int *devids, int num)
int | starpu_worker_get_stream_workerids (unsigned devid, int *workerids, enum starpu_worker_archtype type)
unsigned | starpu_worker_get_sched_ctx_id_stream (unsigned stream_workerid)
hwloc_cpuset_t | starpu_worker_get_hwloc_cpuset (int workerid)
hwloc_obj_t | starpu_worker_get_hwloc_obj (int workerid)
int | starpu_memory_node_get_devid (unsigned node)
unsigned | starpu_worker_get_local_memory_node (void)
unsigned | starpu_worker_get_memory_node (unsigned workerid)
unsigned | starpu_memory_nodes_get_count (void)
int | starpu_memory_node_get_name (unsigned node, char *name, size_t size)
int | starpu_memory_nodes_get_numa_count (void)
int | starpu_memory_nodes_numa_id_to_devid (int osid)
int | starpu_memory_nodes_numa_devid_to_id (unsigned id)
enum starpu_node_kind | starpu_node_get_kind (unsigned node)

## Variables
struct starpu_worker_collection | starpu_worker_list
struct starpu_worker_collection | starpu_worker_tree

## Scheduling operations
int | starpu_worker_sched_op_pending (void)
void | starpu_worker_relax_on (void)
void | starpu_worker_relax_off (void)
int | starpu_worker_get_relax_state (void)
void | starpu_worker_lock (int workerid)
int | starpu_worker_trylock (int workerid)
void | starpu_worker_unlock (int workerid)
void | starpu_worker_lock_self (void)
void | starpu_worker_unlock_self (void)
void | starpu_worker_set_going_to_sleep_callback (void(*callback)(unsigned workerid))
void | starpu_worker_set_waking_up_callback (void(*callback)(unsigned workerid))

## Detailed Description

## Data Structure Documentation

## ◆ starpu_sched_ctx_iterator

struct starpu_sched_ctx_iterator

Structure needed to iterate on the collection

Data Fields
int | cursor | The index of the current worker in the collection, needed when iterating on the collection.
void * | value |
void * | possible_value |
char | visited[STARPU_NMAXWORKERS] |
int | possibly_parallel |

## ◆ starpu_worker_collection

struct starpu_worker_collection

A scheduling context manages a collection of workers that can be memorized using different data structures. Thus, a generic structure is available in order to simplify the choice of its type. Only the list data structure is available but further data structures(like tree) implementations are foreseen.

### Data Fields
int * | workerids
void * | collection_private
unsigned | nworkers
void * | unblocked_workers
unsigned | nunblocked_workers
void * | masters
unsigned | nmasters
char | present [STARPU_NMAXWORKERS]
char | is_unblocked [STARPU_NMAXWORKERS]
char | is_master [STARPU_NMAXWORKERS]
enum starpu_worker_collection_type | type
unsigned(* | has_next )(struct starpu_worker_collection *workers, struct starpu_sched_ctx_iterator *it)
int(* | get_next )(struct starpu_worker_collection *workers, struct starpu_sched_ctx_iterator *it)
int(* | add )(struct starpu_worker_collection *workers, int worker)
int(* | remove )(struct starpu_worker_collection *workers, int worker)
void(* | init )(struct starpu_worker_collection *workers)
void(* | deinit )(struct starpu_worker_collection *workers)
void(* | init_iterator )(struct starpu_worker_collection *workers, struct starpu_sched_ctx_iterator *it)
void(* | init_iterator_for_parallel_tasks )(struct starpu_worker_collection *workers, struct starpu_sched_ctx_iterator *it, struct starpu_task *task)

#### Field Documentation

## ◆ workerids

int* starpu_worker_collection::workerids

The workerids managed by the collection

## ◆ nworkers

unsigned starpu_worker_collection::nworkers

The number of workers in the collection

## ◆ type

enum starpu_worker_collection_type starpu_worker_collection::type

The type of structure

## ◆ has_next

unsigned(* starpu_worker_collection::has_next) (struct starpu_worker_collection *workers, struct starpu_sched_ctx_iterator *it)

Check if there is another element in collection

## ◆ get_next

int(* starpu_worker_collection::get_next) (struct starpu_worker_collection *workers, struct starpu_sched_ctx_iterator *it)

Return the next element in the collection

## ◆ add

int(* starpu_worker_collection::add) (struct starpu_worker_collection *workers, int worker)

Add a new element in the collection

## ◆ remove

int(* starpu_worker_collection::remove) (struct starpu_worker_collection *workers, int worker)

Remove an element from the collection

## ◆ init

void(* starpu_worker_collection::init) (struct starpu_worker_collection *workers)

Initialize the collection

## ◆ deinit

void(* starpu_worker_collection::deinit) (struct starpu_worker_collection *workers)

Deinitialize the colection

## ◆ init_iterator

void(* starpu_worker_collection::init_iterator) (struct starpu_worker_collection *workers, struct starpu_sched_ctx_iterator *it)

Initialize the cursor if there is one

## Macro Definition Documentation

## ◆ starpu_worker_get_id_check

unsigned starpu_worker_get_id_check | ( |  | void | ) |

Similar to starpu_worker_get_id(), but abort when called from outside a worker (i.e. when starpu_worker_get_id() would return `-1`).

## ◆ STARPU_MAXNODES

#define STARPU_MAXNODES

Define the maximum number of memory nodes managed by StarPU. The default value can be modified at configure by using the option --enable-maxnodes. Reducing it allows to considerably reduce memory used by StarPU data structures.

## ◆ STARPU_MAXCPUS

#define STARPU_MAXCPUS

Define the maximum number of CPU workers managed by StarPU. The default value can be modified at configure by using the option --enable-maxcpus.

## ◆ STARPU_MAXNUMANODES

#define STARPU_MAXNUMANODES

Define the maximum number of NUMA nodes managed by StarPU. The default value can be modified at configure by using the option --enable-maxnumanodes.

## ◆ STARPU_NMAXWORKERS

#define STARPU_NMAXWORKERS

Define the maximum number of workers managed by StarPU.

## Enumeration Type Documentation

## ◆ starpu_node_kind

enum starpu_node_kind

Memory node Type

## ◆ starpu_worker_archtype

enum starpu_worker_archtype

Worker Architecture Type

The value 4 which was used by the driver SCC is no longer used as renumbering workers would make unusable old performance model files.

Enumerator
STARPU_CPU_WORKER | CPU core
STARPU_CUDA_WORKER | NVIDIA CUDA device
STARPU_OPENCL_WORKER | OpenCL device
STARPU_MIC_WORKER | Intel MIC device
STARPU_MPI_MS_WORKER | MPI Slave device
STARPU_ANY_WORKER | any worker, used in the hypervisor

## ◆ starpu_worker_collection_type

enum starpu_worker_collection_type

Types of structures the worker collection can implement

Enumerator
STARPU_WORKER_TREE | The collection is a tree
STARPU_WORKER_LIST | The collection is an array

## Function Documentation

## ◆ starpu_worker_get_count()

unsigned starpu_worker_get_count | ( | void |  | ) |

Return the number of workers (i.e. processing units executing StarPU tasks). The return value should be at most STARPU_NMAXWORKERS.

## ◆ starpu_cpu_worker_get_count()

unsigned starpu_cpu_worker_get_count | ( | void |  | ) |

Return the number of CPUs controlled by StarPU. The return value should be at most STARPU_MAXCPUS.

## ◆ starpu_cuda_worker_get_count()

unsigned starpu_cuda_worker_get_count | ( | void |  | ) |

Return the number of CUDA devices controlled by StarPU. The return value should be at most STARPU_MAXCUDADEVS.

## ◆ starpu_opencl_worker_get_count()

unsigned starpu_opencl_worker_get_count | ( | void |  | ) |

Return the number of OpenCL devices controlled by StarPU. The return value should be at most STARPU_MAXOPENCLDEVS.

## ◆ starpu_mic_worker_get_count()

unsigned starpu_mic_worker_get_count | ( | void |  | ) |

Return the number of MIC workers controlled by StarPU.

## ◆ starpu_mpi_ms_worker_get_count()

unsigned starpu_mpi_ms_worker_get_count | ( | void |  | ) |

Return the number of MPI Master Slave workers controlled by StarPU.

## ◆ starpu_mic_device_get_count()

unsigned starpu_mic_device_get_count | ( | void |  | ) |

Return the number of MIC devices controlled by StarPU. The return value should be at most STARPU_MAXMICDEVS.

## ◆ starpu_worker_get_id()

int starpu_worker_get_id | ( | void |  | ) |

Return the identifier of the current worker, i.e the one associated to the calling thread. The return value is either `-1` if the current context is not a StarPU worker (i.e. when called from the application outside a task or a callback), or an integer between `0` and starpu_worker_get_count() - `1`.

## ◆ starpu_worker_get_type()

enum starpu_worker_archtype starpu_worker_get_type | ( | int | id | ) |

Return the type of processing unit associated to the worker `id`. The worker identifier is a value returned by the function starpu_worker_get_id()). The return value indicates the architecture of the worker: STARPU_CPU_WORKER for a CPU core, STARPU_CUDA_WORKER for a CUDA device, and STARPU_OPENCL_WORKER for a OpenCL device. The return value for an invalid identifier is unspecified.

## ◆ starpu_worker_get_count_by_type()

int starpu_worker_get_count_by_type | ( | enum starpu_worker_archtype | type | ) |

Return the number of workers of `type`. A positive (or `NULL`) value is returned in case of success, `-EINVAL` indicates that `type` is not valid otherwise.

## ◆ starpu_worker_get_ids_by_type()

unsigned starpu_worker_get_ids_by_type | ( | enum starpu_worker_archtype | type,
|  | int * | workerids,
|  | unsigned | maxsize
| ) |  |

Get the list of identifiers of workers of `type`. Fill the array `workerids` with the identifiers of the `workers`. The argument `maxsize` indicates the size of the array `workerids`. The return value gives the number of identifiers that were put in the array. `-ERANGE` is returned is `maxsize` is lower than the number of workers with the appropriate type: in that case, the array is filled with the `maxsize` first elements. To avoid such overflows, the value of maxsize can be chosen by the means of the function starpu_worker_get_count_by_type(), or by passing a value greater or equal to STARPU_NMAXWORKERS.

## ◆ starpu_worker_get_by_type()

int starpu_worker_get_by_type | ( | enum starpu_worker_archtype | type,
|  | int | num
| ) |  |

Return the identifier of the `num` -th worker that has the specified `type`. If there is no such worker, -1 is returned.

## ◆ starpu_worker_get_by_devid()

int starpu_worker_get_by_devid | ( | enum starpu_worker_archtype | type,
|  | int | devid
| ) |  |

Return the identifier of the worker that has the specified `type` and device id `devid` (which may not be the n-th, if some devices are skipped for instance). If there is no such worker, `-1` is returned.

## ◆ starpu_worker_get_name()

void starpu_worker_get_name | ( | int | id,
|  | char * | dst,
|  | size_t | maxlen
| ) |  |

Get the name of the worker `id`. StarPU associates a unique human readable string to each processing unit. This function copies at most the `maxlen` first bytes of the unique string associated to the worker `id` into the `dst` buffer. The caller is responsible for ensuring that `dst` is a valid pointer to a buffer of `maxlen` bytes at least. Calling this function on an invalid identifier results in an unspecified behaviour.

## ◆ starpu_worker_display_names()

void starpu_worker_display_names | ( | FILE * | output,
|  | enum starpu_worker_archtype | type
| ) |  |

Display on `output` the list (if any) of all the workers of the given `type`.

## ◆ starpu_worker_get_devid()

int starpu_worker_get_devid | ( | int | id | ) |

Return the device id of the worker `id`. The worker should be identified with the value returned by the starpu_worker_get_id() function. In the case of a CUDA worker, this device identifier is the logical device identifier exposed by CUDA (used by the function `cudaGetDevice()` for instance). The device identifier of a CPU worker is the logical identifier of the core on which the worker was bound; this identifier is either provided by the OS or by the library `hwloc` in case it is available.

## ◆ starpu_worker_get_type_as_string()

char * starpu_worker_get_type_as_string | ( | enum starpu_worker_archtype | type | ) |

Return worker `type` as a string.

## ◆ starpu_worker_get_hwloc_cpuset()

hwloc_cpuset_t starpu_worker_get_hwloc_cpuset | ( | int | workerid | ) |

If StarPU was compiled with `hwloc` support, return a duplicate of the `hwloc` cpuset associated with the worker `workerid`. The returned cpuset is obtained from a `hwloc_bitmap_dup()` function call. It must be freed by the caller using `hwloc_bitmap_free()`.

## ◆ starpu_worker_get_hwloc_obj()

hwloc_obj_t starpu_worker_get_hwloc_obj | ( | int | workerid | ) |

If StarPU was compiled with `hwloc` support, return the `hwloc` object corresponding to the worker `workerid`.

## ◆ starpu_worker_get_local_memory_node()

unsigned starpu_worker_get_local_memory_node | ( | void |  | ) |

Return the memory node associated to the current worker

## ◆ starpu_worker_get_memory_node()

unsigned starpu_worker_get_memory_node | ( | unsigned | workerid | ) |

Return the identifier of the memory node associated to the worker identified by `workerid`.

## ◆ starpu_memory_nodes_get_count()

unsigned starpu_memory_nodes_get_count | ( | void |  | ) |

Return the number of memory nodes

## ◆ starpu_memory_node_get_name()

int starpu_memory_node_get_name | ( | unsigned | node,
|  | char * | name,
|  | size_t | size
| ) |  |

Return in `name` the name of a memory node (NUMA 0, CUDA 0, etc.) `size` is the size of the `name` array.

## ◆ starpu_memory_nodes_get_numa_count()

int starpu_memory_nodes_get_numa_count | ( | void |  | ) |

Return the number of NUMA nodes used by StarPU

## ◆ starpu_memory_nodes_numa_id_to_devid()

int starpu_memory_nodes_numa_id_to_devid | ( | int | osid | ) |

Return the identifier of the memory node associated to the NUMA node identified by `osid` by the Operating System.

## ◆ starpu_memory_nodes_numa_devid_to_id()

int starpu_memory_nodes_numa_devid_to_id | ( | unsigned | id | ) |

Return the Operating System identifier of the memory node whose StarPU identifier is `id`.

## ◆ starpu_node_get_kind()

enum starpu_node_kind starpu_node_get_kind | ( | unsigned | node | ) |

Return the type of `node` as defined by starpu_node_kind. For example, when defining a new data interface, this function should be used in the allocation function to determine on which device the memory needs to be allocated.

## ◆ starpu_worker_sched_op_pending()

int starpu_worker_sched_op_pending | ( | void |  | ) |

Return `!0` if current worker has a scheduling operation in progress, and `0` otherwise.

## ◆ starpu_worker_relax_on()

void starpu_worker_relax_on | ( | void |  | ) |

Allow other threads and workers to temporarily observe the current worker state, even though it is performing a scheduling operation. Must be called by a worker before performing a potentially blocking call such as acquiring a mutex other than its own sched_mutex. This function increases `state_relax_refcnt` from the current worker. No more than `UINT_MAX-1` nested starpu_worker_relax_on() calls should performed on the same worker. This function is automatically called by starpu_worker_lock() to relax the caller worker state while attempting to lock the target worker.

## ◆ starpu_worker_relax_off()

void starpu_worker_relax_off | ( | void |  | ) |

Must be called after a potentially blocking call is complete, to restore the relax state in place before the corresponding starpu_worker_relax_on(). Decreases `state_relax_refcnt`. Calls to starpu_worker_relax_on() and starpu_worker_relax_off() must be properly paired. This function is automatically called by starpu_worker_unlock() after the target worker has been unlocked.

## ◆ starpu_worker_get_relax_state()

int starpu_worker_get_relax_state | ( | void |  | ) |

Return `!0` if the current worker `state_relax_refcnt!=0` and `0` otherwise.

## ◆ starpu_worker_lock()

void starpu_worker_lock | ( | int | workerid | ) |

Acquire the sched mutex of `workerid`. If the caller is a worker, distinct from `workerid`, the caller worker automatically enters a relax state while acquiring the target worker lock.

## ◆ starpu_worker_trylock()

int starpu_worker_trylock | ( | int | workerid | ) |

Attempt to acquire the sched mutex of `workerid`. Returns `0` if successful, `!0` if `workerid` sched mutex is held or the corresponding worker is not in a relax state. If the caller is a worker, distinct from `workerid`, the caller worker automatically enters relax state if successfully acquiring the target worker lock.

## ◆ starpu_worker_unlock()

void starpu_worker_unlock | ( | int | workerid | ) |

Release the previously acquired sched mutex of `workerid`. Restore the relax state of the caller worker if needed.

## ◆ starpu_worker_lock_self()

void starpu_worker_lock_self | ( | void |  | ) |

Acquire the current worker sched mutex.

## ◆ starpu_worker_unlock_self()

void starpu_worker_unlock_self | ( | void |  | ) |

Release the current worker sched mutex.

## ◆ starpu_worker_set_going_to_sleep_callback()

void starpu_worker_set_going_to_sleep_callback | ( | void(*)(unsigned workerid) | callback | ) |

If StarPU was compiled with blocking drivers support and worker callbacks support enabled, allow to specify an external resource manager callback to be notified about workers going to sleep.

## ◆ starpu_worker_set_waking_up_callback()

void starpu_worker_set_waking_up_callback | ( | void(*)(unsigned workerid) | callback | ) |

If StarPU was compiled with blocking drivers support and worker callbacks support enabled, allow to specify an external resource manager callback to be notified about workers waking-up.