StarPU Internal Handbook

coherency.h
Go to the documentation of this file.
1/* StarPU --- Runtime system for heterogeneous multicore architectures.
2 *
3 * Copyright (C) 2008-2021 Université de Bordeaux, CNRS (LaBRI UMR 5800), Inria
4 *
5 * StarPU is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU Lesser General Public License as published by
7 * the Free Software Foundation; either version 2.1 of the License, or (at
8 * your option) any later version.
9 *
10 * StarPU is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
13 *
14 * See the GNU Lesser General Public License in COPYING.LGPL for more details.
15 */
16
17#ifndef __COHERENCY__H__
18#define __COHERENCY__H__
19
22#include <starpu.h>
23#include <common/config.h>
24
25#include <common/starpu_spinlock.h>
26#include <common/rwlock.h>
27#include <common/timing.h>
28#include <common/fxt.h>
29#include <common/list.h>
30
31#include <datawizard/interfaces/data_interface.h>
32#include <datawizard/datastats.h>
33#include <datawizard/memstats.h>
34#include <datawizard/data_request.h>
35
36 enum _starpu_cache_state
37{
38 STARPU_OWNER,
39 STARPU_SHARED,
40 STARPU_INVALID
41};
42
44 struct _starpu_data_replicate
45{
46 starpu_data_handle_t handle;
47
49 void *data_interface;
50
52 int refcnt;
53
54 char memory_node;
55
57 enum _starpu_cache_state state: 2;
58
61 unsigned relaxed_coherency:2;
62
64 unsigned initialized:1;
65
67 unsigned allocated:1;
73 unsigned automatically_allocated:1;
74
82 struct _starpu_data_request *request[STARPU_MAXNODES];
83
85 struct _starpu_mem_chunk * mc;
86};
87
88 struct _starpu_data_requester_prio_list;
89
90 struct _starpu_jobid_list
91{
92 unsigned long id;
93 struct _starpu_jobid_list *next;
94};
95
97 struct _starpu_task_wrapper_list
98{
99 struct starpu_task *task;
100 struct _starpu_task_wrapper_list *next;
101};
102
104 struct _starpu_task_wrapper_dlist
105{
106 struct starpu_task *task;
107 struct _starpu_task_wrapper_dlist *next;
108 struct _starpu_task_wrapper_dlist *prev;
109};
110
111 extern int _starpu_has_not_important_data;
112
113 typedef void (*_starpu_data_handle_unregister_hook)(starpu_data_handle_t);
114
116 struct _starpu_data_state
117{
118 int magic;
119 struct _starpu_data_requester_prio_list req_list;
123 unsigned refcnt;
125 unsigned unlocking_reqs;
129 enum starpu_data_access_mode current_mode;
131 struct _starpu_spinlock header_lock;
132
137 unsigned busy_count;
139 unsigned busy_waiting;
140 starpu_pthread_mutex_t busy_mutex;
141 starpu_pthread_cond_t busy_cond;
142
144 struct _starpu_data_state *root_handle;
145 struct _starpu_data_state *father_handle;
146 starpu_data_handle_t *active_children;
147 starpu_data_handle_t **active_readonly_children;
148 unsigned nactive_readonly_children;
150 unsigned nsiblings;
151 starpu_data_handle_t *siblings;
152 unsigned sibling_index;
153 unsigned depth;
156 starpu_data_handle_t children;
157 unsigned nchildren;
159 unsigned nplans;
161 struct starpu_codelet *switch_cl;
163 unsigned switch_cl_nparts;
170 unsigned partitioned;
172 unsigned readonly:1;
173
175 unsigned active:1;
176 unsigned active_ro:1;
177
179 struct _starpu_data_replicate per_node[STARPU_MAXNODES];
180 struct _starpu_data_replicate *per_worker;
181
182 struct starpu_data_interface_ops *ops;
183
185 uint32_t footprint;
186
188 int home_node;
189
191 uint32_t wt_mask;
192
195 unsigned is_not_important;
196
198 unsigned sequential_consistency;
200 unsigned initialized;
202 unsigned ooc;
203
206 starpu_pthread_mutex_t sequential_consistency_mutex;
207
212 enum starpu_data_access_mode last_submitted_mode;
213 struct starpu_task *last_sync_task;
214 struct _starpu_task_wrapper_dlist last_submitted_accessors;
215
222 unsigned last_submitted_ghost_sync_id_is_valid;
223 unsigned long last_submitted_ghost_sync_id;
224 struct _starpu_jobid_list *last_submitted_ghost_accessors_id;
225
227 struct _starpu_task_wrapper_list *post_sync_tasks;
228 unsigned post_sync_tasks_cnt;
229
230 /*
231 * Reductions
232 */
233
238 struct starpu_codelet *redux_cl;
239 struct starpu_codelet *init_cl;
240
244 unsigned reduction_refcnt;
245
249 struct _starpu_data_requester_prio_list reduction_req_list;
250
251 starpu_data_handle_t *reduction_tmp_handles;
252
254 struct _starpu_data_request *write_invalidation_req;
255
256 unsigned lazy_unregister;
257
258#ifdef STARPU_OPENMP
259 unsigned removed_from_context_hash;
260#endif
261
263 void *mpi_data;
264
265 _starpu_memory_stats_t memory_stats;
266
267 unsigned int mf_node; //XXX
268
270 _starpu_data_handle_unregister_hook unregister_hook;
271
272 struct starpu_arbiter *arbiter;
274 struct _starpu_data_requester_prio_list arbitered_req_list;
275
279 int last_locality;
280
283 unsigned dimensions;
284 int coordinates[5];
285
288 void *user_data;
289};
290
297 int _starpu_fetch_data_on_node(starpu_data_handle_t handle, int node, struct _starpu_data_replicate *replicate,
298 enum starpu_data_access_mode mode, unsigned detached, enum _starpu_is_prefetch is_prefetch, unsigned async,
299 void (*callback_func)(void *), void *callback_arg, int prio, const char *origin);
301 void _starpu_release_data_on_node(struct _starpu_data_state *state, uint32_t default_wt_mask,
302 struct _starpu_data_replicate *replicate);
303
304 void _starpu_update_data_state(starpu_data_handle_t handle,
305 struct _starpu_data_replicate *requesting_replicate,
306 enum starpu_data_access_mode mode);
307
308uint32_t _starpu_get_data_refcnt(struct _starpu_data_state *state, unsigned node);
309
310 size_t _starpu_data_get_size(starpu_data_handle_t handle);
311 size_t _starpu_data_get_alloc_size(starpu_data_handle_t handle);
312
313uint32_t _starpu_data_get_footprint(starpu_data_handle_t handle);
314
315 void __starpu_push_task_output(struct _starpu_job *j);
317 void _starpu_push_task_output(struct _starpu_job *j);
318
319 void _starpu_release_nowhere_task_output(struct _starpu_job *j);
320
321 struct _starpu_worker;
322STARPU_ATTRIBUTE_WARN_UNUSED_RESULT
323 int _starpu_fetch_task_input(struct starpu_task *task, struct _starpu_job *j, int async);
324 void _starpu_fetch_task_input_tail(struct starpu_task *task, struct _starpu_job *j, struct _starpu_worker *worker);
325 void _starpu_fetch_nowhere_task_input(struct _starpu_job *j);
326
327 int _starpu_select_src_node(struct _starpu_data_state *state, unsigned destination);
328 int _starpu_determine_request_path(starpu_data_handle_t handle,
329 int src_node, int dst_node,
330 enum starpu_data_access_mode mode, int max_len,
331 unsigned *src_nodes, unsigned *dst_nodes,
332 unsigned *handling_nodes, unsigned write_invalidation);
333
338 struct _starpu_data_request *_starpu_create_request_to_fetch_data(starpu_data_handle_t handle,
339 struct _starpu_data_replicate *dst_replicate,
340 enum starpu_data_access_mode mode, enum _starpu_is_prefetch is_prefetch,
341 unsigned async,
342 void (*callback_func)(void *), void *callback_arg, int prio, const char *origin);
343
344 void _starpu_redux_init_data_replicate(starpu_data_handle_t handle, struct _starpu_data_replicate *replicate, int workerid);
345 void _starpu_data_start_reduction_mode(starpu_data_handle_t handle);
346 void _starpu_data_end_reduction_mode(starpu_data_handle_t handle);
347 void _starpu_data_end_reduction_mode_terminate(starpu_data_handle_t handle);
348
349 void _starpu_data_set_unregister_hook(starpu_data_handle_t handle, _starpu_data_handle_unregister_hook func);
350
351#endif // __COHERENCY__H__
_starpu_data_state::busy_count
unsigned busy_count
Definition: coherency.h:137
_starpu_data_state::mpi_data
void * mpi_data
Definition: coherency.h:263
_starpu_data_state::post_sync_tasks
struct _starpu_task_wrapper_list * post_sync_tasks
Definition: coherency.h:227
_starpu_data_state::readonly
unsigned readonly
Definition: coherency.h:172
_starpu_data_state::redux_cl
struct starpu_codelet * redux_cl
Definition: coherency.h:238
_starpu_data_state::nactive_readonly_children
unsigned nactive_readonly_children
Definition: coherency.h:148
_starpu_data_state::switch_cl
struct starpu_codelet * switch_cl
Definition: coherency.h:161
_starpu_create_request_to_fetch_data
struct _starpu_data_request * _starpu_create_request_to_fetch_data(starpu_data_handle_t handle, struct _starpu_data_replicate *dst_replicate, enum starpu_data_access_mode mode, enum _starpu_is_prefetch is_prefetch, unsigned async, void(*callback_func)(void *), void *callback_arg, int prio, const char *origin)
_starpu_data_state::arbitered_req_list
struct _starpu_data_requester_prio_list arbitered_req_list
Definition: coherency.h:274
_starpu_data_state::unregister_hook
_starpu_data_handle_unregister_hook unregister_hook
Definition: coherency.h:270
_starpu_data_replicate::request
struct _starpu_data_request * request[STARPU_MAXNODES]
Definition: coherency.h:82
_starpu_data_state::write_invalidation_req
struct _starpu_data_request * write_invalidation_req
Definition: coherency.h:254
_starpu_data_replicate::state
enum _starpu_cache_state state
Definition: coherency.h:57
_starpu_data_state::active
unsigned active
Definition: coherency.h:175
_starpu_data_state::current_mode
enum starpu_data_access_mode current_mode
Definition: coherency.h:129
_starpu_data_state::unlocking_reqs
unsigned unlocking_reqs
Definition: coherency.h:125
_starpu_data_state::siblings
starpu_data_handle_t * siblings
Definition: coherency.h:151
_starpu_data_state::switch_cl_nparts
unsigned switch_cl_nparts
Definition: coherency.h:163
_starpu_data_state::father_handle
struct _starpu_data_state * father_handle
Definition: coherency.h:145
_starpu_data_state::children
starpu_data_handle_t children
Definition: coherency.h:156
_starpu_data_replicate::automatically_allocated
unsigned automatically_allocated
Definition: coherency.h:73
_starpu_data_state::ooc
unsigned ooc
Definition: coherency.h:202
_starpu_data_state::sequential_consistency_mutex
starpu_pthread_mutex_t sequential_consistency_mutex
Definition: coherency.h:206
_starpu_data_state::nplans
unsigned nplans
Definition: coherency.h:159
_starpu_data_replicate::mc
struct _starpu_mem_chunk * mc
Definition: coherency.h:85
_starpu_data_replicate::refcnt
int refcnt
Definition: coherency.h:52
_starpu_data_replicate::relaxed_coherency
unsigned relaxed_coherency
Definition: coherency.h:61
_starpu_data_state::last_submitted_mode
enum starpu_data_access_mode last_submitted_mode
Definition: coherency.h:212
_starpu_data_state::last_locality
int last_locality
Definition: coherency.h:279
_starpu_data_state::per_node
struct _starpu_data_replicate per_node[STARPU_MAXNODES]
Definition: coherency.h:179
_starpu_data_state::header_lock
struct _starpu_spinlock header_lock
Definition: coherency.h:131
_starpu_data_state::busy_waiting
unsigned busy_waiting
Definition: coherency.h:139
_starpu_data_state::partitioned
unsigned partitioned
Definition: coherency.h:170
_starpu_data_state::initialized
unsigned initialized
Definition: coherency.h:200
_starpu_data_replicate::data_interface
void * data_interface
Definition: coherency.h:49
_starpu_data_state::home_node
int home_node
Definition: coherency.h:188
_starpu_data_state::root_handle
struct _starpu_data_state * root_handle
Definition: coherency.h:144
_starpu_data_state::last_submitted_ghost_sync_id_is_valid
unsigned last_submitted_ghost_sync_id_is_valid
Definition: coherency.h:222
_starpu_release_data_on_node
void _starpu_release_data_on_node(struct _starpu_data_state *state, uint32_t default_wt_mask, struct _starpu_data_replicate *replicate)
_starpu_fetch_data_on_node
int _starpu_fetch_data_on_node(starpu_data_handle_t handle, int node, struct _starpu_data_replicate *replicate, enum starpu_data_access_mode mode, unsigned detached, enum _starpu_is_prefetch is_prefetch, unsigned async, void(*callback_func)(void *), void *callback_arg, int prio, const char *origin)
_starpu_push_task_output
void _starpu_push_task_output(struct _starpu_job *j)
_starpu_data_state::wt_mask
uint32_t wt_mask
Definition: coherency.h:191
_starpu_data_state::depth
unsigned depth
Definition: coherency.h:153
_starpu_data_replicate::initialized
unsigned initialized
Definition: coherency.h:64
_starpu_data_state::nsiblings
unsigned nsiblings
Definition: coherency.h:150
_starpu_data_state::reduction_req_list
struct _starpu_data_requester_prio_list reduction_req_list
Definition: coherency.h:249
_starpu_data_state::user_data
void * user_data
Definition: coherency.h:288
_starpu_data_state::refcnt
unsigned refcnt
Definition: coherency.h:123
_starpu_data_state::dimensions
unsigned dimensions
Definition: coherency.h:283
_starpu_data_state::active_readonly_children
starpu_data_handle_t ** active_readonly_children
Definition: coherency.h:147
_starpu_data_state::sequential_consistency
unsigned sequential_consistency
Definition: coherency.h:198
_starpu_data_state::reduction_refcnt
unsigned reduction_refcnt
Definition: coherency.h:244
_starpu_data_state::is_not_important
unsigned is_not_important
Definition: coherency.h:195
_starpu_data_replicate::allocated
unsigned allocated
Definition: coherency.h:67
_starpu_data_state::active_children
starpu_data_handle_t * active_children
Definition: coherency.h:146
_starpu_data_state::footprint
uint32_t footprint
Definition: coherency.h:185
_starpu_data_replicate
Definition: coherency.h:45
_starpu_data_state
Definition: coherency.h:117
_starpu_jobid_list
Definition: coherency.h:91
_starpu_task_wrapper_dlist
Definition: coherency.h:105
_starpu_task_wrapper_list
Definition: coherency.h:98
_starpu_is_prefetch
_starpu_is_prefetch
Definition: copy_driver.h:51
data_interface.h
data_request.h
datastats.h
fxt.h
list.h
memstats.h
rwlock.h
starpu_spinlock.h
_starpu_spinlock
Definition: starpu_spinlock.h:82
_starpu_job
Definition: jobs.h:79
_starpu_worker
Definition: workers.h:70
timing.h