StarPU Internal Handbook
workers.h
1 /* StarPU --- Runtime system for heterogeneous multicore architectures.
2  *
3  * Copyright (C) 2011-2017 Inria
4  * Copyright (C) 2008-2019 Université de Bordeaux
5  * Copyright (C) 2010-2019 CNRS
6  * Copyright (C) 2013 Thibaut Lambert
7  * Copyright (C) 2016 Uppsala University
8  *
9  * StarPU is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU Lesser General Public License as published by
11  * the Free Software Foundation; either version 2.1 of the License, or (at
12  * your option) any later version.
13  *
14  * StarPU is distributed in the hope that it will be useful, but
15  * WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
17  *
18  * See the GNU Lesser General Public License in COPYING.LGPL for more details.
19  */
20 
21 #ifndef __WORKERS_H__
22 #define __WORKERS_H__
23 
25 /* @{ */
26 
27 #include <limits.h>
28 
29 #include <starpu.h>
30 #include <common/config.h>
31 #include <common/timing.h>
32 #include <common/fxt.h>
33 #include <common/thread.h>
34 #include <core/jobs.h>
35 #include <core/perfmodel/perfmodel.h>
36 #include <core/sched_policy.h>
37 #include <core/topology.h>
38 #include <core/errorcheck.h>
39 #include <core/sched_ctx.h>
40 #include <core/sched_ctx_list.h>
41 #include <core/simgrid.h>
42 #ifdef STARPU_HAVE_HWLOC
43 #include <hwloc.h>
44 #endif
45 
46 #include <core/drivers.h>
47 #include <drivers/cuda/driver_cuda.h>
48 #include <drivers/opencl/driver_opencl.h>
49 
50 #ifdef STARPU_USE_MIC
51 #include <drivers/mic/driver_mic_source.h>
52 #endif /* STARPU_USE_MIC */
53 
54 #ifdef STARPU_USE_SCC
55 #include <drivers/scc/driver_scc_source.h>
56 #endif
57 
58 #ifdef STARPU_USE_MPI_MASTER_SLAVE
59 #include <drivers/mpi/driver_mpi_source.h>
60 #endif
61 
62 #include <drivers/cpu/driver_cpu.h>
63 
64 #include <datawizard/datawizard.h>
65 
66 #include <starpu_parameters.h>
67 
68 #define STARPU_MAX_PIPELINE 4
69 
70 enum initialization { UNINITIALIZED = 0, CHANGING, INITIALIZED };
71 
72 struct _starpu_ctx_change_list;
73 
75 LIST_TYPE(_starpu_worker,
76  struct _starpu_machine_config *config;
77  starpu_pthread_mutex_t mutex;
78  enum starpu_worker_archtype arch;
79  uint32_t worker_mask;
80  struct starpu_perfmodel_arch perf_arch;
81  starpu_pthread_t worker_thread;
82  unsigned devid;
83  unsigned subworkerid;
84  int bindid;
85  int workerid;
89  starpu_pthread_cond_t started_cond;
90  starpu_pthread_cond_t ready_cond;
91  unsigned memory_node;
92  unsigned numa_memory_node;
97  starpu_pthread_cond_t sched_cond;
98  starpu_pthread_mutex_t sched_mutex;
99  unsigned state_relax_refcnt;
100 #ifdef STARPU_SPINLOCK_CHECK
101  const char *relax_on_file;
102  int relax_on_line;
103  const char *relax_on_func;
104  const char *relax_off_file;
105  int relax_off_line;
106  const char *relax_off_func;
107 #endif
125  starpu_pthread_t thread_changing_ctx;
133  struct _starpu_ctx_change_list ctx_change_list;
134  struct starpu_task_list local_tasks;
135  struct starpu_task **local_ordered_tasks;
139  struct starpu_task *current_task;
140  struct starpu_task *current_tasks[STARPU_MAX_PIPELINE];
141 #ifdef STARPU_SIMGRID
142  starpu_pthread_wait_t wait;
143 #endif
144 
145  struct timespec cl_start;
146  struct timespec cl_end;
147  unsigned char first_task;
148  unsigned char ntasks;
149  unsigned char pipeline_length;
150  unsigned char pipeline_stuck;
151  struct _starpu_worker_set *set;
152  unsigned worker_is_running;
153  unsigned worker_is_initialized;
154  enum _starpu_worker_status status;
155  unsigned state_keep_awake;
156  char name[64];
157  char short_name[32];
158  unsigned run_by_starpu;
159  struct _starpu_driver_ops *driver_ops;
160 
161  struct _starpu_sched_ctx_list *sched_ctx_list;
162  int tmp_sched_ctx;
163  unsigned nsched_ctxs;
164  struct _starpu_barrier_counter tasks_barrier;
166  unsigned has_prev_init;
168  unsigned removed_from_ctx[STARPU_NMAX_SCHED_CTXS+1];
169 
170  unsigned spinning_backoff ;
174  struct starpu_task *task_transferring;
180  unsigned shares_tasks_lists[STARPU_NMAX_SCHED_CTXS+1];
181 
182  unsigned poped_in_ctx[STARPU_NMAX_SCHED_CTXS+1];
188  unsigned reverse_phase[2];
189 
190  unsigned pop_ctx_priority;
193  struct _starpu_sched_ctx *stream_ctx;
194 
195 #ifdef __GLIBC__
196  cpu_set_t cpu_set;
197 #endif /* __GLIBC__ */
198 #ifdef STARPU_HAVE_HWLOC
199  hwloc_bitmap_t hwloc_cpu_set;
200  hwloc_obj_t hwloc_obj;
201 #endif
202 );
203 
205 {
206  struct starpu_perfmodel_arch perf_arch;
207  uint32_t worker_mask;
208  int worker_size;
209  unsigned memory_node;
210  int combined_workerid[STARPU_NMAXWORKERS];
211 #ifdef STARPU_USE_MP
212  int count;
213  starpu_pthread_mutex_t count_mutex;
214 #endif
215 
216 #ifdef __GLIBC__
217  cpu_set_t cpu_set;
218 #endif /* __GLIBC__ */
219 #ifdef STARPU_HAVE_HWLOC
220  hwloc_bitmap_t hwloc_cpu_set;
221 #endif
222 };
223 
229 {
230  starpu_pthread_mutex_t mutex;
231  starpu_pthread_t worker_thread;
232  unsigned nworkers;
233  unsigned started;
234  void *retval;
235  struct _starpu_worker *workers;
236  starpu_pthread_cond_t ready_cond;
237  unsigned set_is_initialized;
238 };
239 
240 #ifdef STARPU_USE_MPI_MASTER_SLAVE
241 extern struct _starpu_worker_set mpi_worker_set[STARPU_MAXMPIDEVS];
242 #endif
243 
245 {
247  unsigned nworkers;
248 
251 
252  unsigned nsched_ctxs;
253 
254 #ifdef STARPU_HAVE_HWLOC
255 
256  hwloc_topology_t hwtopology;
257 #endif
258 
259  struct starpu_tree *tree;
260 
264  unsigned nhwcpus;
265 
269  unsigned nhwpus;
270 
274  unsigned nhwcudagpus;
275 
279  unsigned nhwopenclgpus;
280 
284  unsigned nhwscc;
285 
289  unsigned nhwmpi;
290 
292  unsigned ncpus;
293 
295  unsigned ncudagpus;
296  unsigned nworkerpercuda;
297  int cuda_th_per_stream;
298  int cuda_th_per_dev;
299 
301  unsigned nopenclgpus;
302 
304  unsigned nsccdevices;
305 
307  unsigned nmpidevices;
308  unsigned nhwmpidevices;
309 
310  unsigned nhwmpicores[STARPU_MAXMPIDEVS];
311  unsigned nmpicores[STARPU_MAXMPIDEVS];
312 
315  unsigned nhwmicdevices;
316  unsigned nmicdevices;
317 
318  unsigned nhwmiccores[STARPU_MAXMICDEVS];
319  unsigned nmiccores[STARPU_MAXMICDEVS];
320 
328  unsigned workers_bindid[STARPU_NMAXWORKERS];
329 
336  unsigned workers_cuda_gpuid[STARPU_NMAXWORKERS];
337 
344  unsigned workers_opencl_gpuid[STARPU_NMAXWORKERS];
345 
346  /*** Indicates the successive MIC devices that should be used
347  * by the MIC driver. It is either filled according to the
348  * user's explicit parameters (from starpu_conf) or according
349  * to the STARPU_WORKERS_MICID env. variable. Otherwise, they
350  * are taken in ID order. */
361  unsigned workers_scc_deviceid[STARPU_NMAXWORKERS];
362 
363  unsigned workers_mpi_ms_deviceid[STARPU_NMAXWORKERS];
364 };
365 
367 {
368  struct _starpu_machine_topology topology;
369 
370 #ifdef STARPU_HAVE_HWLOC
371  int cpu_depth;
372  int pu_depth;
373 #endif
374 
377  char currently_bound[STARPU_NMAXWORKERS];
378  char currently_shared[STARPU_NMAXWORKERS];
379 
382 
385 
388 
391 
394 
407 
410  struct _starpu_worker workers[STARPU_NMAXWORKERS];
411 
414  struct _starpu_combined_worker combined_workers[STARPU_NMAX_COMBINEDWORKERS];
415 
417  struct
418  {
419  int *workerids;
420  unsigned nworkers;
421  } *bindid_workers;
422  unsigned nbindid;
427  uint32_t worker_mask;
428 
430  struct starpu_conf conf;
431 
433  unsigned running;
434 
435  int disable_kernels;
436 
440 
442  struct _starpu_sched_ctx sched_ctxs[STARPU_NMAX_SCHED_CTXS+1];
443 
445  unsigned submitting;
446 
447  int watchdog_ok;
448 
449  starpu_pthread_mutex_t submitted_mutex;
450 };
451 
452 extern int _starpu_worker_parallel_blocks;
453 
454 extern struct _starpu_machine_config _starpu_config STARPU_ATTRIBUTE_INTERNAL;
455 extern int _starpu_keys_initialized STARPU_ATTRIBUTE_INTERNAL;
456 extern starpu_pthread_key_t _starpu_worker_key STARPU_ATTRIBUTE_INTERNAL;
457 extern starpu_pthread_key_t _starpu_worker_set_key STARPU_ATTRIBUTE_INTERNAL;
458 
460 void _starpu_set_argc_argv(int *argc, char ***argv);
461 int *_starpu_get_argc();
462 char ***_starpu_get_argv();
463 
465 void _starpu_conf_check_environment(struct starpu_conf *conf);
466 
468 void _starpu_may_pause(void);
469 
471 static inline unsigned _starpu_machine_is_running(void)
472 {
473  unsigned ret;
474  /* running is just protected by a memory barrier */
475  STARPU_RMB();
476 
477  ANNOTATE_HAPPENS_AFTER(&_starpu_config.running);
478  ret = _starpu_config.running;
479  ANNOTATE_HAPPENS_BEFORE(&_starpu_config.running);
480  return ret;
481 }
482 
483 
485 void _starpu_worker_init(struct _starpu_worker *workerarg, struct _starpu_machine_config *pconfig);
486 
488 uint32_t _starpu_worker_exists(struct starpu_task *);
489 
491 uint32_t _starpu_can_submit_cuda_task(void);
492 
494 uint32_t _starpu_can_submit_cpu_task(void);
495 
497 uint32_t _starpu_can_submit_opencl_task(void);
498 
500 uint32_t _starpu_can_submit_scc_task(void);
501 
504 unsigned _starpu_worker_can_block(unsigned memnode, struct _starpu_worker *worker);
505 
509 void _starpu_block_worker(int workerid, starpu_pthread_cond_t *cond, starpu_pthread_mutex_t *mutex);
510 
512 void _starpu_driver_start(struct _starpu_worker *worker, unsigned fut_key, unsigned sync);
514 void _starpu_worker_start(struct _starpu_worker *worker, unsigned fut_key, unsigned sync);
515 
516 static inline unsigned _starpu_worker_get_count(void)
517 {
518  return _starpu_config.topology.nworkers;
519 }
520 #define starpu_worker_get_count _starpu_worker_get_count
521 
525 static inline void _starpu_set_local_worker_key(struct _starpu_worker *worker)
526 {
527  STARPU_ASSERT(_starpu_keys_initialized);
528  STARPU_PTHREAD_SETSPECIFIC(_starpu_worker_key, worker);
529 }
530 
533 static inline struct _starpu_worker *_starpu_get_local_worker_key(void)
534 {
535  if (!_starpu_keys_initialized)
536  return NULL;
537  return (struct _starpu_worker *) STARPU_PTHREAD_GETSPECIFIC(_starpu_worker_key);
538 }
539 
543 static inline void _starpu_set_local_worker_set_key(struct _starpu_worker_set *worker)
544 {
545  STARPU_ASSERT(_starpu_keys_initialized);
546  STARPU_PTHREAD_SETSPECIFIC(_starpu_worker_set_key, worker);
547 }
548 
552 {
553  if (!_starpu_keys_initialized)
554  return NULL;
555  return (struct _starpu_worker_set *) STARPU_PTHREAD_GETSPECIFIC(_starpu_worker_set_key);
556 }
557 
560 static inline struct _starpu_worker *_starpu_get_worker_struct(unsigned id)
561 {
562  STARPU_ASSERT(id < starpu_worker_get_count());
563  return &_starpu_config.workers[id];
564 }
565 
568 static inline struct _starpu_sched_ctx *_starpu_get_sched_ctx_struct(unsigned id)
569 {
570  return (id > STARPU_NMAX_SCHED_CTXS) ? NULL : &_starpu_config.sched_ctxs[id];
571 }
572 
573 struct _starpu_combined_worker *_starpu_get_combined_worker_struct(unsigned id);
574 
578 {
579  return &_starpu_config;
580 }
581 
583 static inline int _starpu_get_disable_kernels(void)
584 {
585  return _starpu_config.disable_kernels;
586 }
587 
589 static inline enum _starpu_worker_status _starpu_worker_get_status(int workerid)
590 {
591  return _starpu_config.workers[workerid].status;
592 }
593 
596 static inline void _starpu_worker_set_status(int workerid, enum _starpu_worker_status status)
597 {
598  _starpu_config.workers[workerid].status = status;
599 }
600 
602 static inline struct _starpu_sched_ctx* _starpu_get_initial_sched_ctx(void)
603 {
604  return &_starpu_config.sched_ctxs[STARPU_GLOBAL_SCHED_CTX];
605 }
606 
607 int starpu_worker_get_nids_by_type(enum starpu_worker_archtype type, int *workerids, int maxsize);
608 
611 int starpu_worker_get_nids_ctx_free_by_type(enum starpu_worker_archtype type, int *workerids, int maxsize);
612 
613 static inline unsigned _starpu_worker_mutex_is_sched_mutex(int workerid, starpu_pthread_mutex_t *mutex)
614 {
615  struct _starpu_worker *w = _starpu_get_worker_struct(workerid);
616  return &w->sched_mutex == mutex;
617 }
618 
619 static inline int _starpu_worker_get_nsched_ctxs(int workerid)
620 {
621  return _starpu_config.workers[workerid].nsched_ctxs;
622 }
623 
625 static inline unsigned _starpu_get_nsched_ctxs(void)
626 {
627  /* topology.nsched_ctxs may be increased asynchronously in sched_ctx_create */
628  STARPU_RMB();
629  return _starpu_config.topology.nsched_ctxs;
630 }
631 
633 static inline int _starpu_worker_get_id(void)
634 {
635  struct _starpu_worker * worker;
636 
637  worker = _starpu_get_local_worker_key();
638  if (worker)
639  {
640  return worker->workerid;
641  }
642  else
643  {
644  /* there is no worker associated to that thread, perhaps it is
645  * a thread from the application or this is some SPU worker */
646  return -1;
647  }
648 }
649 #define starpu_worker_get_id _starpu_worker_get_id
650 
653 static inline unsigned __starpu_worker_get_id_check(const char *f, int l)
654 {
655  (void) l;
656  (void) f;
657  int id = starpu_worker_get_id();
658  STARPU_ASSERT_MSG(id>=0, "%s:%d Cannot be called from outside a worker\n", f, l);
659  return id;
660 }
661 #define _starpu_worker_get_id_check(f,l) __starpu_worker_get_id_check(f,l)
662 
663 void _starpu_worker_set_stream_ctx(unsigned workerid, struct _starpu_sched_ctx *sched_ctx);
664 
665 struct _starpu_sched_ctx* _starpu_worker_get_ctx_stream(unsigned stream_workerid);
666 
672 static inline void _starpu_worker_request_blocking_in_parallel(struct _starpu_worker * const worker)
673 {
674  _starpu_worker_parallel_blocks = 1;
675  /* flush pending requests to start on a fresh transaction epoch */
676  while (worker->state_unblock_in_parallel_req)
677  STARPU_PTHREAD_COND_WAIT(&worker->sched_cond, &worker->sched_mutex);
678 
679  /* announce blocking intent */
680  STARPU_ASSERT(worker->block_in_parallel_ref_count < UINT_MAX);
681  worker->block_in_parallel_ref_count++;
682 
683  if (worker->block_in_parallel_ref_count == 1)
684  {
685  /* only the transition from 0 to 1 triggers the block_in_parallel_req */
686 
687  STARPU_ASSERT(!worker->state_blocked_in_parallel);
688  STARPU_ASSERT(!worker->state_block_in_parallel_req);
689  STARPU_ASSERT(!worker->state_block_in_parallel_ack);
690  STARPU_ASSERT(!worker->state_unblock_in_parallel_req);
691  STARPU_ASSERT(!worker->state_unblock_in_parallel_ack);
692 
693  /* trigger the block_in_parallel_req */
694  worker->state_block_in_parallel_req = 1;
695  STARPU_PTHREAD_COND_BROADCAST(&worker->sched_cond);
696 #ifdef STARPU_SIMGRID
697  starpu_pthread_queue_broadcast(&_starpu_simgrid_task_queue[worker->workerid]);
698 #endif
699 
700  /* wait for block_in_parallel_req to be processed */
701  while (!worker->state_block_in_parallel_ack)
702  STARPU_PTHREAD_COND_WAIT(&worker->sched_cond, &worker->sched_mutex);
703 
704  STARPU_ASSERT(worker->block_in_parallel_ref_count >= 1);
705  STARPU_ASSERT(worker->state_block_in_parallel_req);
706  STARPU_ASSERT(worker->state_blocked_in_parallel);
707 
708  /* reset block_in_parallel_req state flags */
709  worker->state_block_in_parallel_req = 0;
710  worker->state_block_in_parallel_ack = 0;
711 
712  /* broadcast block_in_parallel_req state flags reset */
713  STARPU_PTHREAD_COND_BROADCAST(&worker->sched_cond);
714  }
715 }
716 
721 static inline void _starpu_worker_request_unblocking_in_parallel(struct _starpu_worker * const worker)
722 {
723  /* flush pending requests to start on a fresh transaction epoch */
724  while (worker->state_block_in_parallel_req)
725  STARPU_PTHREAD_COND_WAIT(&worker->sched_cond, &worker->sched_mutex);
726 
727  /* unblocking may be requested unconditionnally
728  * thus, check is unblocking is really needed */
729  if (worker->state_blocked_in_parallel)
730  {
731  if (worker->block_in_parallel_ref_count == 1)
732  {
733  /* only the transition from 1 to 0 triggers the unblock_in_parallel_req */
734 
735  STARPU_ASSERT(!worker->state_block_in_parallel_req);
736  STARPU_ASSERT(!worker->state_block_in_parallel_ack);
737  STARPU_ASSERT(!worker->state_unblock_in_parallel_req);
738  STARPU_ASSERT(!worker->state_unblock_in_parallel_ack);
739 
740  /* trigger the unblock_in_parallel_req */
741  worker->state_unblock_in_parallel_req = 1;
742  STARPU_PTHREAD_COND_BROADCAST(&worker->sched_cond);
743 
744  /* wait for the unblock_in_parallel_req to be processed */
745  while (!worker->state_unblock_in_parallel_ack)
746  STARPU_PTHREAD_COND_WAIT(&worker->sched_cond, &worker->sched_mutex);
747 
748  STARPU_ASSERT(worker->state_unblock_in_parallel_req);
749  STARPU_ASSERT(!worker->state_blocked_in_parallel);
750 
751  /* reset unblock_in_parallel_req state flags */
752  worker->state_unblock_in_parallel_req = 0;
753  worker->state_unblock_in_parallel_ack = 0;
754 
755  /* broadcast unblock_in_parallel_req state flags reset */
756  STARPU_PTHREAD_COND_BROADCAST(&worker->sched_cond);
757  }
758 
759  /* announce unblocking complete */
760  STARPU_ASSERT(worker->block_in_parallel_ref_count > 0);
761  worker->block_in_parallel_ref_count--;
762  }
763 }
764 
770 static inline void _starpu_worker_process_block_in_parallel_requests(struct _starpu_worker * const worker)
771 {
772  while (worker->state_block_in_parallel_req)
773  {
774  STARPU_ASSERT(!worker->state_blocked_in_parallel);
775  STARPU_ASSERT(!worker->state_block_in_parallel_ack);
776  STARPU_ASSERT(!worker->state_unblock_in_parallel_req);
777  STARPU_ASSERT(!worker->state_unblock_in_parallel_ack);
778  STARPU_ASSERT(worker->block_in_parallel_ref_count > 0);
779 
780  /* enter effective blocked state */
781  worker->state_blocked_in_parallel = 1;
782 
783  /* notify block_in_parallel_req processing */
784  worker->state_block_in_parallel_ack = 1;
785  STARPU_PTHREAD_COND_BROADCAST(&worker->sched_cond);
786 
787  /* block */
788  while (!worker->state_unblock_in_parallel_req)
789  STARPU_PTHREAD_COND_WAIT(&worker->sched_cond, &worker->sched_mutex);
790 
791  STARPU_ASSERT(worker->state_blocked_in_parallel);
792  STARPU_ASSERT(!worker->state_block_in_parallel_req);
793  STARPU_ASSERT(!worker->state_block_in_parallel_ack);
794  STARPU_ASSERT(!worker->state_unblock_in_parallel_ack);
795  STARPU_ASSERT(worker->block_in_parallel_ref_count > 0);
796 
797  /* leave effective blocked state */
798  worker->state_blocked_in_parallel = 0;
799 
800  /* notify unblock_in_parallel_req processing */
801  worker->state_unblock_in_parallel_ack = 1;
802  STARPU_PTHREAD_COND_BROADCAST(&worker->sched_cond);
803  }
804 }
805 
822 #ifdef STARPU_SPINLOCK_CHECK
823 static inline void __starpu_worker_enter_sched_op(struct _starpu_worker * const worker, const char*file, int line, const char* func)
824 #else
825 static inline void _starpu_worker_enter_sched_op(struct _starpu_worker * const worker)
826 #endif
827 {
828  STARPU_ASSERT(!worker->state_sched_op_pending);
830  {
831  /* process pending block requests before entering a sched_op region */
833  while (worker->state_changing_ctx_notice)
834  {
835  STARPU_PTHREAD_COND_WAIT(&worker->sched_cond, &worker->sched_mutex);
836 
837  /* new block requests may have been triggered during the wait,
838  * need to check again */
840  }
841  }
842  else
843  {
844  /* if someone observed the worker state since the last call, postpone block request
845  * processing for one sched_op turn more, because the observer will not have seen
846  * new block requests between its observation and now.
847  *
848  * however, the worker still has to wait for context change operations to complete
849  * before entering sched_op again*/
850  while (worker->state_changing_ctx_notice)
851  {
852  STARPU_PTHREAD_COND_WAIT(&worker->sched_cond, &worker->sched_mutex);
853  }
854  }
855 
856  /* no block request and no ctx change ahead,
857  * enter sched_op */
858  worker->state_sched_op_pending = 1;
860  worker->state_relax_refcnt = 0;
861 #ifdef STARPU_SPINLOCK_CHECK
862  worker->relax_on_file = file;
863  worker->relax_on_line = line;
864  worker->relax_on_func = func;
865 #endif
866 }
867 #ifdef STARPU_SPINLOCK_CHECK
868 #define _starpu_worker_enter_sched_op(worker) __starpu_worker_enter_sched_op((worker), __FILE__, __LINE__, __starpu_func__)
869 #endif
870 
876 #ifdef STARPU_SPINLOCK_CHECK
877 static inline void __starpu_worker_leave_sched_op(struct _starpu_worker * const worker, const char*file, int line, const char* func)
878 #else
879 static inline void _starpu_worker_leave_sched_op(struct _starpu_worker * const worker)
880 #endif
881 {
882  STARPU_ASSERT(worker->state_sched_op_pending);
883  worker->state_relax_refcnt = 1;
884 #ifdef STARPU_SPINLOCK_CHECK
885  worker->relax_off_file = file;
886  worker->relax_off_line = line;
887  worker->relax_off_func = func;
888 #endif
889  worker->state_sched_op_pending = 0;
890  STARPU_PTHREAD_COND_BROADCAST(&worker->sched_cond);
892 }
893 #ifdef STARPU_SPINLOCK_CHECK
894 #define _starpu_worker_leave_sched_op(worker) __starpu_worker_leave_sched_op((worker), __FILE__, __LINE__, __starpu_func__)
895 #endif
896 
897 static inline int _starpu_worker_sched_op_pending(void)
898 {
899  int workerid = starpu_worker_get_id();
900  if (workerid == -1)
901  return 0;
902  struct _starpu_worker *worker = _starpu_get_worker_struct(workerid);
903  STARPU_ASSERT(worker != NULL);
904  return worker->state_sched_op_pending;
905 }
906 
916 static inline void _starpu_worker_enter_changing_ctx_op(struct _starpu_worker * const worker)
917 {
918  STARPU_ASSERT(!starpu_pthread_equal(worker->thread_changing_ctx, starpu_pthread_self()));
919  /* flush pending requests to start on a fresh transaction epoch */
920  while (worker->state_changing_ctx_notice)
921  STARPU_PTHREAD_COND_WAIT(&worker->sched_cond, &worker->sched_mutex);
922 
923  /* announce changing_ctx intent
924  *
925  * - an already started sched_op is allowed to complete
926  * - no new sched_op may be started
927  */
928  worker->state_changing_ctx_notice = 1;
929 
930  worker->thread_changing_ctx = starpu_pthread_self();
931 
932  /* allow for an already started sched_op to complete */
933  if (worker->state_sched_op_pending)
934  {
935  /* request sched_op to broadcast when way is cleared */
936  worker->state_changing_ctx_waiting = 1;
937 
938  /* wait for sched_op completion */
939  STARPU_PTHREAD_COND_BROADCAST(&worker->sched_cond);
940 #ifdef STARPU_SIMGRID
941  starpu_pthread_queue_broadcast(&_starpu_simgrid_task_queue[worker->workerid]);
942 #endif
943  do
944  {
945  STARPU_PTHREAD_COND_WAIT(&worker->sched_cond, &worker->sched_mutex);
946  }
947  while (worker->state_sched_op_pending);
948 
949  /* reset flag so other sched_ops wont have to broadcast state */
950  worker->state_changing_ctx_waiting = 0;
951  }
952 }
953 
958 static inline void _starpu_worker_leave_changing_ctx_op(struct _starpu_worker * const worker)
959 {
960  worker->thread_changing_ctx = (starpu_pthread_t)0;
961  worker->state_changing_ctx_notice = 0;
962  STARPU_PTHREAD_COND_BROADCAST(&worker->sched_cond);
963 }
964 
967 #ifdef STARPU_SPINLOCK_CHECK
968 static inline void __starpu_worker_relax_on(const char*file, int line, const char* func)
969 #else
970 static inline void _starpu_worker_relax_on(void)
971 #endif
972 {
973  struct _starpu_worker *worker = _starpu_get_local_worker_key();
974  if (worker == NULL)
975  return;
976  if (!worker->state_sched_op_pending)
977  return;
978  STARPU_PTHREAD_MUTEX_LOCK_SCHED(&worker->sched_mutex);
979 #ifdef STARPU_SPINLOCK_CHECK
980  STARPU_ASSERT_MSG(worker->state_relax_refcnt<UINT_MAX, "relax last turn on in %s (%s:%d)\n", worker->relax_on_func, worker->relax_on_file, worker->relax_on_line);
981 #else
982  STARPU_ASSERT(worker->state_relax_refcnt<UINT_MAX);
983 #endif
984  worker->state_relax_refcnt++;
985 #ifdef STARPU_SPINLOCK_CHECK
986  worker->relax_on_file = file;
987  worker->relax_on_line = line;
988  worker->relax_on_func = func;
989 #endif
990  STARPU_PTHREAD_COND_BROADCAST(&worker->sched_cond);
991  STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&worker->sched_mutex);
992 }
993 #ifdef STARPU_SPINLOCK_CHECK
994 #define _starpu_worker_relax_on() __starpu_worker_relax_on(__FILE__, __LINE__, __starpu_func__)
995 #endif
996 #define starpu_worker_relax_on _starpu_worker_relax_on
997 
999 #ifdef STARPU_SPINLOCK_CHECK
1000 static inline void __starpu_worker_relax_on_locked(struct _starpu_worker *worker, const char*file, int line, const char* func)
1001 #else
1002 static inline void _starpu_worker_relax_on_locked(struct _starpu_worker *worker)
1003 #endif
1004 {
1005  if (!worker->state_sched_op_pending)
1006  return;
1007 #ifdef STARPU_SPINLOCK_CHECK
1008  STARPU_ASSERT_MSG(worker->state_relax_refcnt<UINT_MAX, "relax last turn on in %s (%s:%d)\n", worker->relax_on_func, worker->relax_on_file, worker->relax_on_line);
1009 #else
1010  STARPU_ASSERT(worker->state_relax_refcnt<UINT_MAX);
1011 #endif
1012  worker->state_relax_refcnt++;
1013 #ifdef STARPU_SPINLOCK_CHECK
1014  worker->relax_on_file = file;
1015  worker->relax_on_line = line;
1016  worker->relax_on_func = func;
1017 #endif
1018  STARPU_PTHREAD_COND_BROADCAST(&worker->sched_cond);
1019 }
1020 #ifdef STARPU_SPINLOCK_CHECK
1021 #define _starpu_worker_relax_on_locked(worker) __starpu_worker_relax_on_locked(worker,__FILE__, __LINE__, __starpu_func__)
1022 #endif
1023 
1024 #ifdef STARPU_SPINLOCK_CHECK
1025 static inline void __starpu_worker_relax_off(const char*file, int line, const char* func)
1026 #else
1027 static inline void _starpu_worker_relax_off(void)
1028 #endif
1029 {
1030  int workerid = starpu_worker_get_id();
1031  if (workerid == -1)
1032  return;
1033  struct _starpu_worker *worker = _starpu_get_worker_struct(workerid);
1034  STARPU_ASSERT(worker != NULL);
1035  if (!worker->state_sched_op_pending)
1036  return;
1037  STARPU_PTHREAD_MUTEX_LOCK_SCHED(&worker->sched_mutex);
1038 #ifdef STARPU_SPINLOCK_CHECK
1039  STARPU_ASSERT_MSG(worker->state_relax_refcnt>0, "relax last turn off in %s (%s:%d)\n", worker->relax_on_func, worker->relax_on_file, worker->relax_on_line);
1040 #else
1041  STARPU_ASSERT(worker->state_relax_refcnt>0);
1042 #endif
1043  worker->state_relax_refcnt--;
1044 #ifdef STARPU_SPINLOCK_CHECK
1045  worker->relax_off_file = file;
1046  worker->relax_off_line = line;
1047  worker->relax_off_func = func;
1048 #endif
1049  STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&worker->sched_mutex);
1050 }
1051 #ifdef STARPU_SPINLOCK_CHECK
1052 #define _starpu_worker_relax_off() __starpu_worker_relax_off(__FILE__, __LINE__, __starpu_func__)
1053 #endif
1054 #define starpu_worker_relax_off _starpu_worker_relax_off
1055 
1056 #ifdef STARPU_SPINLOCK_CHECK
1057 static inline void __starpu_worker_relax_off_locked(const char*file, int line, const char* func)
1058 #else
1059 static inline void _starpu_worker_relax_off_locked(void)
1060 #endif
1061 {
1062  int workerid = starpu_worker_get_id();
1063  if (workerid == -1)
1064  return;
1065  struct _starpu_worker *worker = _starpu_get_worker_struct(workerid);
1066  STARPU_ASSERT(worker != NULL);
1067  if (!worker->state_sched_op_pending)
1068  return;
1069 #ifdef STARPU_SPINLOCK_CHECK
1070  STARPU_ASSERT_MSG(worker->state_relax_refcnt>0, "relax last turn off in %s (%s:%d)\n", worker->relax_on_func, worker->relax_on_file, worker->relax_on_line);
1071 #else
1072  STARPU_ASSERT(worker->state_relax_refcnt>0);
1073 #endif
1074  worker->state_relax_refcnt--;
1075 #ifdef STARPU_SPINLOCK_CHECK
1076  worker->relax_off_file = file;
1077  worker->relax_off_line = line;
1078  worker->relax_off_func = func;
1079 #endif
1080 }
1081 #ifdef STARPU_SPINLOCK_CHECK
1082 #define _starpu_worker_relax_off_locked() __starpu_worker_relax_off_locked(__FILE__, __LINE__, __starpu_func__)
1083 #endif
1084 
1085 static inline int _starpu_worker_get_relax_state(void)
1086 {
1087  int workerid = starpu_worker_get_id();
1088  if (workerid < 0)
1089  return 1;
1090  struct _starpu_worker *worker = _starpu_get_worker_struct(workerid);
1091  STARPU_ASSERT(worker != NULL);
1092  return worker->state_relax_refcnt != 0;
1093 }
1094 #define starpu_worker_get_relax_state _starpu_worker_get_relax_state
1095 
1100 static inline void _starpu_worker_lock(int workerid)
1101 {
1102  struct _starpu_worker *worker = _starpu_get_worker_struct(workerid);
1103  STARPU_ASSERT(worker != NULL);
1104  int cur_workerid = starpu_worker_get_id();
1105  if (workerid != cur_workerid)
1106  {
1107  starpu_worker_relax_on();
1108 
1109  STARPU_PTHREAD_MUTEX_LOCK_SCHED(&worker->sched_mutex);
1110  while (!worker->state_relax_refcnt)
1111  {
1112  STARPU_PTHREAD_COND_WAIT(&worker->sched_cond, &worker->sched_mutex);
1113  }
1114  }
1115  else
1116  {
1117  STARPU_PTHREAD_MUTEX_LOCK_SCHED(&worker->sched_mutex);
1118  }
1119 }
1120 
1121 static inline int _starpu_worker_trylock(int workerid)
1122 {
1123  struct _starpu_worker *cur_worker = _starpu_get_local_worker_key();
1124  int cur_workerid = cur_worker->workerid;
1125  struct _starpu_worker *worker = _starpu_get_worker_struct(workerid);
1126  STARPU_ASSERT(worker != NULL);
1127 
1128  /* Start with ourself */
1129  int ret = STARPU_PTHREAD_MUTEX_TRYLOCK_SCHED(&cur_worker->sched_mutex);
1130  if (ret)
1131  return ret;
1132  if (workerid == cur_workerid)
1133  /* We only needed to lock ourself */
1134  return 0;
1135 
1136  /* Now try to lock the other worker */
1137  ret = STARPU_PTHREAD_MUTEX_TRYLOCK_SCHED(&worker->sched_mutex);
1138  if (!ret)
1139  {
1140  /* Good, check that it is relaxed */
1141  ret = !worker->state_relax_refcnt;
1142  if (ret)
1143  STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&worker->sched_mutex);
1144  }
1145  if (!ret)
1146  _starpu_worker_relax_on_locked(cur_worker);
1147  STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&cur_worker->sched_mutex);
1148  return ret;
1149 }
1150 
1151 static inline void _starpu_worker_unlock(int workerid)
1152 {
1153  struct _starpu_worker *worker = _starpu_get_worker_struct(workerid);
1154  STARPU_ASSERT(worker != NULL);
1155  STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&worker->sched_mutex);
1156  int cur_workerid = starpu_worker_get_id();
1157  if (workerid != cur_workerid)
1158  {
1159  starpu_worker_relax_off();
1160  }
1161 }
1162 
1163 static inline void _starpu_worker_lock_self(void)
1164 {
1165  int workerid = starpu_worker_get_id_check();
1166  struct _starpu_worker *worker = _starpu_get_worker_struct(workerid);
1167  STARPU_ASSERT(worker != NULL);
1168  STARPU_PTHREAD_MUTEX_LOCK_SCHED(&worker->sched_mutex);
1169 }
1170 
1171 static inline void _starpu_worker_unlock_self(void)
1172 {
1173  int workerid = starpu_worker_get_id_check();
1174  struct _starpu_worker *worker = _starpu_get_worker_struct(workerid);
1175  STARPU_ASSERT(worker != NULL);
1176  STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&worker->sched_mutex);
1177 }
1178 
1179 static inline int _starpu_wake_worker_relax(int workerid)
1180 {
1181  _starpu_worker_lock(workerid);
1182  int ret = starpu_wake_worker_locked(workerid);
1183  _starpu_worker_unlock(workerid);
1184  return ret;
1185 }
1186 
1187 int starpu_wake_worker_relax_light(int workerid);
1188 
1193 void _starpu_worker_refuse_task(struct _starpu_worker *worker, struct starpu_task *task);
1194 
1195 /* @}*/
1196 
1197 #endif // __WORKERS_H__
struct starpu_task ** local_ordered_tasks
Definition: workers.h:135
void _starpu_may_pause(void)
unsigned devid
Definition: workers.h:82
static unsigned __starpu_worker_get_id_check(const char *f, int l)
Definition: workers.h:653
static void _starpu_worker_relax_on(void)
Definition: workers.h:970
int pause_depth
Definition: workers.h:439
unsigned char pipeline_length
Definition: workers.h:149
unsigned ncudagpus
Definition: workers.h:295
static void _starpu_worker_request_blocking_in_parallel(struct _starpu_worker *const worker)
Definition: workers.h:672
int workerid
Definition: workers.h:85
unsigned state_unblock_in_parallel_req
Definition: workers.h:115
unsigned current_ordered_task_order
Definition: workers.h:138
static unsigned _starpu_get_nsched_ctxs(void)
Definition: workers.h:625
unsigned subworkerid
Definition: workers.h:83
unsigned nhwcudagpus
Definition: workers.h:274
unsigned nb_buffers_totransfer
Definition: workers.h:173
int current_mpi_deviceid
Definition: workers.h:393
unsigned memory_node
Definition: workers.h:209
unsigned nhwopenclgpus
Definition: workers.h:279
uint32_t _starpu_can_submit_scc_task(void)
void _starpu_worker_refuse_task(struct _starpu_worker *worker, struct starpu_task *task)
static void _starpu_worker_leave_changing_ctx_op(struct _starpu_worker *const worker)
Definition: workers.h:958
unsigned nmpidevices
Definition: workers.h:307
int combined_workerid
Definition: workers.h:86
Definition: workers.h:366
struct starpu_perfmodel_arch perf_arch
Definition: workers.h:206
unsigned has_prev_init
Definition: workers.h:166
unsigned run_by_starpu
Definition: workers.h:158
unsigned nsched_ctxs
Definition: workers.h:163
starpu_pthread_t thread_changing_ctx
Definition: workers.h:125
static struct _starpu_machine_config * _starpu_get_machine_config(void)
Definition: workers.h:577
void _starpu_set_argc_argv(int *argc, char ***argv)
uint32_t _starpu_can_submit_opencl_task(void)
unsigned started
Definition: workers.h:233
static void _starpu_set_local_worker_set_key(struct _starpu_worker_set *worker)
Definition: workers.h:543
int cuda_nodeid
Definition: workers.h:398
uint32_t worker_mask
Definition: workers.h:79
static struct _starpu_sched_ctx * _starpu_get_initial_sched_ctx(void)
Definition: workers.h:602
unsigned current_ordered_task
Definition: workers.h:137
unsigned local_ordered_tasks_size
Definition: workers.h:136
static void _starpu_worker_lock(int workerid)
Definition: workers.h:1100
static void _starpu_worker_request_unblocking_in_parallel(struct _starpu_worker *const worker)
Definition: workers.h:721
unsigned state_relax_refcnt
Definition: workers.h:99
struct starpu_task * task_transferring
Definition: workers.h:174
Definition: workers.h:244
unsigned pop_ctx_priority
Definition: workers.h:190
static void _starpu_set_local_worker_key(struct _starpu_worker *worker)
Definition: workers.h:525
unsigned nhwmpi
Definition: workers.h:289
static struct _starpu_worker_set * _starpu_get_local_worker_set_key(void)
Definition: workers.h:551
unsigned nsccdevices
Definition: workers.h:304
unsigned block_in_parallel_ref_count
Definition: workers.h:124
unsigned submitting
Definition: workers.h:445
int scc_nodeid
Definition: workers.h:404
uint32_t _starpu_can_submit_cpu_task(void)
Definition: workers.h:204
uint32_t worker_mask
Definition: workers.h:207
void _starpu_conf_check_environment(struct starpu_conf *conf)
starpu_pthread_cond_t ready_cond
Definition: workers.h:90
unsigned nb_buffers_transferred
Definition: workers.h:172
unsigned char first_task
Definition: workers.h:147
unsigned memory_node
Definition: workers.h:91
unsigned nhwcpus
Definition: workers.h:264
unsigned nopenclgpus
Definition: workers.h:301
unsigned state_blocked_in_parallel
Definition: workers.h:111
struct starpu_conf conf
Definition: workers.h:430
static unsigned _starpu_machine_is_running(void)
Definition: workers.h:471
Definition: workers.h:75
static enum _starpu_worker_status _starpu_worker_get_status(int workerid)
Definition: workers.h:589
unsigned state_block_in_parallel_ack
Definition: workers.h:114
unsigned nhwmicdevices
Definition: workers.h:315
int bindid
Definition: workers.h:84
static struct _starpu_worker * _starpu_get_worker_struct(unsigned id)
Definition: workers.h:560
starpu_pthread_cond_t ready_cond
Definition: workers.h:236
unsigned state_changing_ctx_waiting
Definition: workers.h:109
int current_bindid
Definition: workers.h:376
static void _starpu_worker_enter_changing_ctx_op(struct _starpu_worker *const worker)
Definition: workers.h:916
unsigned nhwscc
Definition: workers.h:284
unsigned ncpus
Definition: workers.h:292
void _starpu_block_worker(int workerid, starpu_pthread_cond_t *cond, starpu_pthread_mutex_t *mutex)
int current_mic_deviceid
Definition: workers.h:387
static void _starpu_worker_process_block_in_parallel_requests(struct _starpu_worker *const worker)
Definition: workers.h:770
static void _starpu_worker_relax_on_locked(struct _starpu_worker *worker)
Definition: workers.h:1002
unsigned spinning_backoff
Definition: workers.h:170
int current_cuda_gpuid
Definition: workers.h:381
void _starpu_driver_start(struct _starpu_worker *worker, unsigned fut_key, unsigned sync)
static void _starpu_worker_enter_sched_op(struct _starpu_worker *const worker)
Definition: workers.h:825
struct starpu_tree * tree
Definition: workers.h:259
int current_rank
Definition: workers.h:87
unsigned numa_memory_node
Definition: workers.h:92
uint32_t worker_mask
Definition: workers.h:427
unsigned state_sched_op_pending
Definition: workers.h:108
starpu_pthread_t worker_thread
Definition: workers.h:81
int worker_size
Definition: workers.h:88
unsigned ncombinedworkers
Definition: workers.h:250
unsigned nworkers
Definition: workers.h:247
unsigned is_slave_somewhere
Definition: workers.h:191
Definition: workers.h:228
int mic_nodeid
Definition: workers.h:402
unsigned state_block_in_parallel_req
Definition: workers.h:113
uint32_t _starpu_can_submit_cuda_task(void)
uint32_t _starpu_worker_exists(struct starpu_task *)
void _starpu_worker_apply_deferred_ctx_changes(void)
unsigned char pipeline_stuck
Definition: workers.h:150
hwloc_topology_t hwtopology
Definition: workers.h:256
static void _starpu_worker_set_status(int workerid, enum _starpu_worker_status status)
Definition: workers.h:596
unsigned state_changing_ctx_notice
Definition: workers.h:110
int current_scc_deviceid
Definition: workers.h:390
int cpus_nodeid
Definition: workers.h:396
unsigned state_unblock_in_parallel_ack
Definition: workers.h:116
struct starpu_task * current_task
Definition: workers.h:139
starpu_pthread_cond_t sched_cond
Definition: workers.h:97
starpu_pthread_mutex_t sched_mutex
Definition: workers.h:98
static int _starpu_get_disable_kernels(void)
Definition: workers.h:583
unsigned _starpu_worker_can_block(unsigned memnode, struct _starpu_worker *worker)
unsigned nbindid
Definition: workers.h:422
static struct _starpu_worker * _starpu_get_local_worker_key(void)
Definition: workers.h:533
unsigned state_blocked_in_parallel_observed
Definition: workers.h:112
static int _starpu_worker_get_id(void)
Definition: workers.h:633
int opencl_nodeid
Definition: workers.h:400
void _starpu_worker_start(struct _starpu_worker *worker, unsigned fut_key, unsigned sync)
unsigned running
Definition: workers.h:433
starpu_pthread_cond_t started_cond
Definition: workers.h:89
int starpu_worker_get_nids_ctx_free_by_type(enum starpu_worker_archtype type, int *workerids, int maxsize)
int current_opencl_gpuid
Definition: workers.h:384
void _starpu_worker_init(struct _starpu_worker *workerarg, struct _starpu_machine_config *pconfig)
static struct _starpu_sched_ctx * _starpu_get_sched_ctx_struct(unsigned id)
Definition: workers.h:568
unsigned char ntasks
Definition: workers.h:148
unsigned state_keep_awake
Definition: workers.h:155
unsigned nhwpus
Definition: workers.h:269
int mpi_nodeid
Definition: workers.h:406
starpu_pthread_t worker_thread
Definition: workers.h:231