xmp_coarray_mpi.c File Reference

#include <stdio.h>
#include <stdlib.h>
#include "xmp_internal.h"
#include <signal.h>
#include <string.h>

Include dependency graph for xmp_coarray_mpi.c:

Data Structures
struct	_shift_queue_t

Macros
#define	_SYNCIMAGE_SENDRECV
#define	_is_put_blocking   (! _XMP_flag_put_nb)
#define	_is_put_local_blocking   (! _XMP_flag_put_nb)
#define	_is_get_blocking   (! _XMP_flag_get_nb)

Functions
MPI_Win	_XMP_mpi_coarray_get_window (const _XMP_coarray_t *desc, bool is_acc)
char *	_XMP_mpi_coarray_get_remote_addr (const _XMP_coarray_t *desc, const int target_rank, const bool is_acc)
char *	_XMP_mpi_coarray_get_local_addr (const _XMP_coarray_t *desc, const bool is_acc)
void	_XMP_mpi_build_shift_queue (bool is_acc)
	Set initial value to the shift queue. More...
void	_XMP_mpi_destroy_shift_queue (bool is_acc)
	Destroy shift queue. More...
void	_XMP_mpi_coarray_lastly_deallocate (bool is_acc)
	Deallocate memory region when calling _XMP_coarray_lastly_deallocate() More...
void	_XMP_mpi_coarray_deallocate (_XMP_coarray_t *c, bool is_acc)
void	_XMP_mpi_coarray_malloc (_XMP_coarray_t *coarray_desc, void **addr, const size_t coarray_size, bool is_acc)
void	_XMP_mpi_contiguous_put (const int org_target_rank, const _XMP_coarray_t *dst_desc, const _XMP_coarray_t *src_desc, const size_t dst_offset, const size_t src_offset, const size_t dst_elmts, const size_t src_elmts, const size_t elmt_size, const bool is_dst_on_acc, const bool is_src_on_acc)
void	_XMP_mpi_contiguous_get (const int org_target_rank, const _XMP_coarray_t *dst_desc, const _XMP_coarray_t *src_desc, const size_t dst_offset, const size_t src_offset, const size_t dst_elmts, const size_t src_elmts, const size_t elmt_size, const bool is_dst_on_acc, const bool is_src_on_acc)
void	_XMP_mpi_put (const int dst_contiguous, const int src_contiguous, const int org_target_rank, const int dst_dims, const int src_dims, const _XMP_array_section_t *dst_info, const _XMP_array_section_t *src_info, const _XMP_coarray_t *dst_desc, const void *src, const int dst_elmts, const int src_elmts, const int is_dst_on_acc)
void	_XMP_mpi_get (const int src_contiguous, const int dst_contiguous, const int org_target_rank, const int src_dims, const int dst_dims, const _XMP_array_section_t *src_info, const _XMP_array_section_t *dst_info, const _XMP_coarray_t *src_desc, void *dst, const int src_elmts, const int dst_elmts, const int is_src_on_acc)
void	_XMP_mpi_sync_memory ()
	Execute sync_memory. More...
void	_XMP_mpi_sync_all ()
	Execute sync_all. More...
void	_XMP_mpi_coarray_attach (_XMP_coarray_t *coarray_desc, void *addr, const size_t coarray_size, const bool is_acc)
void	_XMP_mpi_coarray_detach (_XMP_coarray_t *coarray_desc, const bool is_acc)
void	_XMP_mpi_build_sync_images_table ()
	Build table and Initialize for sync images. More...
void	_XMP_mpi_sync_images (const int num, const int *image_set, int *status)
	Execute sync images. More...
void	_XMP_sync_images_EXEC (int *status)
void	_XMP_sync_images_COMM (MPI_Comm *comm, int *status)
void	_XMP_mpi_coarray_regmem (_XMP_coarray_t *coarray_desc, void *real_addr, const size_t coarray_size, bool is_acc)
void	_XMP_mpi_coarray_deregmem (_XMP_coarray_t *c)

Variables
int	_XMP_flag_put_nb
	These variables are temporral. More...
int	_XMP_flag_get_nb
int	_XMP_flag_multi_win

Macro Definition Documentation

_is_get_blocking

#define _is_get_blocking   (! _XMP_flag_get_nb)

_is_put_blocking

#define _is_put_blocking   (! _XMP_flag_put_nb)

_is_put_local_blocking

#define _is_put_local_blocking   (! _XMP_flag_put_nb)

_SYNCIMAGE_SENDRECV

#define _SYNCIMAGE_SENDRECV

Function Documentation

_XMP_mpi_build_shift_queue()

void _XMP_mpi_build_shift_queue

(

bool

is_acc

)

Set initial value to the shift queue.

{
  struct _shift_queue_t *shift_queue = is_acc? &_shift_queue_acc : &_shift_queue;
  
  shift_queue->max_size = _XMP_MPI_ONESIDED_COARRAY_SHIFT_QUEUE_INITIAL_SIZE;
  shift_queue->num      = 0;
  shift_queue->shifts   = malloc(sizeof(size_t) * shift_queue->max_size);
  shift_queue->total_shift = 0;
}

Here is the caller graph for this function:

_XMP_mpi_build_sync_images_table()

void _XMP_mpi_build_sync_images_table

(

)

Build table and Initialize for sync images.

{
  size_t table_size = sizeof(unsigned int) * _XMP_world_size;
#ifdef _SYNCIMAGE_SENDRECV
  _sync_images_table = _XMP_alloc(table_size);
#else
  struct _shift_queue_t *shift_queue = &_shift_queue;
  _sync_images_table = (unsigned int*)(_xmp_mpi_onesided_buf + shift_queue->total_shift);
  _sync_images_table_disp = (unsigned int*)(shift_queue->total_shift);
 
  size_t shift;
  if(table_size % _XMP_MPI_ALIGNMENT == 0)
    shift = table_size;
  else{
    shift = ((table_size / _XMP_MPI_ALIGNMENT) + 1) * _XMP_MPI_ALIGNMENT;
  }
  _push_shift_queue(shift_queue, shift);
#endif
 
  for(int i=0;i<_XMP_world_size;i++)
    _sync_images_table[i] = 0;
 
  MPI_Barrier(MPI_COMM_WORLD);
}

Here is the caller graph for this function:

_XMP_mpi_coarray_attach()

void _XMP_mpi_coarray_attach	(	_XMP_coarray_t *	coarray_desc,
		void *	addr,
		const size_t	coarray_size,
		const bool	is_acc
	)

{
  MPI_Win win = MPI_WIN_NULL;
  char **each_addr = NULL;  // head address of a local array on each node
 
  _XMP_nodes_t *nodes = _XMP_get_execution_nodes();
  int comm_size = nodes->comm_size;
  MPI_Comm comm = *(MPI_Comm *)nodes->comm;
 
  XACC_DEBUG("attach addr=%p, size=%zd, is_acc=%d", addr, coarray_size, is_acc);
 
  if(_XMP_flag_multi_win){
    _XMP_mpi_onesided_create_win(&win, addr, coarray_size, comm);
    MPI_Win_lock_all(MPI_MODE_NOCHECK, win);
  }else{
    win = _xmp_mpi_distarray_win;
#ifdef _XMP_XACC
    if(is_acc){
      win = _xmp_mpi_distarray_win_acc;
    }
#endif
    MPI_Win_attach(win, addr, coarray_size);
 
    each_addr = (char**)_XMP_alloc(sizeof(char *) * comm_size);
 
    MPI_Allgather(&addr, sizeof(char *), MPI_BYTE,
                  each_addr, sizeof(char *), MPI_BYTE,
                  comm); // exchange displacement
  }
 
  if(is_acc){
#ifdef _XMP_XACC
    coarray_desc->addr_dev = each_addr;
    coarray_desc->real_addr_dev = addr;
    coarray_desc->win_acc = win;
    coarray_desc->nodes = nodes;
#endif
  }else{
    coarray_desc->addr = each_addr;
    coarray_desc->real_addr = addr;
    coarray_desc->win = win;
    coarray_desc->win_acc = MPI_WIN_NULL;
    coarray_desc->nodes = nodes;
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

_XMP_mpi_coarray_deallocate()

void _XMP_mpi_coarray_deallocate	(	_XMP_coarray_t *	c,
		bool	is_acc
	)

{
  if(_XMP_flag_multi_win){
    MPI_Win_unlock_all(c->win);
    _XMP_barrier_EXEC();
    _XMP_mpi_onesided_dealloc_win(&(c->win), (void **)&(c->real_addr), is_acc);
  }
}

Here is the call graph for this function:

_XMP_mpi_coarray_deregmem()

void _XMP_mpi_coarray_deregmem

(

_XMP_coarray_t *

c

)

{
  if(! _XMP_flag_multi_win){
    _XMP_fatal("single window mode does not support coarray deregmem");
  }
 
  MPI_Win_unlock_all(c->win);
  _XMP_barrier_EXEC();
  _XMP_mpi_onesided_destroy_win(&(c->win));
}

Here is the call graph for this function:

_XMP_mpi_coarray_detach()

void _XMP_mpi_coarray_detach	(	_XMP_coarray_t *	coarray_desc,
		const bool	is_acc
	)

{
  if(_XMP_flag_multi_win){
    MPI_Win win = is_acc? coarray_desc->win_acc : coarray_desc->win;
    MPI_Win_unlock_all(win);
    _XMP_barrier_EXEC();
    _XMP_mpi_onesided_destroy_win(&win);
  }else{
    MPI_Win win = _xmp_mpi_distarray_win;
    void *real_addr = coarray_desc->real_addr;
#ifdef _XMP_XACC
    if(is_acc){
      win = _xmp_mpi_distarray_win_acc;
      real_addr = coarray_desc->real_addr_dev;
    }
#endif
 
    MPI_Win_detach(win, real_addr);
  }
 
  if(is_acc){
#ifdef _XMP_XACC
    _XMP_free(coarray_desc->addr_dev); //FIXME may be wrong
    coarray_desc->addr_dev = NULL;
    coarray_desc->real_addr_dev = NULL;
    coarray_desc->win_acc = MPI_WIN_NULL;
    coarray_desc->nodes = NULL;
#endif
  }else{
    _XMP_free(coarray_desc->addr);
    coarray_desc->addr = NULL;
    coarray_desc->real_addr = NULL;
    coarray_desc->win = MPI_WIN_NULL;
    coarray_desc->nodes = NULL;
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

_XMP_mpi_coarray_get_local_addr()

char* _XMP_mpi_coarray_get_local_addr	(	const _XMP_coarray_t *	desc,
		const bool	is_acc
	)

{
  return get_local_addr(desc, is_acc);
}

_XMP_mpi_coarray_get_remote_addr()

char* _XMP_mpi_coarray_get_remote_addr	(	const _XMP_coarray_t *	desc,
		const int	target_rank,
		const bool	is_acc
	)

{
  return get_remote_addr(desc, target_rank, is_acc);
}

Here is the caller graph for this function:

_XMP_mpi_coarray_get_window()

MPI_Win _XMP_mpi_coarray_get_window	(	const _XMP_coarray_t *	desc,
		bool	is_acc
	)

{
  return get_window(desc, is_acc);
}

Here is the caller graph for this function:

_XMP_mpi_coarray_lastly_deallocate()

void _XMP_mpi_coarray_lastly_deallocate

(

bool

is_acc

)

Deallocate memory region when calling _XMP_coarray_lastly_deallocate()

                                                    {
  struct _shift_queue_t *shift_queue = is_acc? &_shift_queue_acc : &_shift_queue;
  _pop_shift_queue(shift_queue);
}

Here is the caller graph for this function:

_XMP_mpi_coarray_malloc()

void _XMP_mpi_coarray_malloc	(	_XMP_coarray_t *	coarray_desc,
		void **	addr,
		const size_t	coarray_size,
		bool	is_acc
	)

{
  char **each_addr = NULL;  // gap_size on each node
  struct _shift_queue_t *shift_queue = is_acc? &_shift_queue_acc : &_shift_queue;
  size_t shift;
  char *real_addr = NULL;
  MPI_Win win = MPI_WIN_NULL;
  _XMP_nodes_t *nodes = _XMP_get_execution_nodes();
  MPI_Comm comm = *(MPI_Comm *)nodes->comm;
 
  if(coarray_size == 0){
    _XMP_fatal("_XMP_mpi_coarray_malloc: zero size is not allowed");
  }
 
  if(_XMP_flag_multi_win){
    _XMP_mpi_onesided_alloc_win(&win, (void**)&real_addr, coarray_size, comm, is_acc);
    MPI_Win_lock_all(MPI_MODE_NOCHECK, win);
 
    XACC_DEBUG("addr=%p, size=%zd, is_acc=%d", real_addr, coarray_size, is_acc);
  }else{
  each_addr = (char**)_XMP_alloc(sizeof(char *) * _XMP_world_size);
  for(int i=0;i<_XMP_world_size;i++){
    each_addr[i] = (char *)(shift_queue->total_shift);
  }
  real_addr = _xmp_mpi_onesided_buf;
#ifdef _XMP_XACC
  if(is_acc){
    real_addr = _xmp_mpi_onesided_buf_acc;
  }
#endif
  real_addr += shift_queue->total_shift;
 
  XACC_DEBUG("malloc_do: addr=%p, shift=%zd, is_acc=%d", real_addr, shift_queue->total_shift, is_acc);
  
  if(coarray_size % _XMP_MPI_ALIGNMENT == 0)
    shift = coarray_size;
  else{
    shift = ((coarray_size / _XMP_MPI_ALIGNMENT) + 1) * _XMP_MPI_ALIGNMENT;
  }
  
  _push_shift_queue(shift_queue, shift);
 
  size_t total_shift = shift_queue->total_shift;
 
  if(total_shift > _xmp_mpi_onesided_heap_size){
    fprintf(stderr, "_xmp_mpi_onesided_heap_size=%zd\n", _xmp_mpi_onesided_heap_size);
    if(_XMP_world_rank == 0){
      fprintf(stderr, "[ERROR] Cannot allocate coarray. Heap memory size of coarray is too small.\n");
      fprintf(stderr, "        Please set the environmental variable \"XMP_ONESIDED_HEAP_SIZE\".\n");
      fprintf(stderr, "        e.g.) export XMP_ONESIDED_HEAP_SIZE=%zuM (or more).\n",
              (total_shift/1024/1024)+1);
    }
    _XMP_fatal_nomsg();
  }
  }
 
  if(is_acc){
#ifdef _XMP_XACC
    coarray_desc->addr_dev = each_addr;
    coarray_desc->real_addr_dev = real_addr;
    coarray_desc->win_acc = win;
    coarray_desc->nodes = nodes;
#endif
  }else{
    coarray_desc->addr = each_addr;
    coarray_desc->real_addr = real_addr;
    coarray_desc->win = win;
    coarray_desc->win_acc = MPI_WIN_NULL;
    coarray_desc->nodes = nodes;
  }
  *addr = real_addr;
}

Here is the caller graph for this function:

_XMP_mpi_coarray_regmem()

void _XMP_mpi_coarray_regmem	(	_XMP_coarray_t *	coarray_desc,
		void *	real_addr,
		const size_t	coarray_size,
		bool	is_acc
	)

{
  char **each_addr = NULL;
  MPI_Win win = MPI_WIN_NULL;
  _XMP_nodes_t *nodes = _XMP_get_execution_nodes();
  MPI_Comm comm = *(MPI_Comm *)nodes->comm;
 
  if(! _XMP_flag_multi_win){
    _XMP_fatal("single window mode does not support coarray regmem");
  }
 
  if(coarray_size == 0){
    _XMP_fatal("_XMP_mpi_coarray_regmem: zero size is not allowed");
  }
 
  _XMP_mpi_onesided_create_win(&win, real_addr, coarray_size, comm);
  MPI_Win_lock_all(MPI_MODE_NOCHECK, win);
 
  XACC_DEBUG("addr=%p, size=%zd, is_acc=%d", real_addr, coarray_size, is_acc);
 
  if(is_acc){
#ifdef _XMP_XACC
    coarray_desc->addr_dev = each_addr;
    coarray_desc->real_addr_dev = real_addr;
    coarray_desc->win_acc = win;
    coarray_desc->nodes = nodes;
#endif
  }else{
    coarray_desc->addr = each_addr;
    coarray_desc->real_addr = real_addr;
    coarray_desc->win = win;
    coarray_desc->win_acc = MPI_WIN_NULL;
    coarray_desc->nodes = nodes;
  }
}

Here is the call graph for this function:

_XMP_mpi_contiguous_get()

void _XMP_mpi_contiguous_get	(	const int	org_target_rank,
		const _XMP_coarray_t *	dst_desc,
		const _XMP_coarray_t *	src_desc,
		const size_t	dst_offset,
		const size_t	src_offset,
		const size_t	dst_elmts,
		const size_t	src_elmts,
		const size_t	elmt_size,
		const bool	is_dst_on_acc,
		const bool	is_src_on_acc
	)

{
  const int target_rank = _XMP_mpi_trans_rank(src_desc, org_target_rank);
 
  size_t transfer_size = elmt_size * dst_elmts;
  char *dst = get_local_addr(dst_desc, is_dst_on_acc);
  if(dst_elmts == src_elmts){
    _mpi_contiguous(_XMP_N_COARRAY_GET,
                    target_rank,
                    src_desc, dst,
                    src_offset, dst_offset,
                    transfer_size, is_src_on_acc);
  }else if(src_elmts == 1){
    _XMP_array_section_t dst_info;
    dst_info.start = dst_offset / elmt_size;
    dst_info.length = dst_elmts;
    dst_info.stride = 1;
    dst_info.elmts = dst_info.start + dst_info.length;
    dst_info.distance = elmt_size;
    _mpi_scalar_mget(target_rank,
                     dst, src_desc,
                     dst_offset, src_offset,
                     1 /*dst_dims*/,
                     &dst_info,
                     is_src_on_acc);
  }else{
    _XMP_fatal("Coarray Error ! transfer size is wrong.\n");
  }
}

Here is the caller graph for this function:

_XMP_mpi_contiguous_put()

void _XMP_mpi_contiguous_put	(	const int	org_target_rank,
		const _XMP_coarray_t *	dst_desc,
		const _XMP_coarray_t *	src_desc,
		const size_t	dst_offset,
		const size_t	src_offset,
		const size_t	dst_elmts,
		const size_t	src_elmts,
		const size_t	elmt_size,
		const bool	is_dst_on_acc,
		const bool	is_src_on_acc
	)

{
  const int target_rank = _XMP_mpi_trans_rank(dst_desc, org_target_rank);
 
  size_t transfer_size = elmt_size * dst_elmts;
  char *src = get_local_addr(src_desc, is_src_on_acc);
  if(dst_elmts == src_elmts){
    _mpi_contiguous(_XMP_N_COARRAY_PUT,
                    target_rank,
                    dst_desc, src,
                    dst_offset, src_offset,
                    transfer_size, is_dst_on_acc);
  }else if(src_elmts == 1){
    _XMP_array_section_t dst_info;
    dst_info.start = dst_offset / elmt_size;
    dst_info.length = dst_elmts;
    dst_info.stride = 1;
    dst_info.elmts = dst_info.start + dst_info.length;
    dst_info.distance = elmt_size;
 
    _mpi_scalar_mput(target_rank,
                     dst_desc, src,
                     dst_offset, src_offset,
                     1 /*dst_dims*/,
                     &dst_info,
                     is_dst_on_acc);
  }else{
    _XMP_fatal("Coarray Error ! transfer size is wrong.\n");
  }
}

Here is the caller graph for this function:

_XMP_mpi_destroy_shift_queue()

void _XMP_mpi_destroy_shift_queue

(

bool

is_acc

)

Destroy shift queue.

{
  struct _shift_queue_t *shift_queue = is_acc? &_shift_queue_acc : &_shift_queue;
 
  _XMP_free(shift_queue->shifts);
  shift_queue->shifts = NULL;
}

Here is the caller graph for this function:

_XMP_mpi_get()

void _XMP_mpi_get	(	const int	src_contiguous,
		const int	dst_contiguous,
		const int	org_target_rank,
		const int	src_dims,
		const int	dst_dims,
		const _XMP_array_section_t *	src_info,
		const _XMP_array_section_t *	dst_info,
		const _XMP_coarray_t *	src_desc,
		void *	dst,
		const int	src_elmts,
		const int	dst_elmts,
		const int	is_src_on_acc
	)

{
  size_t dst_offset = _XMP_get_offset(dst_info, dst_dims);
  size_t src_offset = _XMP_get_offset(src_info, src_dims);
  size_t transfer_size = src_desc->elmt_size * src_elmts;
 
  const int target_rank = _XMP_mpi_trans_rank(src_desc, org_target_rank);
 
  XACC_DEBUG("_XMP_mpi_get, dst_elmts = %d, src_elmts = %d\n", src_elmts, dst_elmts);
 
  if(src_elmts == dst_elmts){
    if(src_contiguous == _XMP_N_INT_TRUE && dst_contiguous == _XMP_N_INT_TRUE){
      _mpi_contiguous(_XMP_N_COARRAY_GET, 
                      target_rank,
                      src_desc, dst,
                      src_offset, dst_offset,
                      transfer_size, is_src_on_acc);
    }else{
      _mpi_non_contiguous(_XMP_N_COARRAY_GET, target_rank,
                          src_desc, dst,
                          src_offset, dst_offset,
                          src_dims, dst_dims,
                          src_info, dst_info,
                          src_elmts, is_src_on_acc);
    }
  }else if(src_elmts == 1){
    _mpi_scalar_mget(target_rank,
                     dst, src_desc,
                     dst_offset, src_offset,
                     dst_dims,
                     dst_info,
                     is_src_on_acc);
  }else{
    _XMP_fatal("Number of elements is invalid");
  }
}

Here is the call graph for this function:

_XMP_mpi_put()

void _XMP_mpi_put	(	const int	dst_contiguous,
		const int	src_contiguous,
		const int	org_target_rank,
		const int	dst_dims,
		const int	src_dims,
		const _XMP_array_section_t *	dst_info,
		const _XMP_array_section_t *	src_info,
		const _XMP_coarray_t *	dst_desc,
		const void *	src,
		const int	dst_elmts,
		const int	src_elmts,
		const int	is_dst_on_acc
	)

{
  size_t dst_offset = _XMP_get_offset(dst_info, dst_dims);
  size_t src_offset = _XMP_get_offset(src_info, src_dims);
  size_t transfer_size = dst_desc->elmt_size * dst_elmts;
 
  const int target_rank = _XMP_mpi_trans_rank(dst_desc, org_target_rank);
 
  if(dst_elmts == src_elmts){
    if(dst_contiguous == _XMP_N_INT_TRUE && src_contiguous == _XMP_N_INT_TRUE){
      _mpi_contiguous(_XMP_N_COARRAY_PUT,
                      target_rank,
                      dst_desc, src,
                      dst_offset, src_offset,
                      transfer_size, is_dst_on_acc);
    }
    else{
      _mpi_non_contiguous(_XMP_N_COARRAY_PUT, target_rank,
                          dst_desc, src,
                          dst_offset, src_offset,
                          dst_dims, src_dims,
                          dst_info, src_info,
                          dst_elmts, is_dst_on_acc);
    }
  }
  else{
    if(src_elmts == 1){
      _mpi_scalar_mput(target_rank,
                       dst_desc, src,
                       dst_offset, src_offset,
                       dst_dims,
                       dst_info,
                       is_dst_on_acc);
    }
    else{
      _XMP_fatal("Number of elements is invalid");
    }
  }
}

Here is the call graph for this function:

_XMP_mpi_sync_all()

void _XMP_mpi_sync_all

(

)

Execute sync_all.

{
  _XMP_mpi_sync_memory();
  _XMP_barrier_EXEC();
  _XMP_mpi_sync_memory();
}

Here is the call graph for this function:

Here is the caller graph for this function:

_XMP_mpi_sync_images()

void _XMP_mpi_sync_images	(	const int	num,
		const int *	image_set,
		int *	status
	)

Execute sync images.

Parameters

[in]	num	number of nodes
[in]	*image_set	image set
[out]	status	status

{
  _XMP_mpi_sync_memory();
 
  if(num == 0){
    return;
  }
  else if(num < 0){
    fprintf(stderr, "Invalid value is used in xmp_sync_memory. The first argument is %d\n", num);
    _XMP_fatal_nomsg();
  }
 
  _notify_sync_images(num, image_set);
  _wait_sync_images(num, image_set);
 
  _XMP_mpi_sync_memory();
}

Here is the call graph for this function:

Here is the caller graph for this function:

_XMP_mpi_sync_memory()

void _XMP_mpi_sync_memory

(

)

Execute sync_memory.

{
  if(_XMP_flag_multi_win){
    int num = 0;
    _XMP_coarray_t **coarrays = _XMP_coarray_get_list(&num);
    for(int i = 0; i < num; i++){
      MPI_Win win = coarrays[i]->win;
      if(win != MPI_WIN_NULL){
        XACC_DEBUG("flush_all for host a coarray (%ld)", (long)win);
        MPI_Win_flush_all(win);
        XACC_DEBUG("sync for host a coarray (%ld)", (long)win);
        MPI_Win_sync(win);
      }
#ifdef _XMP_XACC
      MPI_Win win_acc = coarrays[i]->win_acc;
      if(win_acc != MPI_WIN_NULL){
        XACC_DEBUG("flush_all for acc a coarray (%ld)", (long)win_acc);
        MPI_Win_flush_all(win_acc);
        XACC_DEBUG("sync for acc a coarray (%ld)", (long)win_acc);
        MPI_Win_sync(win_acc);
      }
#endif
    }
  }else{
    if(! _is_coarray_win_flushed){
      XACC_DEBUG("flush_all for host single coarray(%ld)", (long)_xmp_mpi_onesided_win);
      MPI_Win_flush_all(_xmp_mpi_onesided_win);
 
      _is_coarray_win_flushed = true;
    }
 
    if(! _is_distarray_win_flushed){
      XACC_DEBUG("flush_all for host single distarray(%ld)", (long)_xmp_mpi_distarray_win);
      MPI_Win_flush_all(_xmp_mpi_distarray_win);
 
      _is_distarray_win_flushed = true;
    }
 
#ifdef _XMP_XACC
    if(! _is_coarray_win_acc_flushed){
      XACC_DEBUG("flush_all for acc single coarray(%ld)", (long)_xmp_mpi_onesided_win_acc);
      MPI_Win_flush_all(_xmp_mpi_onesided_win_acc);
 
      _is_coarray_win_acc_flushed = true;
    }
 
    if(! _is_distarray_win_acc_flushed){
      XACC_DEBUG("flush_all for acc single distarray(%ld)", (long)_xmp_mpi_distarray_win_acc);
      MPI_Win_flush_all(_xmp_mpi_distarray_win_acc);
 
      _is_distarray_win_acc_flushed = true;
    }
#endif
 
    _win_sync();
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

_XMP_sync_images_COMM()

void _XMP_sync_images_COMM	(	MPI_Comm *	comm,
		int *	status
	)

{
  _XMP_mpi_sync_memory();
  MPI_Barrier(*comm);
}

_XMP_sync_images_EXEC()

void _XMP_sync_images_EXEC

(

int *

status

)

{
  _XMP_mpi_sync_memory();
  _XMP_barrier_EXEC();
}

Here is the call graph for this function:

Variable Documentation

_XMP_flag_get_nb

int _XMP_flag_get_nb

extern

_XMP_flag_multi_win

int _XMP_flag_multi_win

extern

_XMP_flag_put_nb

int _XMP_flag_put_nb

extern

These variables are temporral.