scale_meshfieldcomm_cubedspheredom2d.F90

Go to the documentation of this file.

!-------------------------------------------------------------------------------
!> module FElib / Data / Communication in 2D cubed-sphere domain
!!
!! @par Description
!!      A module to manage data communication with 2D cubed-sphere domain for element-based methods
!!
!! @author Yuta Kawai, Team SCALE
!<
#include "scaleFElib.h"
module scale_meshfieldcomm_cubedspheredom2d
 
  !-----------------------------------------------------------------------------
  !
  !++ used modules
  !
  use scale_precision
  use scale_io
 
  use scale_element_base, only: elementbase, elementbase2d
  use scale_mesh_cubedspheredom2d, only: meshcubedspheredom2d
  use scale_meshfield_base, only: meshfield2d
  use scale_meshfieldcomm_base, only: &
    meshfieldcommbase,                               &
    meshfieldcommbase_init, meshfieldcommbase_final, &
    meshfieldcommbase_extract_bounddata,             &
    meshfieldcommbase_extract_bounddata2,            &
    meshfieldcommbase_extract_bounddata_2,           &
    meshfieldcommbase_extract_bounddata_3,           &
    meshfieldcommbase_set_bounddata,                 &
    meshfieldcommbase_set_bounddata_3,               &
    meshfieldcontainer
  use scale_localmesh_2d, only: localmesh2d
   
  !-----------------------------------------------------------------------------
  implicit none
  private
 
  !-----------------------------------------------------------------------------
  !
  !++ Public type & procedure
  ! 
 
  !> Derived type to represent covariant vector components
  type :: veccovariantcomp
    type(MeshField2D), pointer :: u1 => null()
    type(MeshField2D), pointer :: u2 => null()
  end type    
 
  !> Base derived type to manage data communication with 2D cubed-sphere domain
  type, public, extends(meshfieldcommbase) :: meshfieldcommcubedspheredom2d
    class(meshcubedspheredom2d), pointer :: mesh2d                         !< Pointer to an object representing 2D cubed-sphere computational mesh
    type(veccovariantcomp), allocatable :: vec_covariant_comp_ptrlist(:)
 
    integer :: halosize_1d !< Halo size for 1D direction
  contains
    procedure, public :: init   => meshfieldcommcubedspheredom2d_init
    procedure, public :: put    => meshfieldcommcubedspheredom2d_put
    procedure, public :: get    => meshfieldcommcubedspheredom2d_get
    procedure, public :: exchange => meshfieldcommcubedspheredom2d_exchange  
    procedure, public :: setcovariantvec => meshfieldcommcubedspheredom2d_set_covariantvec
    procedure, public :: final => meshfieldcommcubedspheredom2d_final
  end type meshfieldcommcubedspheredom2d
 
  !-----------------------------------------------------------------------------
  !
  !++ Public parameters & variables
  !
  
  !-----------------------------------------------------------------------------
  !
  !++ Private type & procedure
  !
  private :: post_exchange_core
  private :: push_localsendbuf
  
  !-----------------------------------------------------------------------------
  !
  !++ Private parameters & variables
  !
  integer, parameter :: comm_face_num = 4
 
contains
!> Initialize an object to manage data communication with 2D cubed-sphere computational mesh
  subroutine meshfieldcommcubedspheredom2d_init( this, &
    sfield_num, hvfield_num, htensorfield_num, mesh2d, &
    haloSize_1D )
 
    use scale_meshutil_cubedsphere2d, only: meshutilcubedsphere2d_genpatchboundarymap_wide
    implicit none
    
    class(meshfieldcommcubedspheredom2d), intent(inout) :: this
    integer, intent(in) :: sfield_num
    integer, intent(in) :: hvfield_num
    integer, intent(in) :: htensorfield_num
    class(meshcubedspheredom2d), intent(in), target :: mesh2d
    integer, intent(in), optional :: halosize_1d
    
    type(localmesh2d), pointer :: lcmesh
    type(elementbase2d), pointer :: elem
    integer :: n
    integer :: nnode_lcmeshface(comm_face_num,mesh2d%local_mesh_num)
    !-----------------------------------------------------------------------------
    
    this%mesh2d => mesh2d
    lcmesh => mesh2d%lcmesh_list(1)
    elem => lcmesh%refElem2D
 
    !-
    if ( present(halosize_1d) ) then
      this%haloSize_1D = halosize_1d
    else
      this%haloSize_1D = 1
    end if
 
    !-
    allocate( this%VMapB_size(this%mesh2d%LOCAL_MESH_NUM) )
 
    this%bufsize_per_field = 2*(lcmesh%NeX + lcmesh%NeY) * elem%Nfp*this%haloSize_1D
 
    do n=1, this%mesh2d%LOCAL_MESH_NUM
      lcmesh => this%mesh2d%lcmesh_list(n)
      nnode_lcmeshface(:,n) = (/ lcmesh%NeX, lcmesh%NeY, lcmesh%NeX, lcmesh%NeY /) * lcmesh%refElem2D%Nfp*this%haloSize_1D
    end do
 
    call meshfieldcommbase_init( this, sfield_num, hvfield_num, htensorfield_num, this%bufsize_per_field, comm_face_num, nnode_lcmeshface, mesh2d)  
 
    !-
    if ( this%haloSize_1D > 1 ) then
      this%use_vmap_wide_flag = .true.
      allocate( this%VMapB2(this%bufsize_per_field) )
 
      lcmesh => this%mesh2d%lcmesh_list(1)      
      call meshutilcubedsphere2d_genpatchboundarymap_wide( this%VMapB2, &
        lcmesh%VMapB, this%haloSize_1D,                      &
        lcmesh%NeX, lcmesh%NeY,                              &
        elem%Nfp )
      !$acc enter data copyin(this%VMapB2)
    else
      this%use_vmap_wide_flag = .false.
    end if
 
    do n=1, this%mesh2d%LOCAL_MESH_NUM
      lcmesh => this%mesh2d%lcmesh_list(n)
      if ( this%use_vmap_wide_flag ) then
        this%VMapB_size(n) = size(this%VMapB2)        
      else
        this%VMapB_size(n) = size(lcmesh%VMapB)
      end if
    end do
    !$acc enter data copyin(this%VMapB_size)
 
    !-
    if (hvfield_num > 0) then
      allocate( this%vec_covariant_comp_ptrlist(hvfield_num) )
    end if
 
    return
  end subroutine meshfieldcommcubedspheredom2d_init
 
!> Finalize an object to manage data communication with 2D cubed-sphere computational mesh
  subroutine meshfieldcommcubedspheredom2d_final( this )
    implicit none
    
    class(meshfieldcommcubedspheredom2d), intent(inout) :: this
    !-----------------------------------------------------------------------------
 
    if ( this%hvfield_num > 0 ) then
      deallocate( this%vec_covariant_comp_ptrlist )
    end if
 
    call meshfieldcommbase_final( this )
    return
  end subroutine meshfieldcommcubedspheredom2d_final
 
!> Register objects to manage covariant vector components
  subroutine meshfieldcommcubedspheredom2d_set_covariantvec( &
    this, hvfield_ID, u1, u2  )
    implicit none
    class(meshfieldcommcubedspheredom2d), intent(inout) :: this
    integer, intent(in) :: hvfield_id
    type(meshfield2d), intent(in), target :: u1
    type(meshfield2d), intent(in), target :: u2
    !--------------------------------------------------------------
 
    this%vec_covariant_comp_ptrlist(hvfield_id)%u1 => u1
    this%vec_covariant_comp_ptrlist(hvfield_id)%u2 => u2
    return
  end subroutine meshfieldcommcubedspheredom2d_set_covariantvec
 
!> Put field data into temporary buffers
  subroutine meshfieldcommcubedspheredom2d_put(this, field_list, varid_s)
    implicit none
    class(meshfieldcommcubedspheredom2d), intent(inout) :: this
    type(meshfieldcontainer), intent(in) :: field_list(:)
    integer, intent(in) :: varid_s
    !-----------------------------------------------------------------------------
 
!    call PROF_rapstart( 'comm_put', 2)
 
    if ( this%use_vmap_wide_flag ) then
       call meshfieldcommbase_extract_bounddata_3( field_list, 2, varid_s, this%mesh2d%lcmesh_list, this%VMapB2, this%VMapB_size(1), this%send_buf )
    else
      call meshfieldcommbase_extract_bounddata_2( &
        field_list, 2, varid_s, this%mesh2d%lcmesh_list, size(this%mesh2d%lcmesh_list(1)%VMapB), & !(in)
        this%send_buf ) ! (out)
    end if
 
!    call PROF_rapend( 'comm_put', 2)
    return
  end subroutine meshfieldcommcubedspheredom2d_put
 
!> Extract field data from temporary buffers
  subroutine meshfieldcommcubedspheredom2d_get(this, field_list, varid_s)
    use scale_meshfieldcomm_base, only: &
      meshfieldcommbase_wait_core
    implicit none
    
    class(meshfieldcommcubedspheredom2d), intent(inout) :: this
    type(meshfieldcontainer), intent(inout) :: field_list(:)
    integer, intent(in) :: varid_s
 
    integer :: i
    integer :: n
    type(localmesh2d), pointer :: lcmesh
 
    integer :: varnum
    integer :: varid_e
    integer :: varid_vec_s
 
    integer, allocatable :: vmapb(:)
    !-----------------------------------------------------------------------------
!    call PROF_rapstart( 'comm_get', 2)
    varnum = size(field_list) 
 
    !--
    if ( this%call_wait_flag_sub_get ) then
      ! This workflow should be reconsidered in near future.
      ! The coordinate convresions for horizontal vector/tensor fields in post_exchange_core are applied for the received data (this%recv_buf).
      ! However, MeshFieldCommBase_wait_core does not store the communication data into this%recv_buf, but directly into the field_list.
      call meshfieldcommbase_wait_core( this, this%commdata_list )
      call post_exchange_core( this )
    else
      do i=1, varnum
      do n=1, this%mesh2d%LOCAL_MESH_NUM
        lcmesh => this%mesh2d%lcmesh_list(n)
        if ( this%use_vmap_wide_flag ) then
          call meshfieldcommbase_set_bounddata_3( this%recv_buf(:,varid_s+i-1,n), lcmesh%refElem, lcmesh, & ! (in)
            this%VMapB2, this%VMapB_size(1),                                                              & ! (in)
            field_list(i)%field2d%local(n)%val )                                                            ! (inout)
        else        
          call meshfieldcommbase_set_bounddata( this%recv_buf(:,varid_s+i-1,n), lcmesh%refElem, lcmesh, & !(in)
            field_list(i)%field2d%local(n)%val )                                                         !(out)
        end if
      end do
      end do
    end if
 
    varid_e = varid_s + varnum - 1
    if ( varid_e > this%sfield_num ) then
      do i=1, this%hvfield_num
 
        varid_vec_s = this%sfield_num + 2*i - 1 
        if ( varid_vec_s > varid_e ) exit
 
        if (       associated(this%vec_covariant_comp_ptrlist(i)%u1 ) &
            .and.  associated(this%vec_covariant_comp_ptrlist(i)%u2 ) ) then
        
          do n=1, this%mesh2d%LOCAL_MESH_NUM
            lcmesh => this%mesh2d%lcmesh_list(n)
 
            if ( this%use_vmap_wide_flag ) then
              allocate(vmapb(this%VMapB_size(n)))
              vmapb(:) = this%VMapB2(:)
            else
              allocate(vmapb(size(lcmesh%VMapB)))
              vmapb(:) = lcmesh%VMapB(:)
            end if
            !$acc enter data copyin(VMapB) async(1)
            call set_boundary_data2d_u1u2( &
              this%recv_buf(:,varid_vec_s,n), this%recv_buf(:,varid_vec_s+1,n),     & ! (in)
              this%bufsize_per_field, lcmesh%refElem2D, lcmesh, lcmesh%G_ij, vmapb, & ! (in)
              this%vec_covariant_comp_ptrlist(i)%u1%local(n)%val,                   & ! (out)
              this%vec_covariant_comp_ptrlist(i)%u2%local(n)%val                    ) ! (out)
            !$acc exit data delete(VMapB) async(1)
            deallocate( vmapb )
          end do
       end if
      end do
    end if
    !$acc wait(1)
 
!    call PROF_rapend( 'comm_get', 2)
    return
  end subroutine meshfieldcommcubedspheredom2d_get
 
  !> Exchange field data between neighboring MPI processes
  !!
  !! @param do_wait Flag whether MPI_waitall is called and move tmp data of LocalMeshCommData object to a recv buffer
!OCL SERIAL
  subroutine meshfieldcommcubedspheredom2d_exchange( this, do_wait )    
    use scale_meshfieldcomm_base, only: &
      meshfieldcommbase_exchange_core,  &
      localmeshcommdata
    use scale_cubedsphere_coord_cnv, only: &
      cubedspherecoordcnv_cs2lonlatvec, &
      cubedspherecoordcnv_lonlat2csvec
    use scale_prc, only: prc_abort
    implicit none
  
    class(meshfieldcommcubedspheredom2d), intent(inout), target :: this
    logical, intent(in), optional :: do_wait
 
    integer :: n, f
    integer :: varid
    integer :: i
 
    real(rp), allocatable :: fpos2d(:,:)
    real(rp), allocatable :: lcfpos2d(:,:)
    real(rp), allocatable :: unity_fac(:)
    real(rp), allocatable :: tmp_svec2d(:,:)
    
    class(elementbase2d), pointer :: elem
    type(localmesh2d), pointer :: lcmesh
    type(localmeshcommdata), pointer :: commdata
 
    integer, pointer :: vmapb(:)
    !-----------------------------------------------------------------------------
 
    ! call PROF_rapstart( 'comm_exchange', 2)
 
    if ( present(do_wait) ) then
      if ( .not. do_wait ) then
        log_info("MeshFieldCommCubedSphereDom2D_exchange",*) "do_wait=False is not currently supported. Check!"
        call prc_abort
      end if
    end if
 
    do n=1, this%mesh%LOCAL_MESH_NUM
      lcmesh => this%mesh2d%lcmesh_list(n)
      elem => lcmesh%refElem2D
 
      allocate( fpos2d(this%Nnode_LCMeshAllFace(n),2) )
 
      if ( this%use_vmap_wide_flag ) then
        vmapb => this%VMapB2
      else
        vmapb => lcmesh%VMapB
      end if
      !$acc data create(fpos2D) copyin(VMapB)
 
      call meshfieldcommbase_extract_bounddata2( lcmesh%pos_en(:,:,1), vmapb, size(vmapb), elem%Np*lcmesh%Ne, fpos2d(:,1) )
      call meshfieldcommbase_extract_bounddata2( lcmesh%pos_en(:,:,2), vmapb, size(vmapb), elem%Np*lcmesh%Ne, fpos2d(:,2) )
 
      do f=1, this%nfaces_comm
        commdata => this%commdata_list(f,n)
 
        call push_localsendbuf( commdata%send_buf,                  & ! (inout)
          this%send_buf(:,:,n), commdata%s_faceID, this%is_f(f,n),  & ! (in)
          commdata%Nnode_LCMeshFace, this%bufsize_per_field,        & ! (in)
          this%field_num_tot, lcmesh, this%HaloSize_1D              ) ! (in)
        
        if ( commdata%s_panelID /= lcmesh%panelID &
            .and. ( this%hvfield_num > 0 .or. this%htensorfield_num > 0 ) ) then
          
          allocate( lcfpos2d(commdata%Nnode_LCMeshFace,2), unity_fac(commdata%Nnode_LCMeshFace) )
          unity_fac(:) = 1.0_rp
          !$acc enter data create(lcfpos2D) copyin(unity_fac) async(1)
 
          call push_localsendbuf( lcfpos2d,                            &
            fpos2d, commdata%s_faceID, this%is_f(f,n),                 &
            commdata%Nnode_LCMeshFace, this%Nnode_LCMeshAllFace(n), 2, &
            lcmesh, this%HaloSize_1D )
        end if
 
        if ( commdata%s_panelID /= lcmesh%panelID ) then
          if ( this%hvfield_num > 0 ) then
            allocate( tmp_svec2d(commdata%Nnode_LCMeshFace,2) )
            !$acc enter data create(tmp_svec2D) async(1)
 
            do varid=this%sfield_num+1, this%field_num_tot-1,2
              !$acc parallel loop present(tmp_svec2D, commdata%send_buf) async(1)
              do i=1, size(tmp_svec2d,1)
                tmp_svec2d(i,1) = commdata%send_buf(i,varid  )
                tmp_svec2d(i,2) = commdata%send_buf(i,varid+1)
              end do
 
              call cubedspherecoordcnv_cs2lonlatvec( &
                lcmesh%panelID, lcfpos2d(:,1), lcfpos2d(:,2), unity_fac,  & ! (in)
                commdata%Nnode_LCMeshFace,                                & ! (in)
                tmp_svec2d(:,1), tmp_svec2d(:,2),                         & ! (in)
                commdata%send_buf(:,varid), commdata%send_buf(:,varid+1), & ! (out)
                gpu_async_id=1 ) ! (in)
            end do
            !$acc exit data delete(tmp_svec2D) async(1)
            deallocate( tmp_svec2d )
          end if
        end if
        
 
        if ( allocated(lcfpos2d) ) then
          !$acc exit data delete(lcfpos2D, unity_fac) async(1)
          deallocate( lcfpos2d, unity_fac )
        end if        
      end do
      !$acc end data
      deallocate( fpos2d )
    end do
    !$acc wait(1)
 
    !-----------------------
 
    call meshfieldcommbase_exchange_core(this, this%commdata_list, do_wait )
    if ( .not. this%call_wait_flag_sub_get ) &
      call post_exchange_core( this )
 
    ! call PROF_rapend( 'comm_exchange', 2)
 
    return
  end subroutine meshfieldcommcubedspheredom2d_exchange
 
!- Private ---------------------------
 
  subroutine post_exchange_core( this )
    use scale_meshfieldcomm_base, only: localmeshcommdata
    use scale_cubedsphere_coord_cnv, only: &
      cubedspherecoordcnv_lonlat2csvec
    implicit none
 
    class(meshfieldcommcubedspheredom2d), intent(inout), target :: this
 
    integer :: n, f
    integer :: varid
 
    real(rp), allocatable :: fpos2d(:,:)
    real(rp), allocatable :: lcfpos2d(:,:)
    real(rp), allocatable :: unity_fac(:)
    
    class(elementbase2d), pointer :: elem
    type(localmesh2d), pointer :: lcmesh
    type(localmeshcommdata), pointer :: commdata
 
    integer :: irs, ire    
    integer, pointer :: vmapb(:)
    !-----------------------------------------------------------------------------
 
    do n=1, this%mesh%LOCAL_MESH_NUM
      lcmesh => this%mesh2d%lcmesh_list(n)
      elem => lcmesh%refElem2D
 
      allocate( fpos2d(this%Nnode_LCMeshAllFace(n),2) )
      if ( this%use_vmap_wide_flag ) then
        vmapb => this%VMapB2
      else
        vmapb => lcmesh%VMapB
      end if
      !$acc data create(fpos2D) copyin(VMapB)
 
      call meshfieldcommbase_extract_bounddata2( lcmesh%pos_en(:,:,1), vmapb, size(vmapb), elem%Np*lcmesh%Ne, fpos2d(:,1) )
      call meshfieldcommbase_extract_bounddata2( lcmesh%pos_en(:,:,2), vmapb, size(vmapb), elem%Np*lcmesh%Ne, fpos2d(:,2) )
 
      irs = 1
      do f=1, this%nfaces_comm
        commdata => this%commdata_list(f,n)
        ire = irs + commdata%Nnode_LCMeshFace - 1
 
        if ( commdata%s_panelID /= lcmesh%panelID &
            .and. ( this%hvfield_num > 0 .or. this%htensorfield_num > 0 ) ) then
          
          allocate( lcfpos2d(commdata%Nnode_LCMeshFace,2), unity_fac(commdata%Nnode_LCMeshFace) )
          unity_fac(:) = 1.0_rp
          !$acc enter data create(lcfpos2D) copyin(unity_fac) async(1)
 
          call push_localsendbuf( lcfpos2d,                            &
            fpos2d, f, this%is_f(f,n),                                 &
            commdata%Nnode_LCMeshFace, this%Nnode_LCMeshAllFace(n), 2, &
            lcmesh, this%HaloSize_1D )
        end if
 
 
        if ( commdata%s_panelID /= lcmesh%panelID ) then
          if ( this%hvfield_num > 0 ) then
            do varid=this%sfield_num+1, this%field_num_tot-1, 2
              call cubedspherecoordcnv_lonlat2csvec( &
                lcmesh%panelID, lcfpos2d(:,1), lcfpos2d(:,2), unity_fac(:),          &
                commdata%Nnode_LCMeshFace,                                           &
                commdata%recv_buf(:,varid), commdata%recv_buf(:,varid+1),            &
                this%recv_buf(irs:ire,varid,n), this%recv_buf(irs:ire,varid+1,n)     )
            end do
          end if
        end if
 
        irs = ire + 1
        if ( allocated(lcfpos2d) ) then
          !$acc exit data delete(lcfpos2D, unity_fac) async(1)
          deallocate( lcfpos2d, unity_fac )
        end if
      end do
 
      !$acc end data
      deallocate( fpos2d )
    end do
    !$acc wait(1)
 
    return    
  end subroutine post_exchange_core
 
  !> Push temporary buffer of the local send data to the communication buffer
  subroutine push_localsendbuf( lc_send_buf, &
    send_buf, s_faceID, is, Nnode_LCMeshFace, bufsize_per_field, var_num, &
    lcmesh, haloSize_1D )
    implicit none
 
    integer, intent(in) :: var_num
    integer, intent(in) :: nnode_lcmeshface
    integer, intent(in) :: bufsize_per_field
    real(rp), intent(inout) :: lc_send_buf(nnode_lcmeshface,var_num)
    real(rp), intent(in) :: send_buf(bufsize_per_field,var_num)
    integer, intent(in) :: s_faceid, is
    type(localmesh2d), intent(in) :: lcmesh
    integer, intent(in) :: halosize_1d
 
    integer :: i, v
    integer :: ne_h1d
    integer :: nfp
    !-----------------------------------------------------------------------------
 
    if ( s_faceid > 0 ) then
      !$omp parallel do
      !$acc parallel loop collapse(2) present(lc_send_buf, send_buf) async(1)
      do v=1, var_num
      do i=1, nnode_lcmeshface
        lc_send_buf(i,v) = send_buf(is+i-1,v)
      end do
      end do
    else if ( s_faceid < 0 ) then
      nfp = lcmesh%refElem2D%Nfp
      ne_h1d = nnode_lcmeshface / ( nfp * halosize_1d )
      call revert_hori( lc_send_buf, &
        send_buf, is, ne_h1d, halosize_1d, nfp )
    end if
 
    return
  contains
    !> Revert the horizontal ordering of the local send buffer for negative s_faceID
    subroutine revert_hori( revert, ori, is_, Ne_h1D_, haloSize_1D_, Nfp_ )
      implicit none
      integer, intent(in) :: is_
      integer, intent(in) :: ne_h1d_
      integer, intent(in) :: halosize_1d_
      integer, intent(in) :: nfp_
 
      real(rp), intent(out) :: revert(nfp_, halosize_1d_, ne_h1d_, var_num)
      real(rp), intent(in) :: ori(bufsize_per_field,var_num)
 
      integer :: p, ph, i, v
      integer :: p_, i_
      integer :: id_src
      !---------------------------------------------------------------------------
 
      !$acc parallel loop collapse(4) present(revert, ori) async(1)
      do v=1, var_num
      do i=1, ne_h1d_
      do ph=1, halosize_1d_
      do p=1, nfp_
        ! reverse orientation along the face
        i_ = ne_h1d_ - i + 1
        p_ = nfp_ - p + 1
        ! source ordering: (p, ph, i)
        id_src = is_ - 1 &
          + p_ &
          + (ph-1) * nfp_ &
          + (i_-1) * nfp_ * halosize_1d_
 
        revert(p,ph,i,v) = ori(id_src,v)
      end do
      end do
      end do
      end do
      return
    end subroutine revert_hori
  end subroutine push_localsendbuf
 
  subroutine set_boundary_data2d_u1u2( buf_U, buf_V, &
    bufsize_per_field, elem, mesh, G_ij, VMapB,      &
    u1, u2)
  
    implicit none
    integer, intent(in) :: bufsize_per_field  
    type(elementbase2d), intent(in) :: elem
    type(localmesh2d), intent(in) :: mesh
    real(dp), intent(in) :: buf_u(bufsize_per_field)
    real(dp), intent(in) :: buf_v(bufsize_per_field)
    real(dp), intent(in) :: g_ij(elem%np * mesh%ne,2,2)
    integer, intent(in) :: vmapb(bufsize_per_field)
    real(dp), intent(inout) :: u1(elem%np * mesh%nea)
    real(dp), intent(inout) :: u2(elem%np * mesh%nea)
 
    integer :: ii, iis
    integer :: vmap_b
    !------------------------------------------------------------
 
#ifdef _OPENACC
    iis = elem%Np*mesh%NeE
    !$acc parallel loop present(buf_U, buf_V, G_ij, u1, u2, VMapB)
    do ii=1, bufsize_per_field
      vmap_b = vmapb(ii)
      u1(iis+ii) = g_ij(vmap_b,1,1) * buf_u(ii) + g_ij(vmap_b,1,2) * buf_v(ii)
      u2(iis+ii) = g_ij(vmap_b,2,1) * buf_u(ii) + g_ij(vmap_b,2,2) * buf_v(ii)
    end do
#else    
    u1(elem%Np*mesh%NeE+1:elem%Np*mesh%NeE+bufsize_per_field) &
      = g_ij(vmapb,1,1) * buf_u(:) + g_ij(vmapb,1,2) * buf_v(:)
    u2(elem%Np*mesh%NeE+1:elem%Np*mesh%NeE+bufsize_per_field) &
      = g_ij(vmapb,2,1) * buf_u(:) + g_ij(vmapb,2,2) * buf_v(:)
#endif
    return
  end subroutine set_boundary_data2d_u1u2
 
end module scale_meshfieldcomm_cubedspheredom2d