[Draft] OpenACC port of halo exchanges

src/core_atmosphere/Registry.xml

@@ -443,6 +443,10 @@
                      units="-"
                      description="Method to use for exchanging halos"
                      possible_values="`mpas_dmpar', `mpas_halo'"/>
+               <nml_option name="config_gpu_aware_mpi" type="logical" default_value="false"
+                     units="-"
+                     description="Whether to use GPU-aware MPI for halo exchanges"
+                     possible_values=".true. or .false."/>
         </nml_record>
 
 #ifdef MPAS_USE_MUSICA

src/core_atmosphere/mpas_atm_halos.F

@@ -48,18 +48,23 @@ subroutine atm_build_halo_groups(domain, ierr)
 
       ! Local variables
       character(len=StrKIND), pointer :: config_halo_exch_method
+      logical, pointer :: config_gpu_aware_mpi
 
 
       !
       ! Determine from the namelist option config_halo_exch_method which halo exchange method to employ
       !
       call mpas_pool_get_config(domain % blocklist % configs, 'config_halo_exch_method', config_halo_exch_method)
+      call mpas_pool_get_config(domain % blocklist % configs, 'config_gpu_aware_mpi', config_gpu_aware_mpi)
 
       if (trim(config_halo_exch_method) == 'mpas_dmpar') then
          call mpas_log_write('')
          call mpas_log_write('*** Using ''mpas_dmpar'' routines for exchanging halos')
          call mpas_log_write('')
 
+         if (config_gpu_aware_mpi) then
+            call mpas_log_write('GPU-aware MPI is not presently supported with config_halo_exch_method = mpas_dmpar',MPAS_LOG_CRIT)
+         end if
          !
          ! Set up halo exchange groups used during atmosphere core initialization
          !

src/framework/Makefile

@@ -112,7 +112,7 @@ mpas_c_interfacing.o:
 xml_stream_parser.o: xml_stream_parser.c
 	$(CC) $(CFLAGS) $(CPPFLAGS) $(CPPINCLUDES) -I../external/ezxml -c xml_stream_parser.c
 
-mpas_halo.o: mpas_derived_types.o mpas_pool_routines.o mpas_log.o
+mpas_halo.o: mpas_derived_types.o mpas_pool_routines.o mpas_log.o mpas_timer.o
 
 mpas_ptscotch_interface.o : mpas_derived_types.o mpas_log.o
 

src/framework/mpas_dmpar.F

@@ -7448,19 +7448,28 @@ end subroutine mpas_dmpar_exch_group_end_halo_exch!}}}
 !>  exchange is complete.
 !
 !-----------------------------------------------------------------------
-   subroutine mpas_dmpar_exch_group_full_halo_exch(domain, groupName, iErr)!{{{
+   subroutine mpas_dmpar_exch_group_full_halo_exch(domain, groupName, withGPUAwareMPI, iErr)!{{{
 
       type (domain_type), intent(inout) :: domain
       character (len=*), intent(in) :: groupName
+      logical, optional, intent(in) :: withGPUAwareMPI
       integer, optional, intent(out) :: iErr
 
       type (mpas_exchange_group), pointer :: exchGroupPtr
       integer :: nLen
+      logical :: useGPUAwareMPI
 
       if ( present(iErr) ) then
          iErr = MPAS_DMPAR_NOERR
       end if
 
+      useGPUAwareMPI = .false.
+      if (present(withGPUAwareMPI)) then
+        if (withGPUAwareMPI) then
+           call mpas_log_write(' GPU-aware MPI not implemented in this module', MPAS_LOG_CRIT)
+        end if
+      end if
+
       nLen = len_trim(groupName)
       DMPAR_DEBUG_WRITE(' -- Trying to perform a full exchange for group ' // trim(groupName))