use default(none) for all OMP directives. PGI has a memory leak issue if...

src/ALE/MOM_ALE.F90

@@ -278,7 +278,7 @@ subroutine ALE_main( G, h, u, v, tv, CS )
   type(ALE_CS), intent(inout) :: CS ! Regridding parameters and options
 
   ! Local variables
-  integer :: nk
+  integer :: nk, i, j, k, isd, ied, jsd, jed
 
   ! Build new grid. The new grid is stored in h_new. The old grid is h.
   ! Both are needed for the subsequent remapping of variables.
@@ -289,8 +289,13 @@ subroutine ALE_main( G, h, u, v, tv, CS )
 
   ! Override old grid with new one. The new grid 'h_new' is built in
   ! one of the 'build_...' routines above.
-  nk = G%ke
-  h(:,:,:) = h(:,:,:) + ( CS%dzRegrid(:,:,1:nk) - CS%dzRegrid(:,:,2:nk+1) )
+  nk = G%ke; isd = G%isd; ied = G%ied; jsd = G%jsd; jed = G%jed
+!$OMP parallel do default(none) shared(isd,ied,jsd,jed,nk,h,CS)
+  do k = 1,nk
+    do j = jsd,jed ; do i = isd,ied   
+      h(i,j,k) = h(i,j,k) + ( CS%dzRegrid(i,j,k) - CS%dzRegrid(i,j,k+1) )
+    enddo ; enddo
+  enddo
 
 end subroutine ALE_main
 
@@ -332,7 +337,8 @@ subroutine pressure_gradient_plm( CS, S_t, S_b, T_t, T_b, G, tv, h )
   ! in 'ALE_memory_allocation'.
 
   ! Determine reconstruction within each column
-!$OMP parallel do default(shared) private(hTmp,ppoly_linear_E,ppoly_linear_coefficients,tmp)
+!$OMP parallel do default(none) shared(G,h,tv,CS,S_t,S_b,T_t,T_b)                     &
+!$OMP                          private(hTmp,ppoly_linear_E,ppoly_linear_coefficients,tmp)
   do j = G%jsc,G%jec+1
     do i = G%isc,G%iec+1
       ! Build current grid
@@ -406,7 +412,8 @@ subroutine pressure_gradient_ppm( CS, S_t, S_b, T_t, T_b, G, tv, h )
   ! in 'ALE_memory_allocation'.
 
   ! Determine reconstruction within each column
-!$OMP parallel do default(shared) private(hTmp,ppoly_parab_E,ppoly_parab_coefficients)
+!$OMP parallel do default(none) shared(G,h,tv,CS,S_t,S_b,T_t,T_b) &
+!$OMP                          private(hTmp,tmp,ppoly_parab_E,ppoly_parab_coefficients)
   do j = G%jsc,G%jec+1
     do i = G%isc,G%iec+1
 

src/ALE/MOM_regridding.F90

@@ -309,7 +309,8 @@ subroutine checkGridsMatch( G, h, dzInterface )
 
   nz = G%ke
   eps =1. ; eps = epsilon(eps)
-
+!$OMP parallel do default(none) shared(G,nz,h,dzInterface,eps) &
+!$OMP                          private(totalHold,zOld,totalHnewF,zNewF,hNewF)
   do j = G%jsc-1,G%jec+1
     do i = G%isc-1,G%iec+1
 
@@ -388,7 +389,9 @@ subroutine buildGridZstar( CS, G, h, dzInterface )
 
   nz = G%ke
 
-!$OMP parallel do default(private) shared(G,dzInterface,CS,nz,h)
+!$OMP parallel do default(none) shared(G,dzInterface,CS,nz,h)                    &
+!$OMP                          private(nominalDepth,totalThickness,minThickness, &
+!$OMP                                  eta,stretching,zNew,dh,zOld)
   do j = G%jsc-1,G%jec+1
     do i = G%isc-1,G%iec+1
 
@@ -456,8 +459,6 @@ subroutine buildGridZstar( CS, G, h, dzInterface )
         call MOM_error( FATAL, &
                'MOM_regridding, buildGridZstar: top surface has moved!!!' )
       endif
-      dzInterface(i,j,1) = 0.
-      dzInterface(i,j,nz+1) = 0.
 #endif
 
     end do

src/ALE/MOM_remapping.F90

@@ -119,7 +119,9 @@ subroutine remapping_main( CS, G, h, dxInterface, tv, u, v )
   nz = G%ke
 
   ! Remap tracer
-!$OMP parallel do default(shared) private(h1,dx,u_column)
+!$OMP parallel default(none) shared(G,h,dxInterface,CS,nz,tv,u,v) &
+!$OMP                       private(h1,dx,u_column)
+!$OMP do
   do j = G%jsc,G%jec
     do i = G%isc,G%iec
       if (G%mask2dT(i,j)>0.) then
@@ -136,7 +138,7 @@ subroutine remapping_main( CS, G, h, dxInterface, tv, u, v )
 
   ! Remap u velocity component
   if ( present(u) ) then
-!$OMP parallel do default(shared) private(h1,dx,u_column)
+!$OMP do
     do j = G%jsc,G%jec
       do i = G%iscB,G%iecB
         if (G%mask2dCu(i,j)>0.) then
@@ -152,7 +154,7 @@ subroutine remapping_main( CS, G, h, dxInterface, tv, u, v )
 
   ! Remap v velocity component
   if ( present(v) ) then
-!$OMP parallel do default(shared) private(h1,dx,u_column)
+!$OMP do
     do j = G%jscB,G%jecB
       do i = G%isc,G%iec
         if (G%mask2dCv(i,j)>0.) then
@@ -165,6 +167,7 @@ subroutine remapping_main( CS, G, h, dxInterface, tv, u, v )
       enddo
     enddo
   endif
+!$OMP end parallel
 
 end subroutine remapping_main
 

src/ALE/PLM_functions.F90

@@ -47,7 +47,7 @@ subroutine PLM_reconstruction( N, h, u, ppoly_E, ppoly_coefficients )
                                                 ! van Leer slopes   
   real          :: slope                ! retained PLM slope
   real          :: a, b                 ! auxiliary variables
-  real, dimension(:,:), allocatable :: E_old
+  real, dimension(N,2) :: E_old
 
   ! Loop on interior cells
   do k = 2,N-1
@@ -116,8 +116,7 @@ subroutine PLM_reconstruction( N, h, u, ppoly_E, ppoly_coefficients )
   ! Second pass: we need to check for nonmonotonic discontinuous edge values.
   ! When this occurs, the PLM slope is redefined so as to ensure monotonic edge
   ! values across edges.
-  allocate( E_old(N,2) )
-  E_old = ppoly_E
+  E_old(1:N,1:2) = ppoly_E(1:N,1:2)
 
   do k = 2,N-1
 
@@ -154,7 +153,6 @@ subroutine PLM_reconstruction( N, h, u, ppoly_E, ppoly_coefficients )
 
   end do ! end loop on interior cells
 
-  deallocate( E_old )
 
 end subroutine PLM_reconstruction
 

src/core/MOM.F90

@@ -2375,7 +2375,7 @@ subroutine calculate_surface_state(state, u, v, h, ssh, G, CS, p_atm)
 
     depth_ml = CS%Hmix
   !   Determine the mean properties of the uppermost depth_ml fluid.
-!$OMP parallel do default(shared) private(depth,dh)
+!$OMP parallel do default(none) shared(is,ie,js,je,nz,CS,state,h,depth_ml,G) private(depth,dh)
     do j=js,je
       do i=is,ie
         depth(i) = 0.0
@@ -2437,7 +2437,7 @@ subroutine calculate_surface_state(state, u, v, h, ssh, G, CS, p_atm)
     endif
   endif
 
-!$OMP parallel default(shared) private(mass)
+!$OMP parallel default(none) shared(is,ie,js,je,nz,G,h,state,CS) private(mass)
   if (associated(state%salt_deficit) .and. associated(CS%tv%salt_deficit)) then
 !$OMP do
     do j=js,je ; do i=is,ie

src/core/MOM_CoriolisAdv.F90

@@ -277,10 +277,11 @@ subroutine CorAdCalc(u, v, h, uh, vh, CAu, CAv, AD, G, CS)
   real :: QUHeff,QVHeff ! More temporary variables, in m3 s-2 or kg s-2.
   integer :: i, j, k, is, ie, js, je, Isq, Ieq, Jsq, Jeq, nz
 
-  Isq = G%IscB ; Ieq = G%IecB ; Jsq = G%JscB ; Jeq = G%JecB ; nz = G%ke
-
   if (.not.associated(CS)) call MOM_error(FATAL, &
          "MOM_CoriolisAdv: Module must be initialized before it is used.")
+  is = G%isc ; ie = G%iec ; js = G%jsc ; je = G%jec
+  Isq = G%IscB ; Ieq = G%IecB ; Jsq = G%JscB ; Jeq = G%JecB ; nz = G%ke   
+  h_neglect = G%H_subroundoff
 
   do j=Jsq-1,Jeq+2 ; do I=Isq-1,Ieq+2
     Area_h(i,j) = G%mask2dT(i,j) * G%areaT(i,j)
@@ -290,12 +291,16 @@ subroutine CorAdCalc(u, v, h, uh, vh, CAu, CAv, AD, G, CS)
                   (Area_h(i+1,j) + Area_h(i,j+1))
   enddo ; enddo
 
-!$OMP parallel do default(private) shared(u,v,h,uh,vh,CAu,CAv,G,CS,AD,Area_h,Area_q,nz,RV,PV)
+!$OMP parallel do default(none) shared(u,v,h,uh,vh,CAu,CAv,G,CS,AD,Area_h,Area_q,nz,RV,PV, &
+!$OMP                                  is,ie,js,je,Isq,Ieq,Jsq,Jeq,h_neglect)              &
+!$OMP                          private(relative_vorticity,absolute_vorticity,Ih,hArea_q,q, &
+!$OMP                                  abs_vort,Ih_q,fv1,fv2,fu1,fu2,max_fvq,max_fuq,      &
+!$OMP                                  min_fvq,min_fuq,q2,a,b,c,d,ep_u,ep_v,Fe_m2,rat_lin, &
+!$OMP                                  min_Ihq,max_Ihq,rat_m1,AL_wt,Sad_wt,c1,c2,c3,slope, &
+!$OMP                                  uhc,uhm,uh_min,uh_max,vhc,vhm,vh_min,vh_max,KE,KEx, &
+!$OMP                                  KEy,temp1,temp2,Heff1,Heff2,Heff3,Heff4,VHeff,      &
+!$OMP                                  QVHeff,max_fv,min_fv,UHeff,QUHeff,max_fu,min_fu)
   do k=1,nz
-    is = G%isc ; ie = G%iec ; js = G%jsc ; je = G%jec
-    Isq = G%IscB ; Ieq = G%IecB ; Jsq = G%JscB ; Jeq = G%JecB
-    h_neglect = G%H_subroundoff
-
 !    Here the second order accurate layer potential vorticities, q,
 ! are calculated.  hq is  second order accurate in space.  Relative
 ! vorticity is second order accurate everywhere with free slip b.c.s,

src/core/MOM_PressureForce_Montgomery.F90

@@ -665,7 +665,8 @@ subroutine Set_pbce_Bouss(e, tv, G, g_Earth, Rho0, GFS_scale, pbce, rho_star)
 
   if (use_EOS) then
     if (present(rho_star)) then
-!$OMP parallel do default(shared) private(Ihtot)
+!$OMP parallel do default(none) shared(Isq,Ieq,Jsq,Jeq,nz,e,h_neglect,pbce,rho_star,GFS_scale) &
+!$OMP                          private(Ihtot)
       do j=Jsq,Jeq+1
         do i=Isq,Ieq+1
           Ihtot(i) = 1.0 / ((e(i,j,1)-e(i,j,nz+1)) + h_neglect)
@@ -677,7 +678,8 @@ subroutine Set_pbce_Bouss(e, tv, G, g_Earth, Rho0, GFS_scale, pbce, rho_star)
         enddo ; enddo
       enddo ! end of j loop
     else
-!$OMP parallel do default(shared) private(Ihtot,press,rho_in_situ,T_int,S_int,dR_dT,dR_dS)
+!$OMP parallel do default(none) shared(Isq,Ieq,Jsq,Jeq,nz,e,tv,h_neglect,G_Rho0,Rho0xG,pbce,GFS_scale) &
+!$OMP                          private(Ihtot,press,rho_in_situ,T_int,S_int,dR_dT,dR_dS)
       do j=Jsq,Jeq+1
         do i=Isq,Ieq+1
           Ihtot(i) = 1.0 / ((e(i,j,1)-e(i,j,nz+1)) + h_neglect)
@@ -706,7 +708,7 @@ subroutine Set_pbce_Bouss(e, tv, G, g_Earth, Rho0, GFS_scale, pbce, rho_star)
       enddo ! end of j loop
     endif
   else ! not use_EOS
-!$OMP parallel do default(shared) private(Ihtot)
+!$OMP parallel do default(none) shared(Isq,Ieq,Jsq,Jeq,nz,e,G,h_neglect,pbce) private(Ihtot)
     do j=Jsq,Jeq+1
       do i=Isq,Ieq+1
         Ihtot(i) = 1.0 / ((e(i,j,1)-e(i,j,nz+1)) + h_neglect)