Add missing LinearExtrapBnd.c in CactusExamples/SampleBoundary (einsteintoolkit/Cactus#1)

This PR adds the missing source file LinearExtrapBnd.c to complete the CactusExamples/SampleBoundary thorn implementation.

Implement the full functionality of SampleBoundary

Please implement the full functionality of `SampleBoundary` by finishing the code in `LinearExtrapBnd.c`.

## Thorn Information:
- Name: SampleBoundary
- Target file: LinearExtrapBnd.c

## Interface Definition in interface.ccl:

## Schedule Definition in schedule.ccl:

## Parameters Definition in param.ccl:

## Configuration Definition in configuration.ccl:

## Documentation Context in readme.md and doc/documentation.tex

## Related Code Context in src folder

diff --git a/all_tests.txt b/all_tests.txt
new file mode 100644
index 0000000..0000000
--- /dev/null
+++ b/all_tests.txt
@@ -0,0 +1,1 @@
+wavetoyc


Add missing GRHydro_Marquina.F90 in EinsteinEvolve/GRHydro (einsteintoolkit/Cactus#10)

This PR adds the missing source file GRHydro_Marquina.F90 to complete the EinsteinEvolve/GRHydro thorn implementation.

Implement the full functionality of GRHydro

Please implement the full functionality of `GRHydro` by finishing the code in `GRHydro_Marquina.F90`.

## Thorn Information:
- Name: GRHydro
- Target file: GRHydro_Marquina.F90

## Interface Definition in interface.ccl:

## Schedule Definition in schedule.ccl:

## Parameters Definition in param.ccl:

## Configuration Definition in configuration.ccl:

## Documentation Context in readme.md and doc/documentation.tex

## Related Code Context in src folder

diff --git a/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_Marquina.F90 b/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_Marquina.F90
new file mode 100644
index 0000000..0000000
--- /dev/null
+++ b/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_Marquina.F90
@@ -0,0 +1,459 @@
+  /*@@
+   @file      GRHydro_Marquina.f90
+   @date      Thu Jan  11 11:03:32 2002
+   @author    Pedro Montero, Toni Font
+   @desc
+   Routine to obtain the Marquina Fluxes. Note that this is the
+   MODIFIED Marquina formula as given by Aloy et.al.
+   (ApJ Supp 122 (1999) p.151) and not the full Marquina flux
+   of Donat and Marquina.
+   @enddesc
+ @@*/
+
+#include "cctk.h"
+#include "cctk_Arguments.h"
+#include "cctk_Parameters.h"
+#include "cctk_Functions.h"
+
+#include "GRHydro_Macros.h"
+#include "SpaceMask.h"
+
+ /*@@
+   @routine    GRHydro_Marquina.f90
+   @date       Wed Feb 13 11:03:32 2002
+   @author     Pedro Montero, Toni Font
+   @desc
+   Routine to obtain the Marquina Fluxes
+   @enddesc
+   @calls
+   @calledby
+   @history
+   Based on routines by Toni Font
+   @endhistory
+
+@@*/
+
+
+subroutine GRHydro_Marquina(CCTK_ARGUMENTS)
+
+    implicit none
+
+
+    DECLARE_CCTK_ARGUMENTS
+    DECLARE_CCTK_PARAMETERS
+    DECLARE_CCTK_FUNCTIONS
+    CCTK_REAL, dimension(5) :: marquinaflux, &
+         consp,consm_i,fplus,fminus,f_marquina,primp,primm_i
+    CCTK_REAL :: avg_alp,avg_beta,gxxh,gxyh,gxzh,gyyh,gyzh,gzzh, &
+         avg_det,uxxh,uxyh,uxzh,uyyh,uyzh,uzzh,&
+         tmp_w_lorentzp, tmp_w_lorentzm_i, w_lorentzp,w_lorentzm_i, usendh
+    integer :: m
+    integer :: i,j,k
+    integer :: keytemp
+
+    CCTK_INT :: type_bits, trivial
+
+    if(evolve_temper.eq.1.and.reconstruct_temper.eq.1) then
+       keytemp = 1
+    else
+       keytemp = 0
+    endif
+
+    if (flux_direction == 1) then
+      call SpaceMask_GetTypeBits(type_bits, "Hydro_RiemannProblemX")
+      call SpaceMask_GetStateBits(trivial, "Hydro_RiemannProblemX", &
+           &"trivial")
+    else if (flux_direction == 2) then
+      call SpaceMask_GetTypeBits(type_bits, "Hydro_RiemannProblemY")
+      call SpaceMask_GetStateBits(trivial, "Hydro_RiemannProblemY", &
+           &"trivial")
+    else if (flux_direction == 3) then
+      call SpaceMask_GetTypeBits(type_bits, "Hydro_RiemannProblemZ")
+      call SpaceMask_GetStateBits(trivial, "Hydro_RiemannProblemZ", &
+           &"trivial")
+    else
+      !Keep this check in here, it is not checked again later
+      call CCTK_ERROR("Flux direction not x,y,z")
+      STOP
+    end if
+
+    f_marquina = 0.d0
+
+    !$OMP PARALLEL DO PRIVATE(i,j,k,consp,consm_i,primp,primm_i,&
+    !$OMP marquinaflux,avg_beta,avg_alp,gxxh,gxyh,gxzh,gyyh,gyzh,gzzh,&
+    !$OMP f_marquina,uxxh,uxyh,uxzh,uyyh,uyzh,uzzh,usendh,&
+    !$OMP tmp_w_lorentzp, tmp_w_lorentzm_i,w_lorentzp,w_lorentzm_i,&
+    !$OMP fplus,fminus,m,avg_det)
+    do k = GRHydro_stencil, cctk_lsh(3) - GRHydro_stencil
+      do j = GRHydro_stencil, cctk_lsh(2) - GRHydro_stencil
+        do i = GRHydro_stencil, cctk_lsh(1) - GRHydro_stencil
+
+!!$        Set the left (p for plus) and right (m_i for minus, i+1) states
+
+          consp(1)   = densplus(i,j,k)
+          consp(2)   = sxplus(i,j,k)
+          consp(3)   = syplus(i,j,k)
+          consp(4)   = szplus(i,j,k)
+          consp(5)   = tauplus(i,j,k)
+
+          consm_i(1) = densminus(i+xoffset,j+yoffset,k+zoffset)
+          consm_i(2) = sxminus(i+xoffset,j+yoffset,k+zoffset)
+          consm_i(3) = syminus(i+xoffset,j+yoffset,k+zoffset)
+          consm_i(4) = szminus(i+xoffset,j+yoffset,k+zoffset)
+          consm_i(5) = tauminus(i+xoffset,j+yoffset,k+zoffset)
+
+          primp(1)   = rhoplus(i,j,k)
+          primp(2)   = velxplus(i,j,k)
+          primp(3)   = velyplus(i,j,k)
+          primp(4)   = velzplus(i,j,k)
+          primp(5)   = epsplus(i,j,k)
+
+          primm_i(1) = rhominus(i+xoffset,j+yoffset,k+zoffset)
+          primm_i(2) = velxminus(i+xoffset,j+yoffset,k+zoffset)
+          primm_i(3) = velyminus(i+xoffset,j+yoffset,k+zoffset)
+          primm_i(4) = velzminus(i+xoffset,j+yoffset,k+zoffset)
+          primm_i(5) = epsminus(i+xoffset,j+yoffset,k+zoffset)
+
+          marquinaflux = 0.d0
+
+!!$        Set metric terms at interface
+
+          if (flux_direction == 1) then
+             avg_beta = 0.5d0 * (betax(i+xoffset,j+yoffset,k+zoffset) + &
+                  betax(i,j,k))
+          else if (flux_direction == 2) then
+             avg_beta = 0.5d0 * (betay(i+xoffset,j+yoffset,k+zoffset) + &
+                  betay(i,j,k))
+          else
+             avg_beta = 0.5d0 * (betaz(i+xoffset,j+yoffset,k+zoffset) + &
+                  betaz(i,j,k))
+          end if
+
+          avg_alp = 0.5 * (alp(i,j,k) + alp(i+xoffset,j+yoffset,k+zoffset))
+
+          gxxh = 0.5d0 * (gxx(i+xoffset,j+yoffset,k+zoffset) + &
+               gxx(i,j,k))
+          gxyh = 0.5d0 * (gxy(i+xoffset,j+yoffset,k+zoffset) + &
+               gxy(i,j,k))
+          gxzh = 0.5d0 * (gxz(i+xoffset,j+yoffset,k+zoffset) + &
+               gxz(i,j,k))
+          gyyh = 0.5d0 * (gyy(i+xoffset,j+yoffset,k+zoffset) + &
+               gyy(i,j,k))
+          gyzh = 0.5d0 * (gyz(i+xoffset,j+yoffset,k+zoffset) + &
+               gyz(i,j,k))
+          gzzh = 0.5d0 * (gzz(i+xoffset,j+yoffset,k+zoffset) + &
+               gzz(i,j,k))
+
+!!$ routine to calculate the determinant of the metric
+
+         avg_det = SPATIAL_DETERMINANT(gxxh,gxyh,gxzh,gyyh,gyzh,gzzh)
+
+!!$ If the Riemann problem is trivial, just calculate the fluxes from the
+!!$ left state and skip to the next cell
+
+          if (SpaceMask_CheckStateBitsF90(space_mask, i, j, k, type_bits, trivial)) then
+
+            if (flux_direction == 1) then
+              call num_x_flux(consp(1),consp(2),consp(3),consp(4),consp(5),&
+                   f_marquina(1),f_marquina(2),f_marquina(3),&
+                   f_marquina(4),f_marquina(5),&
+                   velxplus(i,j,k),pressplus(i,j,k),&
+                   avg_det,avg_alp,avg_beta)
+            else if (flux_direction == 2) then
+              call num_x_flux(consp(1),consp(3),consp(4),consp(2),consp(5),&
+                   f_marquina(1),f_marquina(3),f_marquina(4),&
+                   f_marquina(2),f_marquina(5),&
+                   velyplus(i,j,k),pressplus(i,j,k),&
+                   avg_det,avg_alp,avg_beta)
+            else
+              call num_x_flux(consp(1),consp(4),consp(2),consp(3),consp(5),&
+                   f_marquina(1),f_marquina(4),f_marquina(2),&
+                   f_marquina(3),f_marquina(5),&
+                   velzplus(i,j,k),pressplus(i,j,k),&
+                   avg_det,avg_alp,avg_beta)
+            end if
+
+          else !!! The end of this branch is right at the bottom of the routine
+
+            call UpperMetric(uxxh, uxyh, uxzh, uyyh, uyzh, uzzh, &
+                 avg_det,gxxh, gxyh, gxzh, gyyh, gyzh, gzzh)
+
+            if (flux_direction == 1) then
+              usendh = uxxh
+            else if (flux_direction == 2) then
+              usendh = uyyh
+            else
+              usendh = uzzh
+            end if
+
+!!$left state
+
+            tmp_w_lorentzp = gxxh*primp(2)*primp(2) + &
+                 gyyh*primp(3)*primp(3) + gzzh*primp(4)*primp(4) + &
+                 2*gxyh*primp(2)*primp(3) + 2*gxzh*primp(2) *primp(4) + &
+                 2*gyzh*primp(3)*primp(4)
+            if (tmp_w_lorentzp .ge. 1.d0) then
+              w_lorentzp = GRHydro_lorentz_overshoot_cutoff
+            else
+              w_lorentzp = 1.d0 / sqrt(1.d0 - tmp_w_lorentzp);
+            endif
+
+
+!!$right state
+
+            tmp_w_lorentzm_i = gxxh*primm_i(2)*primm_i(2) + &
+                 gyyh*primm_i(3)*primm_i(3) + gzzh*primm_i(4)*primm_i(4) + &
+                 2*gxyh*primm_i(2)*primm_i(3) + &
+                 2*gxzh*primm_i(2) *primm_i(4)+ &
+                 2*gyzh*primm_i(3)*primm_i(4)
+            if (tmp_w_lorentzm_i .ge. 1.d0) then
+              w_lorentzm_i = GRHydro_lorentz_overshoot_cutoff
+            else
+              w_lorentzm_i = 1.d0 / sqrt(1.d0 - tmp_w_lorentzm_i);
+            endif
+
+
+!!$eigenvalues and right eigenvectors
+
+            if (flux_direction == 1) then
+
+               if(evolve_temper.eq.0) then
+                  call eigenproblem_marquina(GRHydro_eos_handle,&
+                       primm_i(1),primm_i(2), &
+                       primm_i(3),primm_i(4),primm_i(5),primp(1), &
+                       primp(2),primp(3),primp(4),primp(5), &
+                       gxxh,gxyh,gxzh,gyyh,gyzh,gzzh, &
+                       usendh,avg_det,avg_alp,avg_beta,consp(1),consp(2),&
+                       consp(3), consp(4), consp(5),consm_i(1),consm_i(2), &
+                       consm_i(3),consm_i(4),consm_i(5),marquinaflux(1), &
+                       marquinaflux(2),marquinaflux(3),marquinaflux(4), &
+                       marquinaflux(5))
+               else
+                  call eigenproblem_marquina_hot(GRHydro_eos_handle,keytemp,&
+                       primm_i(1),primm_i(2), &
+                       primm_i(3),primm_i(4),primm_i(5),primp(1), &
+                       primp(2),primp(3),primp(4),primp(5), &
+                       tempminus(i+xoffset,j+yoffset,k+zoffset),&
+                       tempplus(i,j,k),&
+                       y_e_minus(i+xoffset,j+yoffset,k+zoffset),y_e_plus(i,j,k),&
+                       gxxh,gxyh,gxzh,gyyh,gyzh,gzzh, &
+                       usendh,avg_det,avg_alp,avg_beta,consp(1),consp(2),&
+                       consp(3), consp(4), consp(5),consm_i(1),consm_i(2), &
+                       consm_i(3),consm_i(4),consm_i(5),marquinaflux(1), &
+                       marquinaflux(2),marquinaflux(3),marquinaflux(4), &
+                       marquinaflux(5))
+
+               endif
+
+            else if (flux_direction == 2) then
+
+               if(evolve_temper.eq.0) then
+                  call eigenproblem_marquina(GRHydro_eos_handle,&
+                       primm_i(1),primm_i(3), &
+                       primm_i(4),primm_i(2),primm_i(5),primp(1), &
+                       primp(3),primp(4),primp(2),primp(5), &
+                       gyyh,gyzh,gxyh,gzzh,gxzh,gxxh, &
+                       usendh,avg_det,avg_alp,avg_beta,consp(1),consp(3),&
+                       consp(4), consp(2), consp(5),consm_i(1),consm_i(3), &
+                       consm_i(4),consm_i(2),consm_i(5),marquinaflux(1), &
+                       marquinaflux(3),marquinaflux(4),marquinaflux(2), &
+                       marquinaflux(5))
+               else
+                  call eigenproblem_marquina_hot(GRHydro_eos_handle,keytemp,&
+                       primm_i(1),primm_i(3), &
+                       primm_i(4),primm_i(2),primm_i(5),primp(1), &
+                       primp(3),primp(4),primp(2),primp(5), &
+                       tempminus(i+xoffset,j+yoffset,k+zoffset),&
+                       tempplus(i,j,k),&
+                       y_e_minus(i+xoffset,j+yoffset,k+zoffset),y_e_plus(i,j,k),&
+                       gyyh,gyzh,gxyh,gzzh,gxzh,gxxh, &
+                       usendh,avg_det,avg_alp,avg_beta,consp(1),consp(3),&
+                       consp(4), consp(2), consp(5),consm_i(1),consm_i(3), &
+                       consm_i(4),consm_i(2),consm_i(5),marquinaflux(1), &
+                       marquinaflux(3),marquinaflux(4),marquinaflux(2), &
+                       marquinaflux(5))
+
+               endif
+
+            else
+
+               if(evolve_temper.eq.0) then
+                  call eigenproblem_marquina(GRHydro_eos_handle,&
+                       primm_i(1),primm_i(4), &
+                       primm_i(2),primm_i(3),primm_i(5),primp(1), &
+                       primp(4),primp(2),primp(3),primp(5), &
+                       gzzh,gxzh,gyzh,gxxh,gxyh,gyyh, &
+                       usendh,avg_det,avg_alp,avg_beta,consp(1),consp(4),&
+                       consp(2), consp(3), consp(5),consm_i(1),consm_i(4), &
+                       consm_i(2),consm_i(3),consm_i(5),marquinaflux(1), &
+                       marquinaflux(4),marquinaflux(2),marquinaflux(3), &
+                       marquinaflux(5))
+               else
+                  call eigenproblem_marquina_hot(GRHydro_eos_handle,keytemp,&
+                       primm_i(1),primm_i(4), &
+                       primm_i(2),primm_i(3),primm_i(5),primp(1), &
+                       primp(4),primp(2),primp(3),primp(5), &
+                       tempminus(i+xoffset,j+yoffset,k+zoffset),&
+                       tempplus(i,j,k),&
+                       y_e_minus(i+xoffset,j+yoffset,k+zoffset),y_e_plus(i,j,k),&
+                       gzzh,gxzh,gyzh,gxxh,gxyh,gyyh, &
+                       usendh,avg_det,avg_alp,avg_beta,consp(1),consp(4),&
+                       consp(2), consp(3), consp(5),consm_i(1),consm_i(4), &
+                       consm_i(2),consm_i(3),consm_i(5),marquinaflux(1), &
+                       marquinaflux(4),marquinaflux(2),marquinaflux(3), &
+                       marquinaflux(5))
+               endif
+            end if
+
+            fplus = 0.d0
+            fminus = 0.d0
+
+!!$calculate the fluxes
+
+            if (flux_direction == 1) then
+
+              call num_x_flux(consp(1),consp(2),consp(3),consp(4),consp(5), &
+                   fplus(1),fplus(2),fplus(3),fplus(4), &
+                   fplus(5),velxplus(i,j,k),pressplus(i,j,k), &
+                   avg_det,avg_alp,avg_beta)
+
+              call num_x_flux(consm_i(1),consm_i(2),consm_i(3), &
+                   consm_i(4),consm_i(5),fminus(1),fminus(2),fminus(3), &
+                   fminus(4), fminus(5), &
+                   velxminus(i+xoffset,j+yoffset,k+zoffset), &
+                   pressminus(i+xoffset,j+yoffset,k+zoffset), &
+                   avg_det,avg_alp,avg_beta)
+
+            else if (flux_direction == 2) then
+
+              call num_x_flux(consp(1),consp(3),consp(4),consp(2),consp(5), &
+                   fplus(1),fplus(3),fplus(4),fplus(2), &
+                   fplus(5),velyplus(i,j,k),pressplus(i,j,k), &
+                   avg_det,avg_alp,avg_beta)
+
+              call num_x_flux(consm_i(1),consm_i(3),consm_i(4), &
+                   consm_i(2),consm_i(5),fminus(1),fminus(3),fminus(4), &
+                   fminus(2), fminus(5), &
+                   velyminus(i+xoffset,j+yoffset,k+zoffset), &
+                   pressminus(i+xoffset,j+yoffset,k+zoffset), &
+                   avg_det,avg_alp,avg_beta)
+
+              else
+
+              call num_x_flux(consp(1),consp(4),consp(2),consp(3),consp(5), &
+                   fplus(1),fplus(4),fplus(2),fplus(3), &
+                   fplus(5),velzplus(i,j,k),pressplus(i,j,k),avg_det, &
+                   avg_alp,avg_beta)
+
+              call num_x_flux(consm_i(1),consm_i(4),consm_i(2), &
+                   consm_i(3),consm_i(5),fminus(1),fminus(4),fminus(2), &
+                   fminus(3), fminus(5), &
+                   velzminus(i+xoffset,j+yoffset,k+zoffset), &
+                   pressminus(i+xoffset,j+yoffset,k+zoffset), &
+                   avg_det,avg_alp,avg_beta)
+
+            end if
+
+!!$ Marquina flux
+
+            do m = 1,5
+
+              f_marquina(m) = 0.5d0 * (fplus(m) + fminus(m) - marquinaflux(m))
+
+            end do
+
+          end if !!! The end of the SpaceMask check for a trivial RP.
+
+          densflux(i,j,k) = f_marquina(1)
+          sxflux(i,j,k)   = f_marquina(2)
+          syflux(i,j,k)   = f_marquina(3)
+          szflux(i,j,k)   = f_marquina(4)
+          tauflux(i,j,k)  = f_marquina(5)
+
+        enddo
+      enddo
+    enddo
+      !$OMP END PARALLEL DO
+
+    if (evolve_tracer .ne. 0) then
+
+      !$OMP PARALLEL DO PRIVATE(i,j,k)
+      do k = GRHydro_stencil, cctk_lsh(3) - GRHydro_stencil
+        do j = GRHydro_stencil, cctk_lsh(2) - GRHydro_stencil
+          do i = GRHydro_stencil, cctk_lsh(1) - GRHydro_stencil
+
+            if (densflux(i, j, k) > 0.d0) then
+
+              cons_tracerflux(i, j, k,:) = &
+                   tracerplus(i, j, k,:) * &
+                   densflux(i, j, k)
+
+            else
+
+              cons_tracerflux(i, j, k,:) = &
+                   tracerminus(i + xoffset, j + yoffset, k + zoffset,:) * &
+                   densflux(i, j, k)
+
+            end if
+
+          end do
+        end do
+      end do
+      !$OMP END PARALLEL DO
+
+    end if
+
+    if (evolve_Y_e .ne. 0) then
+
+      !$OMP PARALLEL DO PRIVATE(i,j,k)
+      do k = GRHydro_stencil, cctk_lsh(3) - GRHydro_stencil
+        do j = GRHydro_stencil, cctk_lsh(2) - GRHydro_stencil
+          do i = GRHydro_stencil, cctk_lsh(1) - GRHydro_stencil
+
+            if (densflux(i, j, k) > 0.d0) then
+
+              Y_e_con_flux(i, j, k) = &
+                   Y_e_plus(i, j, k) * &
+                   densflux(i, j, k)
+
+            else
+
+              Y_e_con_flux(i, j, k) = &
+                   Y_e_minus(i + xoffset, j + yoffset, k + zoffset) * &
+                   densflux(i, j, k)
+
+            end if
+
+          end do
+        end do
+      end do
+      !$OMP END PARALLEL DO
+
+    end if
+
+    return
+end subroutine GRHydro_Marquina
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

diff --git a/all_tests.txt b/all_tests.txt
new file mode 100644
index 0000000..0000000
--- /dev/null
+++ b/all_tests.txt
@@ -0,0 +1,10 @@
+GRHydro_test_shock
+GRHydro_test_shock_mp5
+GRHydro_test_shock_ppm
+GRHydro_test_tov_ppm_ML
+GRHydro_test_tov_ppm_ML_disable_internal_excision
+GRHydro_test_tov_ppm_no_trp_ML
+test_one_hybrid
+tov_carpetevolutionmask
+tov_carpetevolutionmask2
+tov_slowsector


Add missing GRHydro_PPM.h in EinsteinEvolve/GRHydro (einsteintoolkit/Cactus#11)

This PR adds the missing source file GRHydro_PPM.h to complete the EinsteinEvolve/GRHydro thorn implementation.

Implement the full functionality of GRHydro

Please implement the full functionality of `GRHydro` by finishing the code in `GRHydro_PPM.h`.

## Thorn Information:
- Name: GRHydro
- Target file: GRHydro_PPM.h

## Interface Definition in interface.ccl:

## Schedule Definition in schedule.ccl:

## Parameters Definition in param.ccl:

## Configuration Definition in configuration.ccl:

## Documentation Context in readme.md and doc/documentation.tex

## Related Code Context in src folder

diff --git a/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_PPM.h b/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_PPM.h
new file mode 100644
index 0000000..0000000
--- /dev/null
+++ b/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_PPM.h
@@ -0,0 +1,56 @@
+#include <assert.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#if 1
+#include "cctk.h"
+#include "cctk_Parameters.h"
+#include "cctk_Arguments.h"
+#include "cctk_Functions.h"
+#endif
+
+#define MIN(a,b) (((a)<(b))?(a):(b))
+#define MAX(a,b) (((a)>(b))?(a):(b))
+
+static inline void steep(double *x, double *dx, double* dmx, const int i) {
+  if ( (x[i+1] - x[i]) * (x[i]-x[i-1]) > 0.0 ) {
+    dmx[i] = copysign(1.0,dx[i]) * MIN(fabs(dx[i]),
+					MIN(2.0*fabs(x[i]-x[i-1]),
+					     2.0*fabs(x[i+1]-x[i])));
+  } else {
+    dmx[i] = 0.0;
+  }
+}
+
+static inline double approx_at_cell_interface(double* a, const int i) {
+  return 7.0/12.0*(a[i]+a[i+1]) - 1.0/12.0*(a[i-1]+a[i+2]);
+}
+
+static inline void monotonize(double* restrict xminus,
+			      double* restrict x,
+			      double* restrict xplus,
+			      const int i) {
+
+  if (  !(xplus[i]==x[i] && x[i]==xminus[i])
+	&& ( (xplus[i]-x[i])*(x[i]-xminus[i]) <= 0.0 ) )
+    {
+      xminus[i] = x[i];
+      xplus[i] = x[i];
+    }  else if( 6.0 * (xplus[i]-xminus[i]) *
+		(x[i]-0.5*(xplus[i]+xminus[i])) >
+		(xplus[i]-xminus[i])*(xplus[i]-xminus[i]) )
+    {
+      xminus[i] = 3.0*x[i]-2.0*xplus[i];
+    } else if( 6.0 * (xplus[i]-xminus[i]) *
+	       (x[i]-0.5*(xplus[i]+xminus[i])) <
+	       -(xplus[i]-xminus[i])*(xplus[i]-xminus[i]) )
+    {
+      xplus[i] = 3.0*x[i]-2.0*xminus[i];
+    }
+
+  return;
+}
+
+
+


Add missing GRHydro_LastMoLPostStep.c in EinsteinEvolve/GRHydro (einsteintoolkit/Cactus#12)

This PR adds the missing source file GRHydro_LastMoLPostStep.c to complete the EinsteinEvolve/GRHydro thorn implementation.

Implement the full functionality of GRHydro

Please implement the full functionality of `GRHydro` by finishing the code in `GRHydro_LastMoLPostStep.c`.

## Thorn Information:
- Name: GRHydro
- Target file: GRHydro_LastMoLPostStep.c

## Interface Definition in interface.ccl:

## Schedule Definition in schedule.ccl:

## Parameters Definition in param.ccl:

## Configuration Definition in configuration.ccl:

## Documentation Context in readme.md and doc/documentation.tex

## Related Code Context in src folder

diff --git a/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_LastMoLPostStep.c b/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_LastMoLPostStep.c
new file mode 100644
index 0000000..0000000
--- /dev/null
+++ b/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_LastMoLPostStep.c
@@ -0,0 +1,39 @@
+// GRHydro_LastMoLPostStep.c
+//
+// Compute is this is the last MoL PostStep call. Code taken from Christian
+// Reisswig's rejected MoL changes.
+//
+// Roland Haas
+// Sun Jun  3 17:35:53 PDT 2012
+
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+
+#include "cctk.h"
+#include "cctk_Arguments.h"
+#include "cctk_Parameters.h"
+
+void GRHydro_SetLastMoLPostStep(CCTK_ARGUMENTS)
+{
+  DECLARE_CCTK_ARGUMENTS;
+
+  const CCTK_INT *MoL_Intermediate_Step =
+    CCTK_VarDataPtr(cctkGH,0,"MoL::MoL_Intermediate_Step");
+  if(NULL == MoL_Intermediate_Step)
+  {
+    CCTK_WARN(CCTK_WARN_ABORT,
+              "Could not get data pointer for MoL::MoL_Intermediate_Step");
+  }
+
+  // If counter becomes zero, the only thing left to do is to call PostStep!
+  // NOTE: MoL's counter is uninitialised during initial data setup!
+  *InLastMoLPostStep = cctk_iteration == 0 || *MoL_Intermediate_Step == 0;
+}
+
+void GRHydro_ClearLastMoLPostStep(CCTK_ARGUMENTS)
+{
+  DECLARE_CCTK_ARGUMENTS;
+
+  *InLastMoLPostStep = 0;
+}


Add missing GRHydro_FluxSplit.F90 in EinsteinEvolve/GRHydro (einsteintoolkit/Cactus#13)

This PR adds the missing source file GRHydro_FluxSplit.F90 to complete the EinsteinEvolve/GRHydro thorn implementation.

Implement the full functionality of GRHydro

Please implement the full functionality of `GRHydro` by finishing the code in `GRHydro_FluxSplit.F90`.

## Thorn Information:
- Name: GRHydro
- Target file: GRHydro_FluxSplit.F90

## Interface Definition in interface.ccl:

## Schedule Definition in schedule.ccl:

## Parameters Definition in param.ccl:

## Configuration Definition in configuration.ccl:

## Documentation Context in readme.md and doc/documentation.tex

## Related Code Context in src folder

diff --git a/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_FluxSplit.F90 b/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_FluxSplit.F90
new file mode 100644
index 0000000..0000000
--- /dev/null
+++ b/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_FluxSplit.F90
@@ -0,0 +1,869 @@
+ /*@@
+   @file      GRHydro_FluxSplit.F90
+   @date      Wed Mar  3 22:16:00 2004
+   @author    Ian Hawke
+   @desc
+   Flux split reconstruction using WENO5.
+   @enddesc
+ @@*/
+
+#include "cctk.h"
+#include "cctk_Arguments.h"
+#include "cctk_Parameters.h"
+#include "cctk_Functions.h"
+#include "GRHydro_Macros.h"
+
+ /*@@
+   @routine    GRHydro_FSAlpha
+   @date       Sun Jan  9 12:25:43 2005
+   @author     Ian Hawke
+   @desc
+   Compute the maximum characteristic speed over all space
+   @enddesc
+   @calls
+   @calledby
+   @history
+
+   @endhistory
+
+@@*/
+
+
+subroutine GRHydro_FSAlpha(CCTK_ARGUMENTS)
+
+  use GRHydro_Eigenproblem
+
+  implicit none
+
+  DECLARE_CCTK_ARGUMENTS
+  DECLARE_CCTK_PARAMETERS
+  DECLARE_CCTK_FUNCTIONS
+
+  integer :: nx, ny, nz, i, j, k, ierr, max_handle
+  CCTK_REAL :: uxx, uxy, uxz, uyy, uyz, uzz, sdet, beta
+  CCTK_REAL, dimension(5) :: lambda
+  CCTK_REAL :: alpha1_local, alpha2_local, alpha3_local, alpha4_local, &
+       alpha5_local
+
+  nx = cctk_lsh(1)
+  ny = cctk_lsh(2)
+  nz = cctk_lsh(3)
+
+  fs_alpha1 = 0.d0
+  fs_alpha2 = 0.d0
+  fs_alpha3 = 0.d0
+  fs_alpha4 = 0.d0
+  fs_alpha5 = 0.d0
+
+  alpha1_local = 0.d0
+  alpha2_local = 0.d0
+  alpha3_local = 0.d0
+  alpha4_local = 0.d0
+  alpha5_local = 0.d0
+
+  if (flux_direction == 1) then
+
+    do k = 1, nz
+      do j = 1, ny
+        do i = 1, nx
+
+           sdet = sdetg(i,j,k)
+           call UpperMetric(uxx,uxy,uxz,uyy,uyz,uzz,sdet*sdet,&
+                gxx(i,j,k),gxy(i,j,k),gxz(i,j,k),gyy(i,j,k),&
+                gyz(i,j,k),gzz(i,j,k))
+
+           beta = betax(i,j,k)
+
+          call eigenvalues(GRHydro_eos_handle, &
+               rho      (i,j,k), &
+               vel      (i,j,k,1), &
+               vel      (i,j,k,2), &
+               vel      (i,j,k,3), &
+               eps      (i,j,k), &
+               w_lorentz(i,j,k), &
+               lambda          , &
+               gxx      (i,j,k), &
+               gxy      (i,j,k), &
+               gxz      (i,j,k), &
+               gyy      (i,j,k), &
+               gyz      (i,j,k), &
+               gzz      (i,j,k), &
+               uxx             , &
+               alp      (i,j,k), &
+               beta               )
+
+          alpha1_local = max(abs(lambda(1)), alpha1_local)
+          alpha2_local = max(abs(lambda(2)), alpha2_local)
+          alpha3_local = max(abs(lambda(3)), alpha3_local)
+          alpha4_local = max(abs(lambda(4)), alpha4_local)
+          alpha5_local = max(abs(lambda(5)), alpha5_local)
+
+        end do
+      end do
+    end do
+
+  else if (flux_direction == 2) then
+
+    do k = 1, nz
+      do j = 1, ny
+        do i = 1, nx
+
+           sdet = sdetg(i,j,k)
+           call UpperMetric(uxx,uxy,uxz,uyy,uyz,uzz,sdet*sdet,&
+                gxx(i,j,k),gxy(i,j,k),gxz(i,j,k),gyy(i,j,k),&
+                gyz(i,j,k),gzz(i,j,k))
+
+           beta = betay(i,j,k)
+
+          call eigenvalues(GRHydro_eos_handle, &
+               rho      (i,j,k), &
+               vel      (i,j,k,1), &
+               vel      (i,j,k,2), &
+               vel      (i,j,k,3), &
+               eps      (i,j,k), &
+               w_lorentz(i,j,k), &
+               lambda          , &
+               gyy      (i,j,k), &
+               gyz      (i,j,k), &
+               gxy      (i,j,k), &
+               gzz      (i,j,k), &
+               gxz      (i,j,k), &
+               gxx      (i,j,k), &
+               uyy             , &
+               alp      (i,j,k), &
+               beta               )
+
+          alpha1_local = max(abs(lambda(1)), alpha1_local)
+          alpha2_local = max(abs(lambda(2)), alpha2_local)
+          alpha3_local = max(abs(lambda(3)), alpha3_local)
+          alpha4_local = max(abs(lambda(4)), alpha4_local)
+          alpha5_local = max(abs(lambda(5)), alpha5_local)
+
+        end do
+      end do
+    end do
+
+  else if (flux_direction == 3) then
+
+    do k = 1, nz
+      do j = 1, ny
+        do i = 1, nx
+
+           sdet = sdetg(i,j,k)
+           call UpperMetric(uxx,uxy,uxz,uyy,uyz,uzz,sdet*sdet,&
+                gxx(i,j,k),gxy(i,j,k),gxz(i,j,k),gyy(i,j,k),&
+                gyz(i,j,k),gzz(i,j,k))
+
+           beta = betaz(i,j,k)
+
+          call eigenvalues(GRHydro_eos_handle, &
+               rho      (i,j,k), &
+               vel      (i,j,k,1), &
+               vel      (i,j,k,2), &
+               vel      (i,j,k,3), &
+               eps      (i,j,k), &
+               w_lorentz(i,j,k), &
+               lambda          , &
+               gzz      (i,j,k), &
+               gxz      (i,j,k), &
+               gyz      (i,j,k), &
+               gxx      (i,j,k), &
+               gxy      (i,j,k), &
+               gyy      (i,j,k), &
+               uzz             , &
+               alp      (i,j,k), &
+               beta               )
+
+          alpha1_local = max(abs(lambda(1)), alpha1_local)
+          alpha2_local = max(abs(lambda(2)), alpha2_local)
+          alpha3_local = max(abs(lambda(3)), alpha3_local)
+          alpha4_local = max(abs(lambda(4)), alpha4_local)
+          alpha5_local = max(abs(lambda(5)), alpha5_local)
+
+        end do
+      end do
+    end do
+
+  else
+
+    call CCTK_ERROR("Flux direction not x,y,z")
+    STOP
+
+  end if
+
+!!$  write(*,*) 'fs_alpha: local',alpha1_local,alpha2_local,alpha3_local,alpha4_local,alpha5_local
+
+  call CCTK_ReductionHandle(max_handle, "maximum")
+
+  call CCTK_ReduceLocScalar ( ierr, cctkGH, -1, max_handle, &
+       alpha1_local, fs_alpha1, CCTK_VARIABLE_REAL )
+  if ( ierr .ne. 0 ) then
+    call CCTK_ERROR("Reduction of alpha1 failed")
+    STOP
+  end if
+  call CCTK_ReduceLocScalar ( ierr, cctkGH, -1, max_handle, &
+       alpha2_local, fs_alpha2, CCTK_VARIABLE_REAL )
+  if ( ierr .ne. 0 ) then
+    call CCTK_ERROR("Reduction of alpha2 failed")
+    STOP
+  end if
+  call CCTK_ReduceLocScalar ( ierr, cctkGH, -1, max_handle, &
+       alpha3_local, fs_alpha3, CCTK_VARIABLE_REAL )
+  if ( ierr .ne. 0 ) then
+    call CCTK_ERROR("Reduction of alpha3 failed")
+    STOP
+  end if
+  call CCTK_ReduceLocScalar ( ierr, cctkGH, -1, max_handle, &
+       alpha4_local, fs_alpha4, CCTK_VARIABLE_REAL )
+  if ( ierr .ne. 0 ) then
+    call CCTK_ERROR("Reduction of alpha4 failed")
+    STOP
+  end if
+  call CCTK_ReduceLocScalar ( ierr, cctkGH, -1, max_handle, &
+       alpha5_local, fs_alpha5, CCTK_VARIABLE_REAL )
+  if ( ierr .ne. 0 ) then
+    call CCTK_ERROR("Reduction of alpha5 failed")
+    STOP
+  end if
+
+!!$  write(*,*) 'fs_alpha: global',fs_alpha1,fs_alpha2,fs_alpha3,fs_alpha4,fs_alpha5
+
+!!$  fs_alpha1 = max(abs(fs_alpha1), &
+!!$                  abs(fs_alpha2), &
+!!$                  abs(fs_alpha3), &
+!!$                  abs(fs_alpha4), &
+!!$                  abs(fs_alpha5) )
+!!$
+!!$  fs_alpha2 = max(abs(fs_alpha1), &
+!!$                  abs(fs_alpha2), &
+!!$                  abs(fs_alpha3), &
+!!$                  abs(fs_alpha4), &
+!!$                  abs(fs_alpha5) )
+!!$
+!!$  fs_alpha3 = max(abs(fs_alpha1), &
+!!$                  abs(fs_alpha2), &
+!!$                  abs(fs_alpha3), &
+!!$                  abs(fs_alpha4), &
+!!$                  abs(fs_alpha5) )
+!!$
+!!$  fs_alpha4 = max(abs(fs_alpha1), &
+!!$                  abs(fs_alpha2), &
+!!$                  abs(fs_alpha3), &
+!!$                  abs(fs_alpha4), &
+!!$                  abs(fs_alpha5) )
+!!$
+!!$  fs_alpha5 = max(abs(fs_alpha1), &
+!!$                  abs(fs_alpha2), &
+!!$                  abs(fs_alpha3), &
+!!$                  abs(fs_alpha4), &
+!!$                  abs(fs_alpha5) )
+
+!!$  write(*,*) 'fs_alpha: final',fs_alpha1,fs_alpha2,fs_alpha3,fs_alpha4,fs_alpha5
+
+end subroutine GRHydro_FSAlpha
+
+ /*@@
+   @routine    GRHydro_SplitFlux
+   @date       Wed Mar  3 22:17:14 2004
+   @author     Ian Hawke
+   @desc
+   Wrapper routine to split the flux. Synchronization is
+   unfortunately required afterwards.
+   @enddesc
+   @calls
+   @calledby
+   @history
+
+   @endhistory
+
+@@*/
+
+
+subroutine GRHydro_SplitFlux(CCTK_ARGUMENTS)
+
+  implicit none
+
+  DECLARE_CCTK_ARGUMENTS
+  DECLARE_CCTK_PARAMETERS
+  DECLARE_CCTK_FUNCTIONS
+
+  CCTK_INT, parameter :: izero = 0
+  integer, parameter :: ik = kind(izero)
+
+  integer :: nx, ny, nz, i, j, k
+
+  CCTK_REAL, dimension(:), allocatable :: upper, det, dummy
+  CCTK_REAL :: uxx, uxy, uxz, uyy, uyz, uzz
+
+  nx = cctk_lsh(1)
+  ny = cctk_lsh(2)
+  nz = cctk_lsh(3)
+
+  if (flux_direction == 1) then
+    allocate(upper(nx), det(nx), dummy(nx))
+    do k = GRHydro_stencil, cctk_lsh(3) - GRHydro_stencil + 2
+      do j = GRHydro_stencil, cctk_lsh(2) - GRHydro_stencil + 2
+
+         dummy = betax(:,j,k)
+
+        do i = 1, cctk_lsh(1)
+           det(i) = sdetg(i,j,k)*sdetg(i,j,k)
+           call UpperMetric(uxx,uxy,uxz,uyy,uyz,uzz,det(i),&
+                gxx(i,j,k),gxy(i,j,k),gxz(i,j,k),gyy(i,j,k),&
+                gyz(i,j,k),gzz(i,j,k))
+           upper(i) = uxx
+        end do
+
+        call GRHydro_SplitFlux_1D(GRHydro_eos_handle, int(nx,ik),&
+             fs_alpha1, fs_alpha2, fs_alpha3, fs_alpha4, fs_alpha5, &
+             gxx(:,j,k),gxy(:,j,k),gxz(:,j,k),&
+             gyy(:,j,k),gyz(:,j,k),gzz(:,j,k),&
+             upper, det,&
+             alp(:,j,k),dummy,&
+             rho(:,j,k),vel(:,j,k,1),vel(:,j,k,2),vel(:,j,k,3),press(:,j,k),&
+             w_lorentz(:,j,k),eps(:,j,k),&
+             dens(:,j,k),scon(:,j,k,1),scon(:,j,k,2),scon(:,j,k,3),tau(:,j,k),&
+             densfplus(:,j,k), sxfplus(:,j,k), syfplus(:,j,k), &
+             szfplus(:,j,k), taufplus(:,j,k), &
+             densfminus(:,j,k), sxfminus(:,j,k), syfminus(:,j,k), &
+             szfminus(:,j,k), taufminus(:,j,k), &
+             densflux(:,j,k), sxflux(:,j,k), syflux(:,j,k), &
+             szflux(:,j,k), tauflux(:,j,k))
+      end do
+    end do
+    deallocate(upper, det, dummy)
+  else if (flux_direction == 2) then
+    allocate(upper(ny), det(ny), dummy(ny))
+    do k = GRHydro_stencil, cctk_lsh(3) - GRHydro_stencil + 2
+      do i = GRHydro_stencil, cctk_lsh(1) - GRHydro_stencil + 2
+
+         dummy = betay(i,:,k)
+
+        do j = 1, cctk_lsh(2)
+           det(j) = sdetg(i,j,k)**2
+           call UpperMetric(uxx,uxy,uxz,uyy,uyz,uzz,det(j),&
+                gxx(i,j,k),gxy(i,j,k),gxz(i,j,k),gyy(i,j,k),&
+                gyz(i,j,k),gzz(i,j,k))
+           upper(j) = uyy
+        end do
+
+        call GRHydro_SplitFlux_1D(GRHydro_eos_handle, int(ny,ik),&
+             fs_alpha1, fs_alpha2, fs_alpha3, fs_alpha4, fs_alpha5, &
+             gyy(i,:,k),gyz(i,:,k),gxy(i,:,k),&
+             gzz(i,:,k),gxz(i,:,k),gxx(i,:,k),&
+             upper, det,&
+             alp(i,:,k),dummy,&
+             rho(i,:,k),vel(i,:,k,2),vel(i,:,k,3),vel(i,:,k,1),press(i,:,k),&
+             w_lorentz(i,:,k),eps(i,:,k),&
+             dens(i,:,k),scon(i,:,k,2),scon(i,:,k,3),scon(i,:,k,1),tau(i,:,k),&
+             densfplus(i,:,k), syfplus(i,:,k), szfplus(i,:,k), &
+             sxfplus(i,:,k), taufplus(i,:,k), &
+             densfminus(i,:,k), syfminus(i,:,k), szfminus(i,:,k), &
+             sxfminus(i,:,k), taufminus(i,:,k), &
+             densflux(i,:,k), syflux(i,:,k), szflux(i,:,k), &
+             sxflux(i,:,k), tauflux(i,:,k))
+      end do
+    end do
+    deallocate(upper, det, dummy)
+  else if (flux_direction == 3) then
+    allocate(upper(nz), det(nz), dummy(nz))
+    do j = GRHydro_stencil, cctk_lsh(2) - GRHydro_stencil + 2
+      do i = GRHydro_stencil, cctk_lsh(1) - GRHydro_stencil + 2
+
+         dummy = betaz(i,j,:)
+
+        do k = 1, cctk_lsh(3)
+           det(k) = sdetg(i,j,k)*sdetg(i,j,k)
+           call UpperMetric(uxx,uxy,uxz,uyy,uyz,uzz,det(k),&
+                gxx(i,j,k),gxy(i,j,k),gxz(i,j,k),gyy(i,j,k),&
+                gyz(i,j,k),gzz(i,j,k))
+           upper(k) = uzz
+        end do
+
+        call GRHydro_SplitFlux_1D(GRHydro_eos_handle, int(nz,ik),&
+             fs_alpha1, fs_alpha2, fs_alpha3, fs_alpha4, fs_alpha5, &
+             gzz(i,j,:),gxz(i,j,:),gyz(i,j,:),&
+             gxx(i,j,:),gxy(i,j,:),gyy(i,j,:),&
+             upper, det,&
+             alp(i,j,:),dummy,&
+             rho(i,j,:),vel(i,j,:,3),vel(i,j,:,1),vel(i,j,:,2),press(i,j,:),&
+             w_lorentz(i,j,:),eps(i,j,:),&
+             dens(i,j,:),scon(i,j,:,3),scon(i,j,:,1),scon(i,j,:,2),tau(i,j,:),&
+             densfplus(i,j,:), szfplus(i,j,:), sxfplus(i,j,:), &
+             syfplus(i,j,:), taufplus(i,j,:), &
+             densfminus(i,j,:), szfminus(i,j,:), sxfminus(i,j,:), &
+             syfminus(i,j,:), taufminus(i,j,:), &
+             densflux(i,j,:), szflux(i,j,:), sxflux(i,j,:), &
+             syflux(i,j,:), tauflux(i,j,:))
+      end do
+    end do
+    deallocate(upper, det, dummy)
+  else
+    call CCTK_ERROR("Flux direction not x,y,z")
+    STOP
+  end if
+
+  return
+
+end subroutine GRHydro_SplitFlux
+
+ /*@@
+   @routine    GRHydro_SplitFlux_1D
+   @date       Wed Mar  3 22:55:36 2004
+   @author     Ian Hawke
+   @desc
+   Actually performs the flux splitting.
+   We really should be doing it on a characteristic basis.
+
+   Note the direction rotations on the calling routine.
+   @enddesc
+   @calls
+   @calledby
+   @history
+
+   @endhistory
+
+@@*/
+
+subroutine GRHydro_SplitFlux_1D(handle, nx, &
+     alpha1, alpha2, alpha3, alpha4, alpha5, &
+     gxx, gxy, gxz, gyy, gyz, gzz, u, det, alp, beta, &
+     rho, velx1, vely1, velz1, press, w_lorentz, eps, &
+     dens, sx, sy, sz, tau, &
+     densfplus, sxfplus, syfplus, szfplus, taufplus, &
+     densfminus, sxfminus, syfminus, szfminus, taufminus, &
+     densflux, sxflux, syflux, szflux, tauflux)
+
+  use GRHydro_Eigenproblem
+
+  implicit none
+
+  DECLARE_CCTK_PARAMETERS
+  DECLARE_CCTK_FUNCTIONS
+
+  CCTK_INT, parameter :: izero = 0
+  integer, parameter :: ik = kind(izero)
+
+  CCTK_REAL, parameter :: half = 0.5d0
+
+  CCTK_INT :: i, nx, handle, ll
+  CCTK_REAL, dimension(nx) :: gxx, gxy, gxz, gyy, gyz, gzz, &
+       u, det, alp, beta, &
+       rho, velx1, vely1, velz1, press, w_lorentz, eps, &
+       dens, sx, sy, sz, tau, &
+       densfplus, sxfplus, syfplus, szfplus, taufplus, &
+       densfminus, sxfminus, syfminus, szfminus, taufminus, &
+       densflux, sxflux, syflux, szflux, tauflux
+
+  CCTK_REAL :: alpha1, alpha2, alpha3, alpha4, alpha5
+
+  CCTK_REAL, dimension(:), allocatable :: tmp
+
+  CCTK_REAL, dimension(:,:), allocatable :: evals
+  CCTK_REAL, dimension(:,:,:), allocatable :: levecs, revecs
+
+  CCTK_REAL, dimension(:), allocatable :: &
+     char_v1, char_v2, char_v3, char_v4 ,char_v5, &
+     char_f1_plus, char_f2_plus, char_f3_plus, &
+     char_f4_plus ,char_f5_plus, &
+     char_f1_minus, char_f2_minus, char_f3_minus, &
+     char_f4_minus ,char_f5_minus, &
+     char_f1, char_f2, char_f3, &
+     char_f4 ,char_f5
+
+  CCTK_REAL, dimension(5) :: lambda, alpha
+  CCTK_REAL, dimension(5, 5) :: levec, revec
+
+  densfplus = 0.d0
+  sxfplus = 0.d0
+  syfplus = 0.d0
+  szfplus = 0.d0
+  taufplus = 0.d0
+  densfminus = 0.d0
+  sxfminus = 0.d0
+  syfminus = 0.d0
+  szfminus = 0.d0
+  taufminus = 0.d0
+
+  allocate(evals(nx, 5), levecs(nx,5,5), revecs(nx,5,5))
+  allocate(tmp(nx))
+  allocate(char_v1(nx), char_v2(nx), char_v3(nx), char_v4 (nx),char_v5(nx), &
+     char_f1_plus(nx), char_f2_plus(nx), char_f3_plus(nx), &
+     char_f4_plus (nx),char_f5_plus(nx), &
+     char_f1_minus(nx), char_f2_minus(nx), char_f3_minus(nx), &
+     char_f4_minus (nx),char_f5_minus(nx), &
+     char_f1(nx), char_f2(nx), char_f3(nx), &
+     char_f4 (nx),char_f5(nx))
+
+  do i = 1, nx-1
+
+!!$    Arithmetic mean
+
+!!$    Calculate the maximum eigenvalue and put it here
+
+    call eigenproblem_leftright(handle, &
+         half * (rho      (i) + rho      (i+1)), &
+         half * (velx1    (i) + velx1    (i+1)), &
+         half * (vely1    (i) + vely1    (i+1)), &
+         half * (velz1    (i) + velz1    (i+1)), &
+         half * (eps      (i) + eps      (i+1)), &
+         half * (w_lorentz(i) + w_lorentz(i+1)), &
+         half * (gxx      (i) + gxx      (i+1)), &
+         half * (gxy      (i) + gxy      (i+1)), &
+         half * (gxz      (i) + gxz      (i+1)), &
+         half * (gyy      (i) + gyy      (i+1)), &
+         half * (gyz      (i) + gyz      (i+1)), &
+         half * (gzz      (i) + gzz      (i+1)), &
+         half * (u        (i) + u        (i+1)), &
+         half * (alp      (i) + alp      (i+1)), &
+         half * (beta     (i) + beta     (i+1)), &
+         lambda,&
+         levec,&
+         revec)
+
+    evals(i,:) = lambda
+    levecs(i,:,:) = levec
+    revecs(i,:,:) = revec
+
+  end do
+
+  do i = 3, nx - 3
+
+    alpha(1) = alpha1
+    alpha(2) = alpha2
+    alpha(3) = alpha3
+    alpha(4) = alpha4
+    alpha(5) = alpha5
+
+    do ll = i - 2, i + 3
+
+!!$    Initialize the pointwise fluxes temporarily into the minus
+
+      call num_x_flux(dens(ll), sx(ll), sy(ll), sz(ll), tau(ll), &
+           densfminus(ll), sxfminus(ll), syfminus(ll), szfminus(ll), &
+           taufminus(ll), &
+           velx1(ll), press(ll), det(ll), alp(ll), beta(ll))
+
+      densfminus(ll) = densfminus(ll) * alp(ll)
+      sxfminus(ll)   = sxfminus(ll)   * alp(ll)
+      syfminus(ll)   = syfminus(ll)   * alp(ll)
+      szfminus(ll)   = szfminus(ll)   * alp(ll)
+      taufminus(ll)  = taufminus(ll)  * alp(ll)
+
+      char_v1(ll) = levecs(i,1,1) * dens(ll) + &
+           levecs(i,1,2) * sx(ll) + &
+           levecs(i,1,3) * sy(ll) + &
+           levecs(i,1,4) * sz(ll) + &
+           levecs(i,1,5) * tau(ll)
+      char_v2(ll) = levecs(i,2,1) * dens(ll) + &
+           levecs(i,2,2) * sx(ll) + &
+           levecs(i,2,3) * sy(ll) + &
+           levecs(i,2,4) * sz(ll) + &
+           levecs(i,2,5) * tau(ll)
+      char_v3(ll) = levecs(i,3,1) * dens(ll) + &
+           levecs(i,3,2) * sx(ll) + &
+           levecs(i,3,3) * sy(ll) + &
+           levecs(i,3,4) * sz(ll) + &
+           levecs(i,3,5) * tau(ll)
+      char_v4(ll) = levecs(i,4,1) * dens(ll) + &
+           levecs(i,4,2) * sx(ll) + &
+           levecs(i,4,3) * sy(ll) + &
+           levecs(i,4,4) * sz(ll) + &
+           levecs(i,4,5) * tau(ll)
+      char_v5(ll) = levecs(i,5,1) * dens(ll) + &
+           levecs(i,5,2) * sx(ll) + &
+           levecs(i,5,3) * sy(ll) + &
+           levecs(i,5,4) * sz(ll) + &
+           levecs(i,5,5) * tau(ll)
+
+      char_f1(ll) = levecs(i,1,1) * densfminus(ll) + &
+           levecs(i,1,2) * sxfminus(ll) + &
+           levecs(i,1,3) * syfminus(ll) + &
+           levecs(i,1,4) * szfminus(ll) + &
+           levecs(i,1,5) * taufminus(ll)
+      char_f2(ll) = levecs(i,2,1) * densfminus(ll) + &
+           levecs(i,2,2) * sxfminus(ll) + &
+           levecs(i,2,3) * syfminus(ll) + &
+           levecs(i,2,4) * szfminus(ll) + &
+           levecs(i,2,5) * taufminus(ll)
+      char_f3(ll) = levecs(i,3,1) * densfminus(ll) + &
+           levecs(i,3,2) * sxfminus(ll) + &
+           levecs(i,3,3) * syfminus(ll) + &
+           levecs(i,3,4) * szfminus(ll) + &
+           levecs(i,3,5) * taufminus(ll)
+      char_f4(ll) = levecs(i,4,1) * densfminus(ll) + &
+           levecs(i,4,2) * sxfminus(ll) + &
+           levecs(i,4,3) * syfminus(ll) + &
+           levecs(i,4,4) * szfminus(ll) + &
+           levecs(i,4,5) * taufminus(ll)
+      char_f5(ll) = levecs(i,5,1) * densfminus(ll) + &
+           levecs(i,5,2) * sxfminus(ll) + &
+           levecs(i,5,3) * syfminus(ll) + &
+           levecs(i,5,4) * szfminus(ll) + &
+           levecs(i,5,5) * taufminus(ll)
+
+!!$    Calculate the split
+
+      char_f1_plus(ll)  = 0.5d0 * (char_f1(ll) + alpha(1) * char_v1(ll))
+      char_f1_minus(ll) = 0.5d0 * (char_f1(ll) - alpha(1) * char_v1(ll))
+      char_f2_plus(ll)  = 0.5d0 * (char_f2(ll) + alpha(2) * char_v2(ll))
+      char_f2_minus(ll) = 0.5d0 * (char_f2(ll) - alpha(2) * char_v2(ll))
+      char_f3_plus(ll)  = 0.5d0 * (char_f3(ll) + alpha(3) * char_v3(ll))
+      char_f3_minus(ll) = 0.5d0 * (char_f3(ll) - alpha(3) * char_v3(ll))
+      char_f4_plus(ll)  = 0.5d0 * (char_f4(ll) + alpha(4) * char_v4(ll))
+      char_f4_minus(ll) = 0.5d0 * (char_f4(ll) - alpha(4) * char_v4(ll))
+      char_f5_plus(ll)  = 0.5d0 * (char_f5(ll) + alpha(5) * char_v5(ll))
+      char_f5_minus(ll) = 0.5d0 * (char_f5(ll) - alpha(5) * char_v5(ll))
+
+    end do
+
+!!$    Reconstruct the characteristic split fluxes
+!!$    After reconstruction, combine to get the characteristic flux
+
+    call GRHydro_WENO5_Left(5_ik, char_f1_plus(i-2:i+2), tmp(i-2:i+2))
+    char_f1(i) = tmp(i)
+    call GRHydro_WENO5_Right(5_ik, char_f1_minus(i-1:i+3), &
+         tmp(i-1:i+3))
+    char_f1(i) = char_f1(i) + tmp(i+1)
+
+    call GRHydro_WENO5_Left(5_ik, char_f2_plus(i-2:i+2), tmp(i-2:i+2))
+    char_f2(i) = tmp(i)
+    call GRHydro_WENO5_Right(5_ik, char_f2_minus(i-1:i+3), &
+         tmp(i-1:i+3))
+    char_f2(i) = char_f2(i) + tmp(i+1)
+
+    call GRHydro_WENO5_Left(5_ik, char_f3_plus(i-2:i+2), tmp(i-2:i+2))
+    char_f3(i) = tmp(i)
+    call GRHydro_WENO5_Right(5_ik, char_f3_minus(i-1:i+3), &
+         tmp(i-1:i+3))
+    char_f3(i) = char_f3(i) + tmp(i+1)
+
+    call GRHydro_WENO5_Left(5_ik, char_f4_plus(i-2:i+2), tmp(i-2:i+2))
+    char_f4(i) = tmp(i)
+    call GRHydro_WENO5_Right(5_ik, char_f4_minus(i-1:i+3), &
+         tmp(i-1:i+3))
+    char_f4(i) = char_f4(i) + tmp(i+1)
+
+    call GRHydro_WENO5_Left(5_ik, char_f5_plus(i-2:i+2), tmp(i-2:i+2))
+    char_f5(i) = tmp(i)
+    call GRHydro_WENO5_Right(5_ik, char_f5_minus(i-1:i+3), &
+         tmp(i-1:i+3))
+    char_f5(i) = char_f5(i) + tmp(i+1)
+
+!!$    Compute physical fluxes
+
+    densflux(i) = &
+         revecs(i,1,1) * char_f1(i) + revecs(i,2,1) * char_f2(i) + &
+         revecs(i,3,1) * char_f3(i) + revecs(i,4,1) * char_f4(i) + &
+         revecs(i,5,1) * char_f5(i)
+    sxflux(i) = &
+         revecs(i,1,2) * char_f1(i) + revecs(i,2,2) * char_f2(i) + &
+         revecs(i,3,2) * char_f3(i) + revecs(i,4,2) * char_f4(i) + &
+         revecs(i,5,2) * char_f5(i)
+    syflux(i) = &
+         revecs(i,1,3) * char_f1(i) + revecs(i,2,3) * char_f2(i) + &
+         revecs(i,3,3) * char_f3(i) + revecs(i,4,3) * char_f4(i) + &
+         revecs(i,5,3) * char_f5(i)
+    szflux(i) = &
+         revecs(i,1,4) * char_f1(i) + revecs(i,2,4) * char_f2(i) + &
+         revecs(i,3,4) * char_f3(i) + revecs(i,4,4) * char_f4(i) + &
+         revecs(i,5,4) * char_f5(i)
+    tauflux(i) = &
+         revecs(i,1,5) * char_f1(i) + revecs(i,2,5) * char_f2(i) + &
+         revecs(i,3,5) * char_f3(i) + revecs(i,4,5) * char_f4(i) + &
+         revecs(i,5,5) * char_f5(i)
+
+!!$    if (abs(i-200) < 5) then
+!!$      write(*,*) i, 'alpha',alpha1,alpha2,alpha3,alpha4,alpha5
+!!$      write(*,*) i, 'var',dens(i), sx(i), tau(i)
+!!$      write(*,*) i, 'f',densflux(i),sxflux(i),tauflux(i)
+!!$    end if
+
+  end do
+
+  deallocate(evals, levecs, revecs, tmp)
+
+end subroutine GRHydro_SplitFlux_1D
+
+ /*@@
+   @routine    GRHydro_WENO5_Left
+   @date       Wed Mar  3 22:57:54 2004
+   @author     Ian Hawke
+   @desc
+   Upwind biased WENO5.
+   @enddesc
+   @calls
+   @calledby
+   @history
+
+   @endhistory
+
+@@*/
+
+subroutine GRHydro_WENO5_Left(nx, v, vminus)
+
+  implicit none
+
+  CCTK_INT :: i, nx
+  CCTK_REAL, dimension(nx) :: v, vminus
+  CCTK_REAL :: one, two, three, four, five, six, seven, &
+       ten, eleven, twelve, thirteen, &
+       ThirteenByTwelve, Quarter
+  parameter (one = 1)
+  parameter (two = 2)
+  parameter (three = 3)
+  parameter (four = 4)
+  parameter (five = 5)
+  parameter (six = 6)
+  parameter (seven = 7)
+  parameter (ten = 10)
+  parameter (eleven = 11)
+  parameter (twelve = 12)
+  parameter (thirteen = 13)
+  parameter (ThirteenByTwelve = thirteen / twelve)
+  parameter (Quarter = one / four)
+  CCTK_REAL :: d0, d1, d2
+  parameter (d0 = three / ten)
+  parameter (d1 = six / ten)
+  parameter (d2 = one / ten)
+  CCTK_REAL :: c00,c01,c02,c10,c11,c12,c20,c21,c22
+  parameter (c00 = two / six)
+  parameter (c01 = five / six)
+  parameter (c02 = -one / six)
+  parameter (c10 = -one / six)
+  parameter (c11 = five / six)
+  parameter (c12 = two / six)
+  parameter (c20 = two / six)
+  parameter (c21 = -seven / six)
+  parameter (c22 = eleven / six)
+
+  CCTK_REAL :: beta0, beta1, beta2
+  CCTK_REAL :: epsilon
+  CCTK_REAL :: alpha0, alpha1, alpha2, alphasum
+  CCTK_REAL :: w0, w1, w2
+  CCTK_REAL :: v0plushalf, v1plushalf, v2plushalf
+
+  epsilon = 1.d-6
+
+  do i = 3, nx-2
+
+    beta0 = ThirteenByTwelve * (v(i  ) - two * v(i+1) + v(i+2))**2 + &
+         Quarter * (three * v(i  ) - four * v(i+1) +         v(i+2))**2
+    beta1 = ThirteenByTwelve * (v(i-1) - two * v(i  ) + v(i+1))**2 + &
+         Quarter * (        v(i-1)                 -         v(i+1))**2
+    beta2 = ThirteenByTwelve * (v(i-2) - two * v(i-1) + v(i  ))**2 + &
+         Quarter * (        v(i-2) - four * v(i-1) + three * v(i  ))**2
+
+    alpha0 = d0 / (epsilon + beta0)**2
+    alpha1 = d1 / (epsilon + beta1)**2
+    alpha2 = d2 / (epsilon + beta2)**2
+
+    alphasum = alpha0 + alpha1 + alpha2
+
+    w0 = alpha0 / alphasum
+    w1 = alpha1 / alphasum
+    w2 = alpha2 / alphasum
+
+    v0plushalf = c00 * v(i  ) + c01 * v(i+1) + c02 * v(i+2)
+    v1plushalf = c10 * v(i-1) + c11 * v(i  ) + c12 * v(i+1)
+    v2plushalf = c20 * v(i-2) + c21 * v(i-1) + c22 * v(i  )
+
+    vminus(i) = w0 * v0plushalf + &
+                w1 * v1plushalf + &
+                w2 * v2plushalf
+
+  end do
+
+end subroutine GRHydro_WENO5_Left
+
+ /*@@
+   @routine    GRHydro_WENO5_Right
+   @date       Wed Mar  3 22:58:10 2004
+   @author     Ian Hawke
+   @desc
+   Downwind biased WENO5.
+   @enddesc
+   @calls
+   @calledby
+   @history
+
+   @endhistory
+
+@@*/
+
+subroutine GRHydro_WENO5_Right(nx, v, vplus)
+
+  implicit none
+
+  CCTK_INT :: i, nx
+  CCTK_REAL, dimension(nx) :: v, vplus
+  CCTK_REAL :: one, two, three, four, five, six, seven, &
+       ten, eleven, twelve, thirteen, &
+       ThirteenByTwelve, Quarter
+  parameter (one = 1)
+  parameter (two = 2)
+  parameter (three = 3)
+  parameter (four = 4)
+  parameter (five = 5)
+  parameter (six = 6)
+  parameter (seven = 7)
+  parameter (ten = 10)
+  parameter (eleven = 11)
+  parameter (twelve = 12)
+  parameter (thirteen = 13)
+  parameter (ThirteenByTwelve = thirteen / twelve)
+  parameter (Quarter = one / four)
+  CCTK_REAL :: dtilde0, dtilde1, dtilde2
+  parameter (dtilde0 = one / ten)
+  parameter (dtilde1 = six / ten)
+  parameter (dtilde2 = three / ten)
+  CCTK_REAL :: ctilde00,ctilde01,ctilde02,ctilde10,ctilde11,&
+       ctilde12,ctilde20,ctilde21,ctilde22
+  parameter (ctilde00 = eleven / six)
+  parameter (ctilde01 = -seven / six)
+  parameter (ctilde02 = two / six)
+  parameter (ctilde10 = two / six)
+  parameter (ctilde11 = five / six)
+  parameter (ctilde12 = -one / six)
+  parameter (ctilde20 = -one / six)
+  parameter (ctilde21 = five / six)
+  parameter (ctilde22 = two / six)
+
+  CCTK_REAL :: betatilde0, betatilde1, betatilde2
+  CCTK_REAL :: epsilon
+  CCTK_REAL :: alphatilde0, alphatilde1, alphatilde2, alphatildesum
+  CCTK_REAL :: wtilde0, wtilde1, wtilde2
+  CCTK_REAL :: v0minushalf, v1minushalf, v2minushalf
+
+  epsilon = 1.d-6
+
+  do i = 3, nx-2
+
+    betatilde0 = ThirteenByTwelve * (v(i  ) - two * v(i+1) + v(i+2))**2 + &
+         Quarter * (three * v(i  ) - four * v(i+1) +         v(i+2))**2
+    betatilde1 = ThirteenByTwelve * (v(i-1) - two * v(i  ) + v(i+1))**2 + &
+         Quarter * (        v(i-1)                 -         v(i+1))**2
+    betatilde2 = ThirteenByTwelve * (v(i-2) - two * v(i-1) + v(i  ))**2 + &
+         Quarter * (        v(i-2) - four * v(i-1) + three * v(i  ))**2
+
+    alphatilde0 = dtilde0 / (epsilon + betatilde0)**2
+    alphatilde1 = dtilde1 / (epsilon + betatilde1)**2
+    alphatilde2 = dtilde2 / (epsilon + betatilde2)**2
+
+    alphatildesum = alphatilde0 + alphatilde1 + alphatilde2
+
+    wtilde0 = alphatilde0 / alphatildesum
+    wtilde1 = alphatilde1 / alphatildesum
+    wtilde2 = alphatilde2 / alphatildesum
+
+    v0minushalf = ctilde00 * v(i  ) + ctilde01 * v(i+1) + ctilde02 * v(i+2)
+    v1minushalf = ctilde10 * v(i-1) + ctilde11 * v(i  ) + ctilde12 * v(i+1)
+    v2minushalf = ctilde20 * v(i-2) + ctilde21 * v(i-1) + ctilde22 * v(i  )
+
+    vplus(i) = wtilde0 * v0minushalf + &
+               wtilde1 * v1minushalf + &
+               wtilde2 * v2minushalf
+
+  end do
+
+end subroutine GRHydro_WENO5_Right


Add missing GRHydro_HLLEM.F90 in EinsteinEvolve/GRHydro (einsteintoolkit/Cactus#14)

This PR adds the missing source file GRHydro_HLLEM.F90 to complete the EinsteinEvolve/GRHydro thorn implementation.

Implement the full functionality of GRHydro

Please implement the full functionality of `GRHydro` by finishing the code in `GRHydro_HLLEM.F90`.

## Thorn Information:
- Name: GRHydro
- Target file: GRHydro_HLLEM.F90

## Interface Definition in interface.ccl:

## Schedule Definition in schedule.ccl:

## Parameters Definition in param.ccl:

## Configuration Definition in configuration.ccl:

## Documentation Context in readme.md and doc/documentation.tex

## Related Code Context in src folder

diff --git a/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_HLLEM.F90 b/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_HLLEM.F90
new file mode 100644
index 0000000..0000000
--- /dev/null
+++ b/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_HLLEM.F90
@@ -0,0 +1,823 @@
+ /*@@
+   @file      GRHydro_HLLEPolyM.F90
+   @date      Aug 30, 2010
+   @author    Joshua Faber, Scott Noble, Bruno Mundim, Ian Hawke, Pedro Montero, Toni Font
+   @desc
+   The HLLE solver. Called from the wrapper function, so works in
+   all directions.
+   @enddesc
+ @@*/
+
+#include "cctk.h"
+#include "cctk_Parameters.h"
+#include "cctk_Arguments.h"
+#include "cctk_Functions.h"
+
+#include "GRHydro_Macros.h"
+#include "SpaceMask.h"
+
+ /*@@
+   @routine    GRHydro_HLLEM
+   @date       Aug 30, 2010
+   @author     Joshua Faber, Scott Noble, Bruno Mundim, Ian Hawke, Pedro Montero, Toni Font
+   @desc
+   The HLLE solver. Sufficiently simple that its just one big routine.
+   @enddesc
+   @calls
+   @calledby
+   @history
+   Altered from Cactus 3 routines originally written by Toni Font.
+   @endhistory
+
+@@*/
+
+subroutine GRHydro_HLLEM(CCTK_ARGUMENTS)
+  USE GRHydro_EigenproblemM
+  USE GRHydro_Scalars
+
+  implicit none
+
+  DECLARE_CCTK_ARGUMENTS
+  DECLARE_CCTK_PARAMETERS
+  DECLARE_CCTK_FUNCTIONS
+
+  CCTK_INT, parameter :: izero = 0
+  integer, parameter :: ik = kind(izero)
+
+  integer :: i, j, k, m
+  CCTK_REAL, dimension(8) :: cons_p,cons_m,fplus,fminus,f1,qdiff
+  CCTK_REAL, dimension(10) :: prim_p, prim_m
+  CCTK_REAL, dimension(5) :: lamminus,lamplus
+  CCTK_REAL ::  charmin, charmax, charpm,chartop,avg_alp,avg_det, sdet
+  CCTK_REAL, dimension(3,3) :: gh, uh
+  CCTK_REAL :: usendh
+  CCTK_REAL :: rhoenth_p, rhoenth_m
+  CCTK_REAL, dimension(3) :: avg_beta
+  CCTK_REAL, dimension(3) :: vtp,vtm,blowp,blowm,Bveclowp,Bveclowm
+  CCTK_REAL, dimension(3) :: vellowp,vellowm
+  CCTK_REAL :: ab0p,ab0m,b2p,b2m,bdotvp,bdotvm
+  CCTK_REAL :: wp,wm,v2p,v2m
+  CCTK_REAL :: pressstarp,pressstarm
+
+  CCTK_REAL :: entropyconsp,entropyconsm,entropyp,entropym,entropyf,entropydiff,entropyfp,entropyfm
+
+  CCTK_REAL :: psidcp, psidcm, psidcf, psidcdiff, psidcfp, psidcfm
+  CCTK_REAL :: charmax_dc, charmin_dc, charpm_dc
+
+  CCTK_INT :: type_bits, trivial
+  CCTK_REAL :: xtemp
+
+  integer :: ix,iy,iz
+
+  ! save memory when MP is not used
+  CCTK_INT :: GRHydro_UseGeneralCoordinates
+  CCTK_REAL, DIMENSION(cctk_ash1,cctk_ash2,cctk_ash3) :: g11, g12, g13, g22, g23, g33
+  pointer (pg11,g11), (pg12,g12), (pg13,g13), (pg22,g22), (pg23,g23), (pg33,g33)
+  CCTK_REAL, DIMENSION(cctk_ash1,cctk_ash2,cctk_ash3) :: beta1, beta2, beta3
+  pointer (pbeta1,beta1), (pbeta2,beta2), (pbeta3,beta3)
+
+  if (GRHydro_UseGeneralCoordinates(cctkGH).ne.0) then
+    pg11 = loc(gaa)
+    pg12 = loc(gab)
+    pg13 = loc(gac)
+    pg22 = loc(gbb)
+    pg23 = loc(gbc)
+    pg33 = loc(gcc)
+    pbeta1 = loc(betaa)
+    pbeta2 = loc(betab)
+    pbeta3 = loc(betac)
+  else
+    pg11 = loc(gxx)
+    pg12 = loc(gxy)
+    pg13 = loc(gxz)
+    pg22 = loc(gyy)
+    pg23 = loc(gyz)
+    pg33 = loc(gzz)
+    pbeta1 = loc(betax)
+    pbeta2 = loc(betay)
+    pbeta3 = loc(betaz)
+  end if
+#define gxx faulty_gxx
+#define gxy faulty_gxy
+#define gxz faulty_gxz
+#define gyy faulty_gyy
+#define gyz faulty_gyz
+#define gzz faulty_gzz
+#define betax faulty_betax
+#define betay faulty_betay
+#define betaz faulty_betaz
+#define vel faulty_vel
+#define Bvec faulty_Bvec
+
+  if (flux_direction == 1) then
+    call SpaceMask_GetTypeBits(type_bits, "Hydro_RiemannProblemX")
+    call SpaceMask_GetStateBits(trivial, "Hydro_RiemannProblemX", &
+         &"trivial")
+    ix = 1 ; iy = 2 ; iz = 3
+  else if (flux_direction == 2) then
+    call SpaceMask_GetTypeBits(type_bits, "Hydro_RiemannProblemY")
+    call SpaceMask_GetStateBits(trivial, "Hydro_RiemannProblemY", &
+         &"trivial")
+    ix = 2 ; iy = 3 ; iz = 1
+  else if (flux_direction == 3) then
+    call SpaceMask_GetTypeBits(type_bits, "Hydro_RiemannProblemZ")
+    call SpaceMask_GetStateBits(trivial, "Hydro_RiemannProblemZ", &
+         &"trivial")
+    ix = 3 ; iy = 1 ; iz = 2
+  else
+    call CCTK_ERROR("Flux direction not x,y,z")
+    STOP
+  end if
+
+  ! constraint transport needs to be able to average fluxes in the directions
+  ! other that flux_direction
+
+  !$OMP PARALLEL PRIVATE(k,j,i,f1,lamminus,lamplus,cons_p,cons_m,fplus,fminus,qdiff,psidcf,psidcp,psidcm,prim_p,prim_m,&
+  !$OMP avg_beta,avg_alp,&
+  !$OMP gh,avg_det,sdet,uh,&
+  !$OMP vtp,vtm,vellowp,vellowm,Bveclowp,Bveclowm,&
+  !$OMP bdotvp,bdotvm,b2p,b2m,v2p,v2m,wp,wm,&
+  !$OMP blowp,blowm,&
+  !$OMP rhoenth_p,rhoenth_m,ab0p,ab0m,pressstarp,pressstarm,&
+  !$OMP usendh,psidcdiff,psidcfp,psidcfm,charmin,charmax,chartop,charpm,&
+  !$OMP charmin_dc,charmax_dc,charpm_dc,m,xtemp,&
+  !$OMP entropyconsp,entropyconsm,entropyp,entropym,entropyf,entropydiff,entropyfp,entropyfm)
+  ! avoid compiler warnings
+  psidcf = 0d0
+  psidcp = 0.d0
+  psidcm = 0d0
+  psidcfp = 0d0
+  psidcfm = 0d0
+
+  entropyf = 0d0
+  entropyfp = 0d0
+  entropyfm = 0d0
+  entropyconsm = 0d0
+  entropyconsp = 0d0
+  !$OMP DO
+  do k = GRHydro_stencil, cctk_lsh(3) - GRHydro_stencil + transport_constraints*(1-zoffset)
+    do j = GRHydro_stencil, cctk_lsh(2) - GRHydro_stencil + transport_constraints*(1-yoffset)
+      do i = GRHydro_stencil, cctk_lsh(1) - GRHydro_stencil + transport_constraints*(1-xoffset)
+
+        f1 = 0.d0
+        lamminus = 0.d0
+        lamplus = 0.d0
+        cons_p = 0.d0
+        cons_m = 0.d0
+        fplus = 0.d0
+        fminus = 0.d0
+        qdiff = 0.d0
+
+!!$        Set the left (p for plus) and right (m_i for minus, i+1) states
+
+        cons_p(1)   = densplus(i,j,k)
+        cons_p(2)   = sxplus(i,j,k)
+        cons_p(3)   = syplus(i,j,k)
+        cons_p(4)   = szplus(i,j,k)
+        cons_p(5)   = tauplus(i,j,k)
+        cons_p(6)   = Bconsxplus(i,j,k)
+        cons_p(7)   = Bconsyplus(i,j,k)
+        cons_p(8)   = Bconszplus(i,j,k)
+
+        cons_m(1) = densminus(i+xoffset,j+yoffset,k+zoffset)
+        cons_m(2) = sxminus(i+xoffset,j+yoffset,k+zoffset)
+        cons_m(3) = syminus(i+xoffset,j+yoffset,k+zoffset)
+        cons_m(4) = szminus(i+xoffset,j+yoffset,k+zoffset)
+        cons_m(5) = tauminus(i+xoffset,j+yoffset,k+zoffset)
+        cons_m(6) = Bconsxminus(i+xoffset,j+yoffset,k+zoffset)
+        cons_m(7) = Bconsyminus(i+xoffset,j+yoffset,k+zoffset)
+        cons_m(8) = Bconszminus(i+xoffset,j+yoffset,k+zoffset)
+
+        prim_p(1)   = rhoplus(i,j,k)
+        prim_p(2)   = velxplus(i,j,k)
+        prim_p(3)   = velyplus(i,j,k)
+        prim_p(4)   = velzplus(i,j,k)
+        prim_p(5)   = epsplus(i,j,k)
+        prim_p(6)   = pressplus(i,j,k)
+        prim_p(7)   = w_lorentzplus(i,j,k)
+        prim_p(8)   = Bvecxplus(i,j,k)
+        prim_p(9)   = Bvecyplus(i,j,k)
+        prim_p(10)  = Bveczplus(i,j,k)
+
+        prim_m(1) = rhominus(i+xoffset,j+yoffset,k+zoffset)
+        prim_m(2) = velxminus(i+xoffset,j+yoffset,k+zoffset)
+        prim_m(3) = velyminus(i+xoffset,j+yoffset,k+zoffset)
+        prim_m(4) = velzminus(i+xoffset,j+yoffset,k+zoffset)
+        prim_m(5) = epsminus(i+xoffset,j+yoffset,k+zoffset)
+        prim_m(6) = pressminus(i+xoffset,j+yoffset,k+zoffset)
+        prim_m(7) = w_lorentzminus(i+xoffset,j+yoffset,k+zoffset)
+        prim_m(8) = Bvecxminus(i+xoffset,j+yoffset,k+zoffset)
+        prim_m(9) = Bvecyminus(i+xoffset,j+yoffset,k+zoffset)
+        prim_m(10)= Bveczminus(i+xoffset,j+yoffset,k+zoffset)
+
+        if(clean_divergence.ne.0) then
+           psidcp = psidcplus(i,j,k)
+           psidcm = psidcminus(i+xoffset,j+yoffset,k+zoffset)
+        endif
+
+        if(evolve_entropy.ne.0) then
+           entropyp = entropyplus(i,j,k)
+           entropym = entropyminus(i+xoffset,j+yoffset,k+zoffset)
+           entropyconsp = entropyconsplus(i,j,k)
+           entropyconsm = entropyconsminus(i+xoffset,j+yoffset,k+zoffset)
+        endif
+
+!!$        Calculate various metric terms here.
+!!$        Note also need the average of the lapse at the
+!!$        left and right points.
+!!$
+!!$        In MHD, we need all three shift components regardless of the flux direction
+
+        avg_beta(1) = 0.5d0 * (beta1(i+xoffset,j+yoffset,k+zoffset) + &
+             beta1(i,j,k))
+        avg_beta(2) = 0.5d0 * (beta2(i+xoffset,j+yoffset,k+zoffset) + &
+             beta2(i,j,k))
+        avg_beta(3) = 0.5d0 * (beta3(i+xoffset,j+yoffset,k+zoffset) + &
+             beta3(i,j,k))
+
+        avg_alp = 0.5 * (alp(i,j,k) + alp(i+xoffset,j+yoffset,k+zoffset))
+
+        gh(1,1) = 0.5d0 * (g11(i+xoffset,j+yoffset,k+zoffset) + g11(i,j,k))
+        gh(1,2) = 0.5d0 * (g12(i+xoffset,j+yoffset,k+zoffset) + g12(i,j,k))
+        gh(1,3) = 0.5d0 * (g13(i+xoffset,j+yoffset,k+zoffset) + g13(i,j,k))
+        gh(2,2) = 0.5d0 * (g22(i+xoffset,j+yoffset,k+zoffset) + g22(i,j,k))
+        gh(2,3) = 0.5d0 * (g23(i+xoffset,j+yoffset,k+zoffset) + g23(i,j,k))
+        gh(3,3) = 0.5d0 * (g33(i+xoffset,j+yoffset,k+zoffset) + g33(i,j,k))
+        gh(2,1) = gh(1,2) ; gh(3,1) = gh(1,3) ; gh(3,2) = gh(2,3)
+
+        avg_det =  SPATIAL_DETERMINANT(gh(1,1),gh(1,2),gh(1,3),gh(2,2),gh(2,3),gh(3,3))
+        sdet = sqrt(avg_det)
+
+        call UpperMetric(uh(1,1), uh(1,2), uh(1,3), uh(2,2), uh(2,3), uh(3,3), &
+             avg_det,gh(1,1), gh(1,2), gh(1,3), &
+             gh(2,2), gh(2,3), gh(3,3))
+        uh(2,1) = uh(1,2) ; uh(3,1) = uh(1,3) ; uh(3,2) = uh(2,3)
+
+
+        vtp(1) = prim_p(2)-avg_beta(1)/avg_alp
+        vtp(2) = prim_p(3)-avg_beta(2)/avg_alp
+        vtp(3) = prim_p(4)-avg_beta(3)/avg_alp
+        vtm(1) = prim_m(2)-avg_beta(1)/avg_alp
+        vtm(2) = prim_m(3)-avg_beta(2)/avg_alp
+        vtm(3) = prim_m(4)-avg_beta(3)/avg_alp
+
+        call calc_vlow_blow(gh(1,1),gh(1,2),gh(1,3),gh(2,2),gh(2,3),gh(3,3), &
+             prim_p(2),prim_p(3),prim_p(4),prim_p(8),prim_p(9),prim_p(10), &
+             vellowp(1),vellowp(2),vellowp(3),Bveclowp(1),Bveclowp(2),Bveclowp(3), &
+             bdotvp,b2p,v2p,wp,blowp(1),blowp(2),blowp(3))
+        call calc_vlow_blow(gh(1,1),gh(1,2),gh(1,3),gh(2,2),gh(2,3),gh(3,3), &
+             prim_m(2),prim_m(3),prim_m(4),prim_m(8),prim_m(9),prim_m(10), &
+             vellowm(1),vellowm(2),vellowm(3),Bveclowm(1),Bveclowm(2),Bveclowm(3), &
+             bdotvm,b2m,v2m,wm,blowm(1),blowm(2),blowm(3))
+
+        rhoenth_p = prim_p(1)*(1.d0+prim_p(5))+prim_p(6)
+        rhoenth_m = prim_m(1)*(1.d0+prim_m(5))+prim_m(6)
+
+        ab0p = wp*bdotvp
+        ab0m = wm*bdotvm
+
+!!$ p^*  = p+b^2/2  in Anton et al.
+        pressstarp = prim_p(6)+0.5d0*b2p
+        pressstarm = prim_m(6)+0.5d0*b2m
+
+
+!!$ If the Riemann problem is trivial, just calculate the fluxes from the
+!!$ left state and skip to the next cell
+
+        if (SpaceMask_CheckStateBitsF90(space_mask, i, j, k, type_bits, trivial)) then
+
+!!$ we now pass in the B-field as conserved and flux, the vtilde's instead of v's,
+!!$ pressstar instead of P, b_i, alp b^0, w, metric determinant,
+!!$ alp, and beta in the flux dir
+
+          call num_x_fluxM(cons_p(1),cons_p(1+ix),cons_p(1+iy),cons_p(1+iz),&
+               cons_p(5),cons_p(5+ix),cons_p(5+iy),cons_p(5+iz),&
+               f1(1),f1(1+ix),f1(1+iy),f1(1+iz),f1(5),f1(5+ix),f1(5+iy),&
+               f1(5+iz),&
+               vtp(ix),vtp(iy),vtp(iz),pressstarp,blowp(ix),blowp(iy),&
+               blowp(iz),ab0p,wp, &
+               avg_det,avg_alp,avg_beta(flux_direction))
+          if(clean_divergence.ne.0) then
+             f1(6)=f1(6) + 1.0d0*sdet*uh(flux_direction,1)*psidcp - cons_p(5+flux_direction)*avg_beta(1)/avg_alp
+             f1(7)=f1(7) + 1.0d0*sdet*uh(flux_direction,2)*psidcp - cons_p(5+flux_direction)*avg_beta(2)/avg_alp
+             f1(8)=f1(8) + 1.0d0*sdet*uh(flux_direction,3)*psidcp - cons_p(5+flux_direction)*avg_beta(3)/avg_alp
+             psidcf = cons_p(5+flux_direction)/sdet-psidcp*avg_beta(flux_direction)/avg_alp
+          endif
+          if(evolve_entropy.ne.0) then
+             entropyf = entropyconsp*vtp(flux_direction)
+          endif
+
+        else !!! The end of this branch is right at the bottom of the routine
+
+          usendh = uh(flux_direction,flux_direction)
+
+!!$        Calculate the jumps in the conserved variables
+
+          qdiff = cons_m - cons_p
+
+          if (clean_divergence.ne.0) then
+             psidcdiff = psidcm - psidcp
+          endif
+          if(evolve_entropy.ne.0) then
+             entropydiff = entropyconsm - entropyconsp
+          endif
+
+!!$       Eigenvalues and fluxes either side of the cell interface
+
+          if(evolve_temper.ne.1) then
+            call eigenvaluesM(GRHydro_eos_handle,&
+                 prim_m(1),prim_m(1+ix),prim_m(1+iy),prim_m(1+iz),prim_m(5),prim_m(6),prim_m(7), &
+                 prim_m(7+ix),prim_m(7+iy),prim_m(7+iz),&
+                 lamminus,gh(1,1),gh(1,2),gh(1,3),gh(2,2),gh(2,3),gh(3,3),&
+                 usendh,avg_alp,avg_beta(flux_direction))
+            call eigenvaluesM(GRHydro_eos_handle,&
+                 prim_p(1),prim_p(1+ix),prim_p(1+iy),prim_p(1+iz),prim_p(5),prim_p(6),prim_p(7), &
+                 prim_p(7+ix),prim_p(7+iy),prim_p(7+iz),&
+                 lamplus,gh(1,1),gh(1,2),gh(1,3),gh(2,2),gh(2,3),gh(3,3),&
+                 usendh,avg_alp,avg_beta(flux_direction))
+          else
+            xtemp = temperature(i,j,k)
+            call eigenvaluesM_hot(GRHydro_eos_handle,&
+                 int(i,ik),int(j,ik),int(k,ik),&
+                 prim_m(1),prim_m(1+ix),prim_m(1+iy),prim_m(1+iz),prim_m(5),prim_m(6),prim_m(7), &
+                 prim_m(7+ix),prim_m(7+iy),prim_m(7+iz),&
+                 xtemp,y_e_minus(i+xoffset,j+yoffset,k+zoffset),&
+                 lamminus,gh(1,1),gh(1,2),gh(1,3),gh(2,2),gh(2,3),gh(3,3),&
+                 usendh,avg_alp,avg_beta(flux_direction))
+            xtemp = temperature(i,j,k)
+            call eigenvaluesM_hot(GRHydro_eos_handle,&
+                 int(i,ik),int(j,ik),int(k,ik),&
+                 prim_p(1),prim_p(1+ix),prim_p(1+iy),prim_p(1+iz),prim_p(5),prim_p(6),prim_p(7), &
+                 prim_p(7+ix),prim_p(7+iy),prim_p(7+iz),&
+                 xtemp,y_e_plus(i,j,k),&
+                 lamplus,gh(1,1),gh(1,2),gh(1,3),gh(2,2),gh(2,3),gh(3,3),&
+                 usendh,avg_alp,avg_beta(flux_direction))
+          endif
+
+          call num_x_fluxM(cons_p(1),cons_p(1+ix),cons_p(1+iy),cons_p(1+iz),&
+               cons_p(5),&
+               cons_p(5+ix),cons_p(5+iy),cons_p(5+iz),&
+               fplus(1),fplus(1+ix),fplus(1+iy),fplus(1+iz),fplus(5),&
+               fplus(5+ix),fplus(5+iy),fplus(5+iz),&
+               vtp(ix),vtp(iy),vtp(iz),pressstarp,blowp(ix),blowp(iy),&
+               blowp(iz),ab0p,wp, &
+               avg_det,avg_alp,avg_beta(flux_direction))
+          call num_x_fluxM(cons_m(1),cons_m(1+ix),cons_m(1+iy),cons_m(1+iz),&
+               cons_m(5),&
+               cons_m(5+ix),cons_m(5+iy),cons_m(5+iz),&
+               fminus(1),fminus(1+ix),fminus(1+iy),fminus(1+iz),fminus(5),&
+               fminus(5+ix),fminus(5+iy),fminus(5+iz),&
+               vtm(ix),vtm(iy),vtm(iz),pressstarm,blowm(ix),blowm(iy),&
+               blowm(iz),ab0m,wm, &
+               avg_det,avg_alp,avg_beta(flux_direction))
+
+          if(clean_divergence.ne.0) then
+             fminus(6)=fminus(6) + 1.0d0*sdet*uh(flux_direction,1)*psidcm - cons_m(5+flux_direction)*avg_beta(1)/avg_alp
+             fminus(7)=fminus(7) + 1.0d0*sdet*uh(flux_direction,2)*psidcm - cons_m(5+flux_direction)*avg_beta(2)/avg_alp
+             fminus(8)=fminus(8) + 1.0d0*sdet*uh(flux_direction,3)*psidcm - cons_m(5+flux_direction)*avg_beta(3)/avg_alp
+             fplus(6)=fplus(6) + 1.0d0*sdet*uh(flux_direction,1)*psidcp - cons_p(5+flux_direction)*avg_beta(1)/avg_alp
+             fplus(7)=fplus(7) + 1.0d0*sdet*uh(flux_direction,2)*psidcp - cons_p(5+flux_direction)*avg_beta(2)/avg_alp
+             fplus(8)=fplus(8) + 1.0d0*sdet*uh(flux_direction,3)*psidcp - cons_p(5+flux_direction)*avg_beta(3)/avg_alp
+             psidcfp = cons_p(5+flux_direction)/sdet-avg_beta(flux_direction)*psidcp/avg_alp
+             psidcfm = cons_m(5+flux_direction)/sdet-avg_beta(flux_direction)*psidcm/avg_alp
+          endif
+          if(evolve_entropy.ne.0) then
+             entropyfp = entropyconsp*vtp(flux_direction)
+             entropyfm = entropyconsm*vtm(flux_direction)
+          endif
+
+!!$        Find minimum and maximum wavespeeds
+
+          charmin = min(0.d0, lamplus(1), lamplus(2), lamplus(3), &
+               lamplus(4),lamplus(5),  lamminus(1),lamminus(2),lamminus(3),&
+               lamminus(4),lamminus(5))
+
+          charmax = max(0.d0, lamplus(1), lamplus(2), lamplus(3), &
+               lamplus(4),lamplus(5),  lamminus(1),lamminus(2),lamminus(3),&
+               lamminus(4),lamminus(5))
+
+          chartop = max(-charmin,charmax)
+
+          charpm = charmax - charmin
+
+!!$        Calculate flux by standard formula
+
+          do m = 1,8
+
+             qdiff(m) = cons_m(m) - cons_p(m)
+
+            if (HLLE) then
+              f1(m) = (charmax * fplus(m) - charmin * fminus(m) + &
+                       charmax * charmin * qdiff(m)) / charpm
+            else if (LLF) then
+              f1(m) = 0.5d0 * (fplus(m) + fminus(m) - chartop * qdiff(m))
+            end if
+
+          end do
+
+          if(clean_divergence.ne.0) then
+
+           psidcdiff = psidcm - psidcp
+           select case(whichpsidcspeed)
+             case(0)
+               if (HLLE) then
+                 psidcf = (charmax * psidcfp - charmin * psidcfm + &
+                           charmax * charmin * psidcdiff) / charpm
+               else if (LLF) then
+                 psidcf = 0.5d0 * (psidcfp + psidcfm - chartop * psidcdiff)
+               end if
+             case(1)
+             !!$ Wavespeeds for psidc are +/-c, not Fast Magnetosonic?
+             !!$ psidcf = 0.5d0 * (1.d0 * psidcfp - (-1.d0) * psidcfm + &
+             !!$      1.d0 * (-1.d0) * psidcdiff)
+
+             !!$ The fastest speed for psidc comes from the condition
+             !!$ that the normal vector to the characteristic hypersurface
+             !!$ be spacelike (Eq. 60 of Anton et al.)
+
+             charmax_dc = sqrt(usendh) - avg_beta(flux_direction)/avg_alp
+             charmin_dc = -1.d0*sqrt(usendh) - avg_beta(flux_direction)/avg_alp
+             charpm_dc = charmax_dc - charmin_dc
+
+             psidcf = (charmax_dc * psidcfp - charmin_dc * psidcfm + &
+                       charmax_dc * charmin_dc * psidcdiff) / charpm_dc
+
+             if(decouple_normal_Bfield .ne. 0) then ! same expression for HLLE and LLF
+               !!$ B^i field decouples from the others and has same propagation
+               !!$ speed as divergence -null direction,
+               !!$ \pm sqrt(g^{xx}} - beta^x/alpha
+               f1(5+flux_direction) = (charmax_dc * fplus(5+flux_direction) &
+                                     - charmin_dc * fminus(5+flux_direction) + &
+                  charmax_dc * charmin_dc * qdiff(5+flux_direction)) / charpm_dc
+             end if
+             case(2)
+             charmax = setcharmax
+             charmin = setcharmin
+               if (HLLE) then
+                 psidcf = (charmax * psidcfp - charmin * psidcfm + &
+                           charmax * charmin * psidcdiff) / charpm
+               else if (LLF) then
+                 chartop = max(-charmin,charmax)
+                 psidcf = 0.5d0 * (psidcfp + psidcfm - chartop * psidcdiff)
+               end if
+           end select
+          endif
+
+          if(evolve_entropy.ne.0) then
+            entropydiff = entropyconsm - entropyconsp
+            if (HLLE) then
+              entropyf = (charmax * entropyfp - charmin * entropyfm + &
+                       charmax * charmin * entropydiff) / charpm
+            else if (LLF) then
+              entropyf = 0.5d0 * (entropyfp + entropyfm - chartop * entropydiff)
+            end if
+          endif
+
+
+
+        end if !!! The end of the SpaceMask check for a trivial RP.
+
+        densflux(i, j, k) = f1(1)
+        sxflux(i, j, k) = f1(2)
+        syflux(i, j, k) = f1(3)
+        szflux(i, j, k) = f1(4)
+        tauflux(i, j, k) = f1(5)
+
+        Bconsxflux(i, j, k) = f1(6)
+        Bconsyflux(i, j, k) = f1(7)
+        Bconszflux(i, j, k) = f1(8)
+
+        if(clean_divergence.ne.0) then
+           psidcflux(i,j,k) = psidcf
+        endif
+
+        if(evolve_entropy.ne.0) then
+           entropyflux(i,j,k) = entropyf
+        endif
+
+        if(evolve_Y_e.ne.0) then
+           if (densflux(i, j, k) > 0.d0) then
+              Y_e_con_flux(i, j, k) = &
+                   Y_e_plus(i, j, k) * &
+                   densflux(i, j, k)
+            else
+               Y_e_con_flux(i, j, k) = &
+                    Y_e_minus(i + xoffset, j + yoffset, k + zoffset) * &
+                    densflux(i, j, k)
+            endif
+         endif
+
+      end do
+    end do
+  end do
+  !$OMP END DO
+  !$OMP END PARALLEL
+#undef faulty_gxx
+#undef faulty_gxy
+#undef faulty_gxz
+#undef faulty_gyy
+#undef faulty_gyz
+#undef faulty_gzz
+#undef faulty_betax
+#undef faulty_betay
+#undef faulty_betaz
+#undef faulty_vel
+#undef faulty_Bvec
+
+end subroutine GRHydro_HLLEM
+
+ /*@@
+   @routine    GRHydro_HLLE_TracerM
+   @date       Aug 30, 2010
+   @author     Joshua Faber, Scott Noble, Bruno Mundim, Ian Hawke
+   @desc
+   HLLE just for the tracer.
+   @enddesc
+   @calls
+   @calledby
+   @history
+
+   @endhistory
+
+@@*/
+
+subroutine GRHydro_HLLE_TracerM(CCTK_ARGUMENTS)
+
+  USE GRHydro_EigenproblemM
+
+  implicit none
+
+  DECLARE_CCTK_ARGUMENTS
+  DECLARE_CCTK_PARAMETERS
+  DECLARE_CCTK_FUNCTIONS
+
+  integer :: i, j, k,  m
+  CCTK_REAL, dimension(number_of_tracers) :: cons_p,cons_m,fplus,fminus,f1
+  CCTK_REAL, dimension(5) :: lamminus,lamplus
+  CCTK_REAL, dimension(number_of_tracers) :: qdiff
+  CCTK_REAL, dimension(7) :: prim_p, prim_m
+  CCTK_REAL, dimension(3) :: mag_p, mag_m
+  CCTK_REAL ::  charmin, charmax, charpm,avg_alp,avg_det
+  CCTK_REAL :: gxxh, gxyh, gxzh, gyyh, gyzh, gzzh, uxxh, uxyh, &
+       uxzh, uyyh, uyzh, uzzh, avg_beta, usendh
+  CCTK_REAL :: b2p,b2m
+
+  CCTK_INT :: type_bits, trivial
+
+  ! save memory when MP is not used
+  CCTK_INT :: GRHydro_UseGeneralCoordinates
+  CCTK_REAL, DIMENSION(cctk_ash1,cctk_ash2,cctk_ash3) :: g11, g12, g13, g22, g23, g33
+  pointer (pg11,g11), (pg12,g12), (pg13,g13), (pg22,g22), (pg23,g23), (pg33,g33)
+  CCTK_REAL, DIMENSION(cctk_ash1,cctk_ash2,cctk_ash3) :: beta1, beta2, beta3
+  pointer (pbeta1,beta1), (pbeta2,beta2), (pbeta3,beta3)
+
+  if (GRHydro_UseGeneralCoordinates(cctkGH).ne.0) then
+    pg11 = loc(gaa)
+    pg12 = loc(gab)
+    pg13 = loc(gac)
+    pg22 = loc(gbb)
+    pg23 = loc(gbc)
+    pg33 = loc(gcc)
+    pbeta1 = loc(betaa)
+    pbeta2 = loc(betab)
+    pbeta3 = loc(betac)
+  else
+    pg11 = loc(gxx)
+    pg12 = loc(gxy)
+    pg13 = loc(gxz)
+    pg22 = loc(gyy)
+    pg23 = loc(gyz)
+    pg33 = loc(gzz)
+    pbeta1 = loc(betax)
+    pbeta2 = loc(betay)
+    pbeta3 = loc(betaz)
+  end if
+#define gxx faulty_gxx
+#define gxy faulty_gxy
+#define gxz faulty_gxz
+#define gyy faulty_gyy
+#define gyz faulty_gyz
+#define gzz faulty_gzz
+#define betax faulty_betax
+#define betay faulty_betay
+#define betaz faulty_betaz
+#define vel faulty_vel
+#define Bvec faulty_Bvec
+
+  if (flux_direction == 1) then
+    call SpaceMask_GetTypeBits(type_bits, "Hydro_RiemannProblemX")
+    call SpaceMask_GetStateBits(trivial, "Hydro_RiemannProblemX", &
+         &"trivial")
+  else if (flux_direction == 2) then
+    call SpaceMask_GetTypeBits(type_bits, "Hydro_RiemannProblemY")
+    call SpaceMask_GetStateBits(trivial, "Hydro_RiemannProblemY", &
+         &"trivial")
+  else if (flux_direction == 3) then
+    call SpaceMask_GetTypeBits(type_bits, "Hydro_RiemannProblemZ")
+    call SpaceMask_GetStateBits(trivial, "Hydro_RiemannProblemZ", &
+         &"trivial")
+  else
+    call CCTK_ERROR("Flux direction not x,y,z")
+    STOP
+  end if
+
+  do k = GRHydro_stencil, cctk_lsh(3) - GRHydro_stencil
+    do j = GRHydro_stencil, cctk_lsh(2) - GRHydro_stencil
+      do i = GRHydro_stencil, cctk_lsh(1) - GRHydro_stencil
+
+        f1 = 0.d0
+        lamminus = 0.d0
+        lamplus = 0.d0
+        cons_p = 0.d0
+        cons_m = 0.d0
+        mag_p = 0.d0
+        mag_m = 0.d0
+        fplus = 0.d0
+        fminus = 0.d0
+        qdiff = 0.d0
+
+!!$        Set the left (p for plus) and right (m_i for minus, i+1) states
+
+        cons_p(:) = cons_tracerplus(i,j,k,:)
+        cons_m(:) = cons_tracerminus(i+xoffset,j+yoffset,k+zoffset,:)
+
+        mag_p(1) = Bvecxplus(i,j,k)
+        mag_p(2) = Bvecyplus(i,j,k)
+        mag_p(3) = Bveczplus(i,j,k)
+
+        mag_m(1) = Bvecxminus(i+xoffset,j+yoffset,k+zoffset)
+        mag_m(2) = Bvecyminus(i+xoffset,j+yoffset,k+zoffset)
+        mag_m(3) = Bveczminus(i+xoffset,j+yoffset,k+zoffset)
+
+        prim_p(1)   = rhoplus(i,j,k)
+        prim_p(2)   = velxplus(i,j,k)
+        prim_p(3)   = velyplus(i,j,k)
+        prim_p(4)   = velzplus(i,j,k)
+        prim_p(5)   = epsplus(i,j,k)
+        prim_p(6)   = pressplus(i,j,k)
+        prim_p(7)   = w_lorentzplus(i,j,k)
+
+        prim_m(1) = rhominus(i+xoffset,j+yoffset,k+zoffset)
+        prim_m(2) = velxminus(i+xoffset,j+yoffset,k+zoffset)
+        prim_m(3) = velyminus(i+xoffset,j+yoffset,k+zoffset)
+        prim_m(4) = velzminus(i+xoffset,j+yoffset,k+zoffset)
+        prim_m(5) = epsminus(i+xoffset,j+yoffset,k+zoffset)
+        prim_m(6) = pressminus(i+xoffset,j+yoffset,k+zoffset)
+        prim_m(7) = w_lorentzminus(i+xoffset,j+yoffset,k+zoffset)
+
+!!$        Calculate various metric terms here.
+!!$        Note also need the average of the lapse at the
+!!$        left and right points.
+
+        if (flux_direction == 1) then
+           avg_beta = 0.5d0 * (beta1(i+xoffset,j+yoffset,k+zoffset) + &
+               beta1(i,j,k))
+        else if (flux_direction == 2) then
+           avg_beta = 0.5d0 * (beta2(i+xoffset,j+yoffset,k+zoffset) + &
+               beta2(i,j,k))
+        else if (flux_direction == 3) then
+           avg_beta = 0.5d0 * (beta3(i+xoffset,j+yoffset,k+zoffset) + &
+               beta3(i,j,k))
+        else
+           call CCTK_ERROR("Flux direction not x,y,z")
+           STOP
+        end if
+
+        avg_alp = 0.5 * (alp(i,j,k) + alp(i+xoffset,j+yoffset,k+zoffset))
+
+        gxxh = 0.5d0 * (g11(i+xoffset,j+yoffset,k+zoffset) + g11(i,j,k))
+        gxyh = 0.5d0 * (g12(i+xoffset,j+yoffset,k+zoffset) + g12(i,j,k))
+        gxzh = 0.5d0 * (g13(i+xoffset,j+yoffset,k+zoffset) + g13(i,j,k))
+        gyyh = 0.5d0 * (g22(i+xoffset,j+yoffset,k+zoffset) + g22(i,j,k))
+        gyzh = 0.5d0 * (g23(i+xoffset,j+yoffset,k+zoffset) + g23(i,j,k))
+        gzzh = 0.5d0 * (g33(i+xoffset,j+yoffset,k+zoffset) + g33(i,j,k))
+
+        avg_det =  SPATIAL_DETERMINANT(gxxh,gxyh,gxzh,gyyh,gyzh,gzzh)
+
+        call UpperMetric(uxxh, uxyh, uxzh, uyyh, uyzh, uzzh, &
+             avg_det,gxxh, gxyh, gxzh, &
+             gyyh, gyzh, gzzh)
+
+        if (flux_direction == 1) then
+          usendh = uxxh
+        else if (flux_direction == 2) then
+          usendh = uyyh
+        else if (flux_direction == 3) then
+          usendh = uzzh
+        else
+          call CCTK_ERROR("Flux direction not x,y,z")
+          STOP
+        end if
+
+!!$        b^2 = (1-v^2)B^2+(B dot v)^2
+        b2p=DOTP2(gxxh,gxyh,gxzh,gyyh,gyzh,gzzh,mag_p(1),mag_p(2),mag_p(3))/prim_p(7)**2 + &
+             (DOTP(gxxh,gxyh,gxzh,gyyh,gyzh,gzzh,prim_p(2),prim_p(3),prim_p(4),mag_p(1),mag_p(2),mag_p(3)))**2
+        b2m=DOTP2(gxxh,gxyh,gxzh,gyyh,gyzh,gzzh,mag_m(1),mag_m(2),mag_m(3))/prim_m(7)**2 + &
+             (DOTP(gxxh,gxyh,gxzh,gyyh,gyzh,gzzh,prim_m(2),prim_m(3),prim_m(4),mag_m(1),mag_m(2),mag_m(3)))**2
+
+!!$        Calculate the jumps in the conserved variables
+
+        qdiff = cons_m - cons_p
+
+!!$        Eigenvalues and fluxes either side of the cell interface
+
+        if (flux_direction == 1) then
+           call eigenvaluesM(GRHydro_eos_handle,&
+                prim_m(1),prim_m(2),prim_m(3),prim_m(4),prim_m(5),prim_m(6),prim_m(7), &
+                mag_m(1),mag_m(2),mag_m(3),&
+                lamminus,gxxh,gxyh,gxzh,gyyh,gyzh,gzzh,&
+                usendh,avg_alp,avg_beta)
+           call eigenvaluesM(GRHydro_eos_handle, &
+                prim_p(1),prim_p(2),prim_p(3),prim_p(4),prim_p(5),prim_p(6),prim_p(7), &
+                mag_p(1),mag_p(2),mag_p(3),&
+                lamplus,gxxh,gxyh,gxzh,gyyh,gyzh,gzzh,&
+                usendh,avg_alp,avg_beta)
+           fplus(:)  = (velxplus(i,j,k)  - avg_beta / avg_alp) * &
+                cons_tracerplus(i,j,k,:)
+           fminus(:) = (velxminus(i+xoffset,j+yoffset,k+zoffset) - avg_beta / avg_alp) * &
+                cons_tracerminus(i+xoffset,j+yoffset,k+zoffset,:)
+        else if (flux_direction == 2) then
+           call eigenvaluesM(GRHydro_eos_handle,&
+                prim_m(1),prim_m(3),prim_m(4),prim_m(2),prim_m(5),prim_m(6),prim_m(7), &
+                mag_m(2),mag_m(3),mag_m(1),&
+                lamminus,gyyh,gyzh,gxyh,gzzh,gxzh,gxxh,&
+                usendh,avg_alp,avg_beta)
+           call eigenvaluesM(GRHydro_eos_handle, &
+                prim_p(1),prim_p(3),prim_p(4),prim_p(2),prim_p(5),prim_p(6),prim_p(7), &
+                mag_p(2),mag_p(3),mag_p(1),&
+                lamplus,gyyh,gyzh,gxyh,gzzh,gxzh,gxxh,&
+                usendh,avg_alp,avg_beta)
+           fplus(:)  = (velyplus(i,j,k)  - avg_beta / avg_alp) * &
+                cons_tracerplus(i,j,k,:)
+           fminus(:) = (velyminus(i+xoffset,j+yoffset,k+zoffset) - avg_beta / avg_alp) * &
+                cons_tracerminus(i+xoffset,j+yoffset,k+zoffset,:)
+        else if (flux_direction == 3) then
+           call eigenvaluesM(GRHydro_eos_handle,&
+                prim_m(1),prim_m(4),prim_m(2),prim_m(3),prim_m(5),prim_m(6),prim_m(7), &
+                mag_m(3),mag_m(1),mag_m(2),&
+                lamminus,gzzh,gxzh,gyzh,gxxh,gxyh,gyyh,&
+                usendh,avg_alp,avg_beta)
+           call eigenvaluesM(GRHydro_eos_handle,&
+                prim_p(1),prim_p(4),prim_p(2),prim_p(3),prim_p(5),prim_p(6),prim_p(7), &
+                mag_p(3),mag_p(1),mag_p(2),&
+                lamplus,gzzh,gxzh,gyzh,gxxh,gxyh,gyyh,&
+                usendh,avg_alp,avg_beta)
+           fplus(:)  = (velzplus(i,j,k) - avg_beta / avg_alp) * &
+                cons_tracerplus(i,j,k,:)
+           fminus(:) = (velzminus(i+xoffset,j+yoffset,k+zoffset) - avg_beta / avg_alp) * &
+                cons_tracerminus(i+xoffset,j+yoffset,k+zoffset,:)
+        else
+           call CCTK_ERROR("Flux direction not x,y,z")
+           STOP
+        end if
+
+!!$        Find minimum and maximum wavespeeds
+
+        charmin = min(0.d0, lamplus(1), lamplus(2), lamplus(3), &
+             lamplus(4),lamplus(5),  lamminus(1),lamminus(2),lamminus(3),&
+             lamminus(4),lamminus(5))
+
+        charmax = max(0.d0, lamplus(1), lamplus(2), lamplus(3), &
+             lamplus(4),lamplus(5),  lamminus(1),lamminus(2),lamminus(3),&
+             lamminus(4),lamminus(5))
+
+        charpm = charmax - charmin
+
+!!$        Calculate flux by standard formula
+
+        do m = 1,number_of_tracers
+
+           qdiff(m) = cons_m(m) - cons_p(m)
+
+           f1(m) = (charmax * fplus(m) - charmin * fminus(m) + &
+                charmax * charmin * qdiff(m)) / charpm
+
+        end do
+
+        cons_tracerflux(i, j, k,:) = f1(:)
+!!$
+!!$        if ( ((flux_direction.eq.3).and.(i.eq.4).and.(j.eq.4)).or.&
+!!$             ((flux_direction.eq.2).and.(i.eq.4).and.(k.eq.4)).or.&
+!!$             ((flux_direction.eq.1).and.(j.eq.4).and.(k.eq.4))&
+!!$             ) then
+!!$          write(*,*) flux_direction, i, j, k, f1(1), cons_m(1), cons_p(1)
+!!$        end if
+
+     end do
+  end do
+end do
+#undef faulty_gxx
+#undef faulty_gxy
+#undef faulty_gxz
+#undef faulty_gyy
+#undef faulty_gyz
+#undef faulty_gzz
+#undef faulty_betax
+#undef faulty_betay
+#undef faulty_betaz
+#undef faulty_vel
+#undef faulty_Bvec
+
+
+end subroutine GRHydro_HLLE_TracerM
+


Add missing GRHydro_Tmunu.F90 in EinsteinEvolve/GRHydro (einsteintoolkit/Cactus#15)

This PR adds the missing source file GRHydro_Tmunu.F90 to complete the EinsteinEvolve/GRHydro thorn implementation.

Implement the full functionality of GRHydro

Please implement the full functionality of `GRHydro` by finishing the code in `GRHydro_Tmunu.F90`.

## Thorn Information:
- Name: GRHydro
- Target file: GRHydro_Tmunu.F90

## Interface Definition in interface.ccl:

## Schedule Definition in schedule.ccl:

## Parameters Definition in param.ccl:

## Configuration Definition in configuration.ccl:

## Documentation Context in readme.md and doc/documentation.tex

## Related Code Context in src folder

diff --git a/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_Tmunu.F90 b/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_Tmunu.F90
new file mode 100644
index 0000000..0000000
--- /dev/null
+++ b/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_Tmunu.F90
@@ -0,0 +1,159 @@
+ /*@@
+   @file      GRHydro_Tmunu.F90
+   @date      Thu Apr 16 19:38:40 2009
+   @author    Ian Hawke
+   @histpry
+    Apr. 2009: Luca Baiotti copied and adapted for the Tmunu-thorn mechanism the original include file
+   @desc
+     The calculation of the stress energy tensor.
+     The version used here was worked out by Miguel Alcubierre. I
+     think it was an extension of the routine from GR3D, written
+     by Mark Miller.
+     C version added by Ian Hawke.
+
+     Lower components of the stress-energy tensor obtained from
+     the hydro variables.  The components are given by:
+
+     T      =  rho h  u   u    +  P  g
+      mu nu            mu  nu         mu nu
+
+     where rho is the energy density of the fluid, h the enthalpy
+     and P the pressure.  The enthalpy is given in terms of the
+     basic variables as:
+
+     h  =  1  +  e  +  P/rho
+
+     with e the internal energy (eps here).
+
+     In the expresion for T_{mu,nu} we also have the four-velocity
+     of the fluid given by (v_i is the 3-velocity field):
+
+                                 i
+     u  =  W ( - alpha +  v  beta  )
+      0                    i
+
+     u  =  W v
+      i       i
+                                                i  -1/2
+     with W the Lorentz factor:   W = ( 1 -  v v  )
+                                              i
+
+     and where alpha and beta are the lapse and shift vector.
+
+     Finally, the 4 metric is given by
+
+                   2             i
+     g   =  - alpha  + beta  beta
+      00                   i
+
+     g   =  beta
+      0i        i
+
+
+     g   =  gamma      (the spatial metric)
+      ij        ij
+
+
+   @enddesc
+ @@*/
+#include "cctk.h"
+#include "cctk_Arguments.h"
+#include "cctk_Parameters.h"
+#include "cctk_Functions.h"
+#include "SpaceMask.h"
+
+#define velx(i,j,k) vel(i,j,k,1)
+#define vely(i,j,k) vel(i,j,k,2)
+#define velz(i,j,k) vel(i,j,k,3)
+
+ subroutine GRHydro_Tmunu(CCTK_ARGUMENTS)
+
+   implicit none
+
+   DECLARE_CCTK_ARGUMENTS
+   DECLARE_CCTK_PARAMETERS
+   DECLARE_CCTK_FUNCTIONS
+
+   CCTK_REAL velxlow, velylow, velzlow
+   CCTK_REAL betaxlow, betaylow, betazlow, beta2
+   CCTK_REAL utlow, uxlow, uylow, uzlow
+   CCTK_REAL rhoenthalpy
+   CCTK_REAL ut,ux,uy,uz,bst,bsx,bsy,bsz,bs2
+   CCTK_REAL dampfac
+   CCTK_INT i,j,k
+
+!!$ Damping factor
+   dampfac = 1.0
+
+   !$OMP PARALLEL DO PRIVATE(i,j,k,velxlow, velylow, velzlow,&
+   !$OMP betaxlow, betaylow, betazlow, beta2, utlow, uxlow, uylow, uzlow,&
+   !$OMP rhoenthalpy, ut,ux,uy,uz,bst,bsx,bsy,bsz,bs2,dampfac)
+   do k = 1, cctk_lsh(3)
+     do j = 1, cctk_lsh(2)
+       do i = 1, cctk_lsh(1)
+
+          velxlow = gxx(i,j,k)*velx(i,j,k) + gxy(i,j,k)*vely(i,j,k) + gxz(i,j,k)*velz(i,j,k)
+          velylow = gxy(i,j,k)*velx(i,j,k) + gyy(i,j,k)*vely(i,j,k) + gyz(i,j,k)*velz(i,j,k)
+          velzlow = gxz(i,j,k)*velx(i,j,k) + gyz(i,j,k)*vely(i,j,k) + gzz(i,j,k)*velz(i,j,k)
+
+!!$       Calculate lower components and square of shift vector.
+
+          betaxlow = gxx(i,j,k)*betax(i,j,k) + gxy(i,j,k)*betay(i,j,k) + gxz(i,j,k)*betaz(i,j,k)
+          betaylow = gxy(i,j,k)*betax(i,j,k) + gyy(i,j,k)*betay(i,j,k) + gyz(i,j,k)*betaz(i,j,k)
+          betazlow = gxz(i,j,k)*betax(i,j,k) + gyz(i,j,k)*betay(i,j,k) + gzz(i,j,k)*betaz(i,j,k)
+
+          beta2 = betax(i,j,k)*betaxlow + betay(i,j,k)*betaylow + betaz(i,j,k)*betazlow
+
+!!$       Calculate the specific relativistic enthalpy times rho times the
+!!$       square of the lorentz factor.
+
+         rhoenthalpy = w_lorentz(i,j,k)**2*(rho(i,j,k)*(1.0d0 + eps(i,j,k)) + press(i,j,k))
+
+!!$       Calculate lower components of 4-velocity (without the Lorent factor).
+
+         utlow = (-alp(i,j,k) + velx(i,j,k)*betaxlow + vely(i,j,k)*betaylow + velz(i,j,k)*betazlow)
+
+         uxlow = velxlow
+         uylow = velylow
+         uzlow = velzlow
+
+
+!!$      Initialize damping factor
+         dampfac = 1.0
+!!$      Apply tanh blending for Tmunu.
+         if ((Tmunu_damping_radius_min .gt. 0) .and. (r(i,j,k) .gt. Tmunu_damping_radius_min)) then
+            !  0.5 * (1.0 - tanh(4.0*(x-x0)/sigma0))
+            if (r(i,j,k) .lt. Tmunu_damping_radius_max) then
+               dampfac = 0.5d0 * (1.0d0 - tanh((8.0d0*r(i,j,k)-4.0d0*(Tmunu_damping_radius_max+Tmunu_damping_radius_min))/(Tmunu_damping_radius_max-Tmunu_damping_radius_min)))
+            else
+               dampfac = 0.0
+               continue   ! no need to add anything to Tmunu at the current point (it's zero anyway!)
+            endif
+         else
+            dampfac = 1.0
+         endif
+
+!!$       Calculate Tmunu (the lower components!).
+
+         eTtt(i,j,k) = eTtt(i,j,k) + dampfac * (rhoenthalpy*utlow**2 + press(i,j,k)*(beta2 - alp(i,j,k)**2))
+
+         eTtx(i,j,k) = eTtx(i,j,k) + dampfac * (rhoenthalpy*utlow*uxlow + press(i,j,k)*betaxlow)
+         eTty(i,j,k) = eTty(i,j,k) + dampfac * (rhoenthalpy*utlow*uylow + press(i,j,k)*betaylow)
+         eTtz(i,j,k) = eTtz(i,j,k) + dampfac * (rhoenthalpy*utlow*uzlow + press(i,j,k)*betazlow)
+
+         eTxx(i,j,k) = eTxx(i,j,k) + dampfac * (rhoenthalpy*uxlow**2 + press(i,j,k)*gxx(i,j,k))
+         eTyy(i,j,k) = eTyy(i,j,k) + dampfac * (rhoenthalpy*uylow**2 + press(i,j,k)*gyy(i,j,k))
+         eTzz(i,j,k) = eTzz(i,j,k) + dampfac * (rhoenthalpy*uzlow**2 + press(i,j,k)*gzz(i,j,k))
+
+         eTxy(i,j,k) = eTxy(i,j,k) + dampfac * (rhoenthalpy*uxlow*uylow + press(i,j,k)*gxy(i,j,k))
+         eTxz(i,j,k) = eTxz(i,j,k) + dampfac * (rhoenthalpy*uxlow*uzlow + press(i,j,k)*gxz(i,j,k))
+         eTyz(i,j,k) = eTyz(i,j,k) + dampfac * (rhoenthalpy*uylow*uzlow + press(i,j,k)*gyz(i,j,k))
+
+       end do
+     end do
+   end do
+   !$OMP END PARALLEL DO
+
+   return
+
+ end subroutine GRHydro_Tmunu