Add missing GRHydro_TmunuM.F90 in EinsteinEvolve/GRHydro (einsteintoolkit/Cactus#16)

This PR adds the missing source file GRHydro_TmunuM.F90 to complete the EinsteinEvolve/GRHydro thorn implementation.

Implement the full functionality of GRHydro

Please implement the full functionality of `GRHydro` by finishing the code in `GRHydro_TmunuM.F90`.

## Thorn Information:
- Name: GRHydro
- Target file: GRHydro_TmunuM.F90

## Interface Definition in interface.ccl:

## Schedule Definition in schedule.ccl:

## Parameters Definition in param.ccl:

## Configuration Definition in configuration.ccl:

## Documentation Context in readme.md and doc/documentation.tex

## Related Code Context in src folder

diff --git a/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_TmunuM.F90 b/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_TmunuM.F90
new file mode 100644
index 0000000..0000000
--- /dev/null
+++ b/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_TmunuM.F90
@@ -0,0 +1,167 @@
+ /*@@
+   @file      GRHydro_Tmunu.F90
+   @date      Aug 30, 2010
+   @author    Joshua Faber, Scott Noble, Bruno Mundim, Ian Hawke
+   @histpry
+    Apr. 2009: Luca Baiotti copied and adapted for the Tmunu-thorn mechanism the original include file
+   @desc
+     The calculation of the stress energy tensor.
+     The version used here was worked out by Miguel Alcubierre. I
+     think it was an extension of the routine from GR3D, written
+     by Mark Miller.
+     C version added by Ian Hawke.
+
+     Lower components of the stress-energy tensor obtained from
+     the hydro variables.  The components are given by:
+
+     T      =  (rho h +b^2) u   u    +  (P+b^2/2)  g      - b   b
+      mu nu                   mu  nu                mu nu     mu nu
+
+     where rho is the energy density of the fluid, h the enthalpy
+     and P the pressure.  The enthalpy is given in terms of the
+     basic variables as:
+
+     h  =  1  +  e  +  P/rho
+
+     with e the internal energy (eps here).
+
+     In the expresion for T_{mu,nu} we also have the four-velocity
+     of the fluid given by (v_i is the 3-velocity field):
+
+                                 i
+     u  =  W ( - alpha +  v  beta  )
+      0                    i
+
+     u  =  W v
+      i       i
+                                                i  -1/2
+     with W the Lorentz factor:   W = ( 1 -  v v  )
+                                              i
+
+     and where alpha and beta are the lapse and shift vector.
+
+     Finally, the 4 metric is given by
+
+                   2             i
+     g   =  - alpha  + beta  beta
+      00                   i
+
+     g   =  beta
+      0i        i
+
+
+     g   =  gamma      (the spatial metric)
+      ij        ij
+
+
+   @enddesc
+ @@*/
+#include "cctk.h"
+#include "cctk_Arguments.h"
+#include "cctk_Parameters.h"
+#include "cctk_Functions.h"
+#include "SpaceMask.h"
+
+#define velx(i,j,k) vel(i,j,k,1)
+#define vely(i,j,k) vel(i,j,k,2)
+#define velz(i,j,k) vel(i,j,k,3)
+#define Bvecx(i,j,k) Bvec(i,j,k,1)
+#define Bvecy(i,j,k) Bvec(i,j,k,2)
+#define Bvecz(i,j,k) Bvec(i,j,k,3)
+#define bcomx(i,j,k) bcom(i,j,k,1)
+#define bcomy(i,j,k) bcom(i,j,k,2)
+#define bcomz(i,j,k) bcom(i,j,k,3)
+
+ subroutine GRHydro_TmunuM(CCTK_ARGUMENTS)
+
+   implicit none
+
+   DECLARE_CCTK_ARGUMENTS
+   DECLARE_CCTK_PARAMETERS
+   DECLARE_CCTK_FUNCTIONS
+
+   CCTK_REAL :: velxlow, velylow, velzlow
+   CCTK_REAL :: betaxlow, betaylow, betazlow, beta2
+   CCTK_REAL :: Bvecxlow,Bvecylow,Bveczlow
+   CCTK_REAL :: bdotv,b2,bxlow,bylow,bzlow,btlow,dum1,dum2
+   CCTK_REAL :: utlow,rhohstarw2,pstar
+   CCTK_REAL :: bdotbeta,vdotbeta
+   CCTK_INT :: i,j,k
+
+
+   !$OMP PARALLEL DO PRIVATE(i,j,k,velxlow, velylow, velzlow,&
+   !$OMP Bvecxlow,Bvecylow,Bveczlow, bdotv,dum1,dum2,b2,bxlow,bylow,bzlow,&
+   !$OMP betaxlow, betaylow, betazlow, beta2, bdotbeta,vdotbeta,utlow, btlow,&
+   !$OMP rhohstarw2,pstar)
+
+   do k = 1, cctk_lsh(3)
+     do j = 1, cctk_lsh(2)
+       do i = 1, cctk_lsh(1)
+
+          ! need separate dum1, dum2 b/c of Fortrans aliasing rules
+          call calc_vlow_blow(gxx(i,j,k),gxy(i,j,k),gxz(i,j,k),&
+               gyy(i,j,k),gyz(i,j,k),gzz(i,j,k), &
+               velx(i,j,k),vely(i,j,k),velz(i,j,k),Bvecx(i,j,k),Bvecy(i,j,k),Bvecz(i,j,k), &
+               velxlow,velylow,velzlow,Bvecxlow,Bvecylow,Bveczlow, &
+               bdotv,b2,dum1,dum2,bxlow,bylow,bzlow)
+
+!!$       Calculate lower components and square of shift vector.
+
+
+          betaxlow = gxx(i,j,k)*betax(i,j,k) + gxy(i,j,k)*betay(i,j,k) + gxz(i,j,k)*betaz(i,j,k)
+          betaylow = gxy(i,j,k)*betax(i,j,k) + gyy(i,j,k)*betay(i,j,k) + gyz(i,j,k)*betaz(i,j,k)
+          betazlow = gxz(i,j,k)*betax(i,j,k) + gyz(i,j,k)*betay(i,j,k) + gzz(i,j,k)*betaz(i,j,k)
+          beta2 = betax(i,j,k)*betaxlow + betay(i,j,k)*betaylow + betaz(i,j,k)*betazlow
+
+          bdotbeta = betaxlow*Bvecx(i,j,k)+betaylow*Bvecy(i,j,k)+betazlow*Bvecz(i,j,k)
+          vdotbeta = betaxlow*velx(i,j,k)+betaylow*vely(i,j,k)+betazlow*velz(i,j,k)
+
+!!$   u0 low is missing the w_lorentz factor (see below)!!
+          utlow = -1.d0*alp(i,j,k) + vdotbeta
+
+          btlow = -1.0d0*w_lorentz(i,j,k)*alp(i,j,k)*bdotv + &
+               bdotbeta/w_lorentz(i,j,k) + w_lorentz(i,j,k)*bdotv*vdotbeta
+
+!!$       Calculate the specific relativistic enthalpy times rho + the mag. field contribution times the
+!!$       square of the lorentz factor.
+
+         rhohstarw2 = w_lorentz(i,j,k)**2*(rho(i,j,k)*(1.0d0 + eps(i,j,k)) + press(i,j,k) + b2)
+         pstar =  press(i,j,k)+0.5d0*b2
+
+!!$       Calculate lower components of 4-velocity (without the Lorent factor).
+!!$         uxlow = velxlow
+!!$         uylow = velylow
+!!$         uzlow = velzlow
+
+!!$       Calculate Tmunu (the lower components!).
+
+         eTtt(i,j,k) = eTtt(i,j,k) + rhohstarw2*utlow**2 + pstar*(beta2 - alp(i,j,k)**2) - btlow**2
+
+         eTtx(i,j,k) = eTtx(i,j,k) + rhohstarw2*utlow*velxlow + pstar*betaxlow - btlow*bxlow
+         eTty(i,j,k) = eTty(i,j,k) + rhohstarw2*utlow*velylow + pstar*betaylow - btlow*bylow
+         eTtz(i,j,k) = eTtz(i,j,k) + rhohstarw2*utlow*velzlow + pstar*betazlow - btlow*bzlow
+
+         eTxx(i,j,k) = eTxx(i,j,k) + rhohstarw2*velxlow**2 + pstar*gxx(i,j,k) - bxlow**2
+         eTyy(i,j,k) = eTyy(i,j,k) + rhohstarw2*velylow**2 + pstar*gyy(i,j,k) - bylow**2
+         eTzz(i,j,k) = eTzz(i,j,k) + rhohstarw2*velzlow**2 + pstar*gzz(i,j,k) - bzlow**2
+
+         eTxy(i,j,k) = eTxy(i,j,k) + rhohstarw2*velxlow*velylow + pstar*gxy(i,j,k) - bxlow*bylow
+         eTxz(i,j,k) = eTxz(i,j,k) + rhohstarw2*velxlow*velzlow + pstar*gxz(i,j,k) - bxlow*bzlow
+         eTyz(i,j,k) = eTyz(i,j,k) + rhohstarw2*velylow*velzlow + pstar*gyz(i,j,k) - bylow*bzlow
+
+         if(calculate_bcom .ne. 0) then
+           bcom_sq(i,j,k) = b2
+           bcom0(i,j,k) = w_lorentz(i,j,k)*bdotv/alp(i,j,k)
+           bcomx(i,j,k) = Bvecx(i,j,k)/w_lorentz(i,j,k) + bcom0(i,j,k)*(alp(i,j,k)*velx(i,j,k)-betax(i,j,k))
+           bcomy(i,j,k) = Bvecy(i,j,k)/w_lorentz(i,j,k) + bcom0(i,j,k)*(alp(i,j,k)*vely(i,j,k)-betay(i,j,k))
+           bcomz(i,j,k) = Bvecz(i,j,k)/w_lorentz(i,j,k) + bcom0(i,j,k)*(alp(i,j,k)*velz(i,j,k)-betaz(i,j,k))
+         endif
+
+       end do
+     end do
+   end do
+   !$OMP END PARALLEL DO
+
+   return
+
+ end subroutine GRHydro_TmunuM


Add missing GRHydro_Minima.cc in EinsteinEvolve/GRHydro (einsteintoolkit/Cactus#17)

This PR adds the missing source file GRHydro_Minima.cc to complete the EinsteinEvolve/GRHydro thorn implementation.

Implement the full functionality of GRHydro

Please implement the full functionality of `GRHydro` by finishing the code in `GRHydro_Minima.cc`.

## Thorn Information:
- Name: GRHydro
- Target file: GRHydro_Minima.cc

## Interface Definition in interface.ccl:

## Schedule Definition in schedule.ccl:

## Parameters Definition in param.ccl:

## Configuration Definition in configuration.ccl:

## Documentation Context in readme.md and doc/documentation.tex

## Related Code Context in src folder

diff --git a/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_Minima.cc b/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_Minima.cc
new file mode 100644
index 0000000..0000000
--- /dev/null
+++ b/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_Minima.cc
@@ -0,0 +1,175 @@
+ /*@@
+   @file      GRHydro_Minima.cc
+   @date      Tue Aug 29 18:52:10 2006
+   @author
+   @desc
+        Sets up the scalars used for the atmosphere, after initial data.
+   @enddesc
+ @@*/
+
+#include "cctk.h"
+#include "cctk_Parameters.h"
+#include "cctk_Arguments.h"
+
+//#include "Carpet/Carpet/src/carpet.hh"
+#include "carpet.hh"
+
+#ifdef HAVE_CARPET
+using namespace Carpet;
+#endif
+
+#ifdef __cplusplus
+  extern "C" {
+#endif
+
+    /* Scheduled functions */
+    void GRHydro_Rho_Minima_Setup_Final(CCTK_ARGUMENTS);
+
+    void GRHydro_Rho_Minima_Setup_Final_PUGH(CCTK_ARGUMENTS);
+
+#ifdef __cplusplus
+  } /* extern "C" */
+#endif
+
+ /*@@
+   @routine    GRHydro_Rho_Minima_Setup_Final
+   @date       Tue Aug 29 18:54:05 2006
+   @author     Luca Baiotti
+   @desc
+        After initial data, set up the scalar GRHydro_rho_min used for checking the atmosphere.
+        This is computed taking into account the actual maximum rest-mass density on the grid.
+        Version for Carpet.
+   @enddesc
+   @calls
+   @calledby
+   @history
+
+   @endhistory
+@@*/
+
+void GRHydro_Rho_Minima_Setup_Final(CCTK_ARGUMENTS)
+{
+
+#ifdef HAVE_CARPET
+
+  DECLARE_CCTK_ARGUMENTS;
+  DECLARE_CCTK_PARAMETERS;
+
+  CCTK_REAL  max_rho;
+
+  static int flag = true;
+
+  if (flag) // Actually run this reduction only the first time the routine is called.
+    {
+      if (rho_abs_min > 0.0)
+        {
+          *GRHydro_rho_min = rho_abs_min;
+        }
+      else
+        {
+          // Go to global mode
+
+          BEGIN_GLOBAL_MODE (cctkGH) {
+
+            // Find the global maximum of rho
+
+            const int Reduction_Handle = CCTK_ReductionHandle ("maximum");
+
+            const CCTK_INT input_array_variable_indices= {CCTK_VarIndex("HydroBase::rho")};
+
+            const int ierr = CCTK_Reduce(cctkGH,
+                                         -1, // target processors; -1 -> all
+                                         Reduction_Handle,
+                                         1,  // number of output variables
+                                         CCTK_VARIABLE_REAL,
+                                         &max_rho,
+                                         1,  // number of variables to be reduced
+                                         input_array_variable_indices);
+
+            if (ierr != 0)
+              {
+                CCTK_WARN(0, "Failed to compute the global maximum of rho");
+              }
+
+            *GRHydro_rho_min = max_rho * rho_rel_min;
+
+            // Go back to local mode
+          } END_GLOBAL_MODE;
+
+        }
+      // After this has run once, set the flag so that this does not run again
+      flag = false;
+    } // end if (flag)
+  else
+    {
+      return;
+    }
+
+#endif
+
+  //Debug stuff
+  // char warnline;
+
+  //CCTK_VInfo (CCTK_THORNSTRING, "STEP 2: compute rho max; rho min: %13.12e \n",*GRHydro_rho_min);
+
+ //  printf(warnline,"STEP 2: compute rho max; rho min: %13.12e \n",*GRHydro_rho_min);
+ //CCTK_WARN(1,warnline);
+
+  //  printf(warnline,"STEP 3: recompute ID with new atmosphere; rho min: ', GRHydro_rho_min, 'reflev: ',%i), GRHydro_rho_min, GRHydro_reflevel);
+
+
+
+}
+
+
+ /*@@
+   @routine    GRHydro_Rho_Minima_Setup_Final_PUGH
+   @date       Tue Aug 29 18:57:42 2006
+   @author     Luca Baiotti
+   @desc
+         As above, but for PUGH.
+   @enddesc
+   @calls
+   @calledby
+   @history
+
+   @endhistory
+@@*/
+
+void GRHydro_Rho_Minima_Setup_Final_PUGH(CCTK_ARGUMENTS)
+{
+  DECLARE_CCTK_ARGUMENTS;
+  DECLARE_CCTK_PARAMETERS;
+
+  CCTK_REAL  max_rho;
+
+  if (rho_abs_min > 0.0)
+    {
+      *GRHydro_rho_min = rho_abs_min;
+    }
+  else
+    {
+      // Find the global maximum of rho
+
+      const int Reduction_Handle = CCTK_ReductionHandle("maximum");
+
+      const CCTK_INT input_array_variable_indices={CCTK_VarIndex("HydroBase::rho")};
+
+      const int ierr = CCTK_Reduce(cctkGH,
+                         -1, // target processors; -1 -> all
+                         Reduction_Handle,
+                         1,  // number of output variables
+                         CCTK_VARIABLE_REAL,
+                         &max_rho,
+                         1,  // number of variables to be reduced
+                         input_array_variable_indices);
+
+      if (ierr != 0)
+        {
+          CCTK_WARN(0, "Failed to compute the global maximum of rho");
+        }
+
+      *GRHydro_rho_min = max_rho * rho_rel_min;
+    }
+}
+

diff --git a/all_tests.txt b/all_tests.txt
new file mode 100644
index 0000000..0000000
--- /dev/null
+++ b/all_tests.txt
@@ -0,0 +1,10 @@
+GRHydro_test_shock
+GRHydro_test_shock_mp5
+GRHydro_test_shock_ppm
+GRHydro_test_tov_ppm_ML
+GRHydro_test_tov_ppm_ML_disable_internal_excision
+GRHydro_test_tov_ppm_no_trp_ML
+test_one_hybrid
+tov_carpetevolutionmask
+tov_carpetevolutionmask2
+tov_slowsector


Add missing GRHydro_Source.cc in EinsteinEvolve/GRHydro (einsteintoolkit/Cactus#18)

This PR adds the missing source file GRHydro_Source.cc to complete the EinsteinEvolve/GRHydro thorn implementation.

Implement the full functionality of GRHydro

Please implement the full functionality of `GRHydro` by finishing the code in `GRHydro_Source.cc`.

## Thorn Information:
- Name: GRHydro
- Target file: GRHydro_Source.cc

## Interface Definition in interface.ccl:

## Schedule Definition in schedule.ccl:

## Parameters Definition in param.ccl:

## Configuration Definition in configuration.ccl:

## Documentation Context in readme.md and doc/documentation.tex

## Related Code Context in src folder

diff --git a/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_Source.cc b/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_Source.cc
new file mode 100644
index 0000000..0000000
--- /dev/null
+++ b/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_Source.cc
@@ -0,0 +1,709 @@
+ /*@@
+   @file      GRHydro_Source.cc
+   @date      Nov 29, 2013
+   @author    Christian Reisswig, Joshua Faber, Scott Noble, Bruno Mundim, Ian Hawke
+   @desc
+   The geometric source terms for the matter evolution
+   @enddesc
+ @@*/
+
+
+
+
+#include "cctk.h"
+#include "cctk_Parameters.h"
+#include "cctk_Arguments.h"
+#include "cctk_Functions.h"
+
+#include <cstring>
+#include <cassert>
+
+#define velx vup
+#define vely (&vup[N])
+#define velz (&vup[2*N])
+#define Bvecx Bprim
+#define Bvecy (&Bprim[N])
+#define Bvecz (&Bprim[2*N])
+#define Avecx Avec
+#define Avecy (&Avec[N])
+#define Avecz (&Avec[2*N])
+#define Avecrhsx Avecrhs
+#define Avecrhsy (&Avecrhs[N])
+#define Avecrhsz (&Avecrhs[2*N])
+#define Bconsx Bcons
+#define Bconsy (&Bcons[N])
+#define Bconsz (&Bcons[2*N])
+#define Bconsrhsx Bconsrhs
+#define Bconsrhsy (&Bconsrhs[N])
+#define Bconsrhsz (&Bconsrhs[2*N])
+
+
+extern "C" CCTK_INT GRHydro_UseGeneralCoordinates(const cGH * cctkGH);
+
+template <typename T> static inline T SQR (T const & x) { return x*x; }
+
+
+struct alldiff2 {
+  static void apply(const cGH* const restrict cctkGH,
+                    CCTK_REAL dvars[][3],
+                    const CCTK_REAL* const restrict * const restrict vars,
+                    const int i, const int j, const int k,
+                    const CCTK_REAL* restrict const ih,
+                    const int nvars)
+  {
+     for (int n=0; n < nvars; ++n) {
+        dvars[n][0] = (vars[n][CCTK_GFINDEX3D(cctkGH, i+1,j,k)] - vars[n][CCTK_GFINDEX3D(cctkGH, i-1,j,k)]) * 0.5 * ih[0];
+        dvars[n][1] = (vars[n][CCTK_GFINDEX3D(cctkGH, i,j+1,k)] - vars[n][CCTK_GFINDEX3D(cctkGH, i,j-1,k)]) * 0.5 * ih[1];
+        dvars[n][2] = (vars[n][CCTK_GFINDEX3D(cctkGH, i,j,k+1)] - vars[n][CCTK_GFINDEX3D(cctkGH, i,j,k-1)]) * 0.5 * ih[2];
+     }
+  }
+};
+
+struct alldiff4 {
+  static void apply(const cGH* const restrict cctkGH,
+                    CCTK_REAL dvars[][3],
+                    const CCTK_REAL* const restrict * const restrict vars,
+                    const int i, const int j, const int k,
+                    const CCTK_REAL* restrict const ih,
+                    const int nvars)
+  {
+     for (int n=0; n < nvars; ++n) {
+        dvars[n][0] = (        vars[n][CCTK_GFINDEX3D(cctkGH, i-2,j,k)] - vars[n][CCTK_GFINDEX3D(cctkGH, i+2,j,k)]
+                      + 8.0 * (vars[n][CCTK_GFINDEX3D(cctkGH, i+1,j,k)] - vars[n][CCTK_GFINDEX3D(cctkGH, i-1,j,k)])) * (1.0 / 12.0) * ih[0];
+        dvars[n][1] = (        vars[n][CCTK_GFINDEX3D(cctkGH, i,j-2,k)] - vars[n][CCTK_GFINDEX3D(cctkGH, i,j+2,k)]
+                      + 8.0 * (vars[n][CCTK_GFINDEX3D(cctkGH, i,j+1,k)] - vars[n][CCTK_GFINDEX3D(cctkGH, i,j-1,k)])) * (1.0 / 12.0) * ih[1];
+        dvars[n][2] = (        vars[n][CCTK_GFINDEX3D(cctkGH, i,j,k-2)] - vars[n][CCTK_GFINDEX3D(cctkGH, i,j,k+2)]
+                      + 8.0 * (vars[n][CCTK_GFINDEX3D(cctkGH, i,j,k+1)] - vars[n][CCTK_GFINDEX3D(cctkGH, i,j,k-1)])) * (1.0 / 12.0) * ih[2];
+     }
+  }
+};
+
+
+static inline void UpperMetric(CCTK_REAL& restrict uxx,
+                                CCTK_REAL& restrict uxy,
+                                CCTK_REAL& restrict uxz,
+                                CCTK_REAL& restrict uyy,
+                                CCTK_REAL& restrict uyz,
+                                CCTK_REAL& restrict uzz,
+                                const CCTK_REAL det,
+                                const CCTK_REAL gxx,
+                                const CCTK_REAL gxy,
+                                const CCTK_REAL gxz,
+                                const CCTK_REAL gyy,
+                                const CCTK_REAL gyz,
+                                const CCTK_REAL gzz)
+{
+  const CCTK_REAL invdet = 1.0 / det;
+  uxx = (-gyz*gyz + gyy*gzz)*invdet;
+  uxy = (gxz*gyz - gxy*gzz)*invdet;
+  uxz = (-gxz*gyy + gxy*gyz)*invdet;
+  uyy = (-gxz*gxz + gxx*gzz)*invdet;
+  uyz = (gxy*gxz - gxx*gyz)*invdet;
+  uzz = (-gxy*gxy + gxx*gyy)*invdet;
+
+  return;
+}
+
+
+template <class alldiff,
+          bool do_mhd,
+          bool do_clean_divergence,
+          bool do_Avec>
+static void SourceTerms_LL(CCTK_ARGUMENTS)
+{
+  DECLARE_CCTK_ARGUMENTS;
+  DECLARE_CCTK_PARAMETERS;
+
+  /*
+     Set up multipatch stuff
+  */
+
+  const CCTK_REAL * restrict vup;
+  const CCTK_REAL * restrict Bprim;
+  const CCTK_REAL * restrict g11;
+  const CCTK_REAL * restrict g12;
+  const CCTK_REAL * restrict g13;
+  const CCTK_REAL * restrict g22;
+  const CCTK_REAL * restrict g23;
+  const CCTK_REAL * restrict g33;
+  const CCTK_REAL * restrict k11;
+  const CCTK_REAL * restrict k12;
+  const CCTK_REAL * restrict k13;
+  const CCTK_REAL * restrict k22;
+  const CCTK_REAL * restrict k23;
+  const CCTK_REAL * restrict k33;
+  const CCTK_REAL * restrict beta1;
+  const CCTK_REAL * restrict beta2;
+  const CCTK_REAL * restrict beta3;
+
+  //Multipatch related pointers
+  if(GRHydro_UseGeneralCoordinates(cctkGH)) {
+    vup=lvel;
+    g11=gaa; g12=gab; g13=gac;
+    g22=gbb; g23=gbc;
+    g33=gcc;
+    k11=kaa; k12=kab; k13=kac;
+    k22=kbb; k23=kbc;
+    k33=kcc;
+    Bprim=lBvec;
+    beta1 = betaa;
+    beta2 = betab;
+    beta3 = betac;
+  } else {
+    vup=vel;
+    g11=gxx; g12=gxy; g13=gxz;
+    g22=gyy; g23=gyz;
+    g33=gzz;
+    k11=kxx; k12=kxy; k13=kxz;
+    k22=kyy; k23=kyz;
+    k33=kzz;
+    Bprim=Bvec;
+    beta1 = betax;
+    beta2 = betay;
+    beta3 = betaz;
+  }
+
+  // TODO: for vectorization we might want to use cctk_ash (have to make sure
+  // sensible values exist everywhere)
+  const int nx=cctk_lsh[0];
+  const int ny=cctk_lsh[1];
+  const int nz=cctk_lsh[2];
+
+  const int N = cctk_ash[0]*cctk_ash[1]*cctk_ash[2];
+
+  const CCTK_REAL one = 1.00;
+  const CCTK_REAL two = 2.00;
+  const CCTK_REAL half = 0.50;
+  const CCTK_REAL dx = CCTK_DELTA_SPACE(0);
+  const CCTK_REAL dy = CCTK_DELTA_SPACE(1);
+  const CCTK_REAL dz = CCTK_DELTA_SPACE(2);
+
+  const CCTK_REAL ih[3] = { 1.0/dx, 1.0/dy, 1.0/dz };
+
+#pragma omp parallel for
+  for (int k=GRHydro_stencil; k < nz-GRHydro_stencil; ++k) {
+    for (int j=GRHydro_stencil; j < ny-GRHydro_stencil; ++j) {
+      for (int i=GRHydro_stencil; i < nx-GRHydro_stencil; ++i) {
+
+        const int ijk = CCTK_GFINDEX3D(cctkGH, i, j, k);
+
+        // TODO: these are not needed.
+        const CCTK_REAL localgxx = g11[ijk];
+        const CCTK_REAL localgxy = g12[ijk];
+        const CCTK_REAL localgxz = g13[ijk];
+        const CCTK_REAL localgyy = g22[ijk];
+        const CCTK_REAL localgyz = g23[ijk];
+        const CCTK_REAL localgzz = g33[ijk];
+
+        const CCTK_REAL sqrtdet = sdetg[ijk];
+        const CCTK_REAL invsqrtdet = 1./sqrtdet;
+
+        CCTK_REAL uxx, uxy, uxz, uyy, uyz, uzz;
+        UpperMetric(uxx, uxy, uxz, uyy, uyz, uzz, sqrtdet*sqrtdet, localgxx,
+                    localgxy, localgxz, localgyy, localgyz, localgzz);
+
+        const CCTK_REAL shiftx = beta1[ijk];
+        const CCTK_REAL shifty = beta2[ijk];
+        const CCTK_REAL shiftz = beta3[ijk];
+
+        //  Derivatives of the lapse, metric and shift
+
+        const int nvars = 10;
+        const CCTK_REAL* const restrict vars[nvars] = { beta1, beta2, beta3, alp,
+                                                     g11, g12, g13,
+                                                     g22, g23,
+                                                     g33 };
+        CCTK_REAL dvars[nvars][3];
+
+        alldiff::apply(cctkGH, dvars, vars, i, j, k, ih, nvars);
+
+        const CCTK_REAL dx_betax = dvars[0][0];
+        const CCTK_REAL dx_betay = dvars[1][0];
+        const CCTK_REAL dx_betaz = dvars[2][0];
+
+        const CCTK_REAL dy_betax = dvars[0][1];
+        const CCTK_REAL dy_betay = dvars[1][1];
+        const CCTK_REAL dy_betaz = dvars[2][1];
+
+        const CCTK_REAL dz_betax = dvars[0][2];
+        const CCTK_REAL dz_betay = dvars[1][2];
+        const CCTK_REAL dz_betaz = dvars[2][2];
+
+        const CCTK_REAL dx_alp = dvars[3][0];
+        const CCTK_REAL dy_alp = dvars[3][1];
+        const CCTK_REAL dz_alp = dvars[3][2];
+
+        const CCTK_REAL dx_gxx = dvars[4][0];
+        const CCTK_REAL dx_gxy = dvars[5][0];
+        const CCTK_REAL dx_gxz = dvars[6][0];
+        const CCTK_REAL dx_gyy = dvars[7][0];
+        const CCTK_REAL dx_gyz = dvars[8][0];
+        const CCTK_REAL dx_gzz = dvars[9][0];
+        const CCTK_REAL dy_gxx = dvars[4][1];
+        const CCTK_REAL dy_gxy = dvars[5][1];
+        const CCTK_REAL dy_gxz = dvars[6][1];
+        const CCTK_REAL dy_gyy = dvars[7][1];
+        const CCTK_REAL dy_gyz = dvars[8][1];
+        const CCTK_REAL dy_gzz = dvars[9][1];
+        const CCTK_REAL dz_gxx = dvars[4][2];
+        const CCTK_REAL dz_gxy = dvars[5][2];
+        const CCTK_REAL dz_gxz = dvars[6][2];
+        const CCTK_REAL dz_gyy = dvars[7][2];
+        const CCTK_REAL dz_gyz = dvars[8][2];
+        const CCTK_REAL dz_gzz = dvars[9][2];
+
+
+        const CCTK_REAL invalp = 1.0 / alp[ijk];
+        const CCTK_REAL invalp2 = SQR(invalp);
+        const CCTK_REAL velxshift = velx[ijk] - shiftx*invalp;
+        const CCTK_REAL velyshift = vely[ijk] - shifty*invalp;
+        const CCTK_REAL velzshift = velz[ijk] - shiftz*invalp;
+
+        // vel_i  = g_ij v^j
+        // B_i = g_ij B^i
+
+        const CCTK_REAL vlowx = g11[ijk]*velx[ijk] + g12[ijk]*vely[ijk] + g13[ijk]*velz[ijk];
+        const CCTK_REAL vlowy = g12[ijk]*velx[ijk] + g22[ijk]*vely[ijk] + g23[ijk]*velz[ijk];
+        const CCTK_REAL vlowz = g13[ijk]*velx[ijk] + g23[ijk]*vely[ijk] + g33[ijk]*velz[ijk];
+        const CCTK_REAL Bvecxlow = do_mhd ? g11[ijk]*Bvecx[ijk] + g12[ijk]*Bvecy[ijk] + g13[ijk]*Bvecz[ijk] : 0;
+        const CCTK_REAL Bvecylow = do_mhd ? g12[ijk]*Bvecx[ijk] + g22[ijk]*Bvecy[ijk] + g23[ijk]*Bvecz[ijk] : 0;
+        const CCTK_REAL Bveczlow = do_mhd ? g13[ijk]*Bvecx[ijk] + g23[ijk]*Bvecy[ijk] + g33[ijk]*Bvecz[ijk] : 0;
+
+        //  B^i v_i (= b^0/u^0)
+        const CCTK_REAL Bdotv = do_mhd ? vlowx*Bvecx[ijk]+vlowy*Bvecy[ijk]+vlowz*Bvecz[ijk] : 0;
+
+        // v^2 = v_i v^i; w=(1-v^2)^{-1/2}
+
+        const CCTK_REAL v2 = vlowx*velx[ijk] + vlowy*vely[ijk] + vlowz*velz[ijk];
+        const CCTK_REAL invw = sqrt(1.0-v2);
+        const CCTK_REAL w = 1./invw;
+
+        // b^2 = B^i B_i / w^2 + (b^0/u^0)^2
+
+        const CCTK_REAL b2 = do_mhd ? (Bvecx[ijk]*Bvecxlow+Bvecy[ijk]*Bvecylow+Bvecz[ijk]*Bveczlow)*SQR(invw)+SQR(Bdotv) : 0;
+
+        // b_i = B_i/w +w*(B dot v)*v_i
+        const CCTK_REAL bxlow = do_mhd ? Bvecxlow*invw+w*Bdotv*vlowx : 0;
+        const CCTK_REAL bylow = do_mhd ? Bvecylow*invw+w*Bdotv*vlowy : 0;
+        const CCTK_REAL bzlow = do_mhd ? Bveczlow*invw+w*Bdotv*vlowz : 0;
+
+
+        // These are the contravariant components
+        const CCTK_REAL bt = do_mhd ? w*invalp*Bdotv : 0;
+        const CCTK_REAL bx = do_mhd ? Bvecx[ijk]*invw+w*Bdotv*velxshift : 0;
+        const CCTK_REAL by = do_mhd ? Bvecy[ijk]*invw+w*Bdotv*velyshift : 0;
+        const CCTK_REAL bz = do_mhd ? Bvecz[ijk]*invw+w*Bdotv*velzshift : 0;
+
+        // TODO: all of these can be expressed much more easily in terms of the
+        // conservatives
+        const CCTK_REAL rhohstarW2 = (rho[ijk]*(one + eps[ijk]) + (press[ijk] + b2)) *
+                                       SQR(w);
+        const CCTK_REAL pstar = press[ijk]+0.50*b2;
+
+        //  For a change, these are T^{ij}
+
+        // TODO: strict IEEE compliance does not let the compiler remove "+ 0"
+        // terms, so we have to do something else here
+        const CCTK_REAL t00 = (rhohstarW2 - pstar)*invalp2-SQR(bt);
+        const CCTK_REAL t0x = rhohstarW2*velxshift*invalp +
+             pstar*shiftx*invalp2-bt*bx;
+        const CCTK_REAL t0y = rhohstarW2*velyshift*invalp +
+             pstar*shifty*invalp2-bt*by;
+        const CCTK_REAL t0z = rhohstarW2*velzshift*invalp +
+             pstar*shiftz*invalp2-bt*bz;
+        const CCTK_REAL txx = rhohstarW2*velxshift*velxshift +
+             pstar*(uxx - shiftx*shiftx*invalp2)-SQR(bx);
+        const CCTK_REAL txy = rhohstarW2*velxshift*velyshift +
+             pstar*(uxy - shiftx*shifty*invalp2)-bx*by;
+        const CCTK_REAL txz = rhohstarW2*velxshift*velzshift +
+             pstar*(uxz - shiftx*shiftz*invalp2)-bx*bz;
+        const CCTK_REAL tyy = rhohstarW2*velyshift*velyshift +
+             pstar*(uyy - shifty*shifty*invalp2)-SQR(by);
+        const CCTK_REAL tyz = rhohstarW2*velyshift*velzshift +
+             pstar*(uyz - shifty*shiftz*invalp2)-by*bz;
+        const CCTK_REAL tzz = rhohstarW2*velzshift*velzshift +
+             pstar*(uzz - shiftz*shiftz*invalp2)-SQR(bz);
+
+//        Contract the shift with the extrinsic curvature
+
+        const CCTK_REAL shiftshiftk = shiftx*shiftx*k11[ijk] +
+                                      shifty*shifty*k22[ijk] +
+                                      shiftz*shiftz*k33[ijk] +
+             two*(shiftx*shifty*k12[ijk] +
+                  shiftx*shiftz*k13[ijk] +
+                  shifty*shiftz*k23[ijk]);
+
+        const CCTK_REAL shiftkx = shiftx*k11[ijk] + shifty*k12[ijk] + shiftz*k13[ijk];
+        const CCTK_REAL shiftky = shiftx*k12[ijk] + shifty*k22[ijk] + shiftz*k23[ijk];
+        const CCTK_REAL shiftkz = shiftx*k13[ijk] + shifty*k23[ijk] + shiftz*k33[ijk];
+
+//        Contract the matter terms with the extrinsic curvature
+
+        const CCTK_REAL sumTK = txx*k11[ijk] + tyy*k22[ijk] + tzz*k33[ijk]
+                         + two*(txy*k12[ijk] + txz*k13[ijk] + tyz*k23[ijk]);
+
+//        Update term for tau
+
+        const CCTK_REAL tau_source = t00*
+             (shiftshiftk - (shiftx*dx_alp + shifty*dy_alp + shiftz*dz_alp) )
+             + t0x*(-dx_alp + two*shiftkx)
+             + t0y*(-dy_alp + two*shiftky)
+             + t0z*(-dz_alp + two*shiftkz)
+             + sumTK;
+
+//        The following looks very little like the terms in the
+//        standard papers. Take a look in the ThornGuide to see why
+//        it is really the same thing.
+
+//        Contract the shift with derivatives of the metric
+
+        const CCTK_REAL halfshiftdgx = half*(shiftx*shiftx*dx_gxx +
+             shifty*shifty*dx_gyy + shiftz*shiftz*dx_gzz) +
+             shiftx*shifty*dx_gxy + shiftx*shiftz*dx_gxz +
+             shifty*shiftz*dx_gyz;
+        const CCTK_REAL halfshiftdgy = half*(shiftx*shiftx*dy_gxx +
+             shifty*shifty*dy_gyy + shiftz*shiftz*dy_gzz) +
+             shiftx*shifty*dy_gxy + shiftx*shiftz*dy_gxz +
+             shifty*shiftz*dy_gyz;
+        const CCTK_REAL halfshiftdgz = half*(shiftx*shiftx*dz_gxx +
+             shifty*shifty*dz_gyy + shiftz*shiftz*dz_gzz) +
+             shiftx*shifty*dz_gxy + shiftx*shiftz*dz_gxz +
+             shifty*shiftz*dz_gyz;
+
+//        Contract the matter with derivatives of the metric
+
+        const CCTK_REAL halfTdgx = half*(txx*dx_gxx + tyy*dx_gyy + tzz*dx_gzz) +
+             txy*dx_gxy + txz*dx_gxz + tyz*dx_gyz;
+        const CCTK_REAL halfTdgy = half*(txx*dy_gxx + tyy*dy_gyy + tzz*dy_gzz) +
+             txy*dy_gxy + txz*dy_gxz + tyz*dy_gyz;
+        const CCTK_REAL halfTdgz = half*(txx*dz_gxx + tyy*dz_gyy + tzz*dz_gzz) +
+             txy*dz_gxy + txz*dz_gxz + tyz*dz_gyz;
+
+
+       const CCTK_REAL sx_source = t00*
+             (halfshiftdgx - alp[ijk]*dx_alp) + halfTdgx +
+             t0x*(shiftx*dx_gxx + shifty*dx_gxy + shiftz*dx_gxz) +
+             t0y*(shiftx*dx_gxy + shifty*dx_gyy + shiftz*dx_gyz) +
+             t0z*(shiftx*dx_gxz + shifty*dx_gyz + shiftz*dx_gzz) +
+             rhohstarW2*invalp*(vlowx*dx_betax + vlowy*dx_betay + vlowz*dx_betaz) -
+             bt*(bxlow*dx_betax + bylow*dx_betay + bzlow*dx_betaz);
+
+       const CCTK_REAL sy_source = t00*
+             (halfshiftdgy - alp[ijk]*dy_alp) + halfTdgy +
+             t0x*(shiftx*dy_gxx + shifty*dy_gxy + shiftz*dy_gxz) +
+             t0y*(shiftx*dy_gxy + shifty*dy_gyy + shiftz*dy_gyz) +
+             t0z*(shiftx*dy_gxz + shifty*dy_gyz + shiftz*dy_gzz) +
+             rhohstarW2*invalp*(vlowx*dy_betax + vlowy*dy_betay + vlowz*dy_betaz) -
+             bt*(bxlow*dy_betax + bylow*dy_betay + bzlow*dy_betaz);
+
+       const CCTK_REAL sz_source = t00*
+             (halfshiftdgz - alp[ijk]*dz_alp) + halfTdgz +
+             t0x*(shiftx*dz_gxx + shifty*dz_gxy + shiftz*dz_gxz) +
+             t0y*(shiftx*dz_gxy + shifty*dz_gyy + shiftz*dz_gyz) +
+             t0z*(shiftx*dz_gxz + shifty*dz_gyz + shiftz*dz_gzz) +
+             rhohstarW2*invalp*(vlowx*dz_betax + vlowy*dz_betay + vlowz*dz_betaz) -
+             bt*(bxlow*dz_betax + bylow*dz_betay + bzlow*dz_betaz);
+
+        densrhs[ijk] = 0.0;
+        srhs[ijk]        = alp[ijk]*sqrtdet*sx_source;
+        srhs[ijk + N]    = alp[ijk]*sqrtdet*sy_source;
+        srhs[ijk + 2*N]  = alp[ijk]*sqrtdet*sz_source;
+        taurhs[ijk]      = alp[ijk]*sqrtdet*tau_source;
+
+        if (do_Avec) {
+
+          // B^i and A^i both live in cell centers currently
+          const CCTK_REAL Avcx_source = alp[ijk]*sqrtdet*(velyshift*Bvecz[ijk] - velzshift*Bvecy[ijk]);
+          const CCTK_REAL Avcy_source = alp[ijk]*sqrtdet*(velzshift*Bvecx[ijk] - velxshift*Bvecz[ijk]);
+          const CCTK_REAL Avcz_source = alp[ijk]*sqrtdet*(velxshift*Bvecy[ijk] - velyshift*Bvecx[ijk]);
+
+          Avecrhsx[ijk] = Avcx_source;
+          Avecrhsy[ijk] = Avcy_source;
+          Avecrhsz[ijk] = Avcz_source;
+
+        }
+
+        if(do_clean_divergence) {
+
+           // g^{jk} d_i g_{kj} = d_i (g) / det
+           const CCTK_REAL dx_det_bydet = uxx*dx_gxx + uyy*dx_gyy + uzz*dx_gzz +
+                two*(uxy*dx_gxy+uxz*dx_gxz+uyz*dx_gyz);
+           const CCTK_REAL dy_det_bydet = uxx*dy_gxx + uyy*dy_gyy + uzz*dy_gzz +
+                two*(uxy*dy_gxy+uxz*dy_gxz+uyz*dy_gyz);
+           const CCTK_REAL dz_det_bydet = uxx*dz_gxx + uyy*dz_gyy + uzz*dz_gzz +
+                two*(uxy*dz_gxy+uxz*dz_gxz+uyz*dz_gyz);
+
+           // g^{ik} d_k g_{li}
+           const CCTK_REAL gdg_x = uxx*dx_gxx + uxy*dy_gxx + uxz*dz_gxx +
+                   uxy*dx_gxy + uyy*dy_gxy + uyz*dz_gxy +
+                   uxz*dx_gxz + uyz*dy_gxz + uzz*dz_gxz;
+
+           const CCTK_REAL gdg_y = uxx*dx_gxy + uxy*dy_gxy + uxz*dz_gxy +
+                   uxy*dx_gyy + uyy*dy_gyy + uyz*dz_gyy +
+                   uxz*dx_gyz + uyz*dy_gyz + uzz*dz_gyz;
+
+           const CCTK_REAL gdg_z = uxx*dx_gxz + uxy*dy_gxz + uxz*dz_gxz +
+                   uxy*dx_gyz + uyy*dy_gyz + uyz*dz_gyz +
+                   uxz*dx_gzz + uyz*dy_gzz + uzz*dz_gzz;
+
+           CCTK_REAL bvcx_source, bvcy_source, bvcz_source;
+           psidcrhs[ijk] = -one * (kap_dc*alp[ijk] +
+                dx_betax + dy_betay + dz_betaz ) * psidc[ijk] +
+                Bconsx[ijk] * (dx_alp - half*alp[ijk] *
+                 ( uxx*dx_gxx + uyy*dx_gyy + uzz*dx_gzz + two*uxy*dx_gxy +
+                   two*uxz*dx_gxz + two*uyz*dx_gyz ) )*invsqrtdet +
+                Bconsy[ijk] * (dy_alp - half*alp[ijk] *
+                 ( uxx*dy_gxx + uyy*dy_gyy + uzz*dy_gzz + two*uxy*dy_gxy +
+                   two*uxz*dy_gxz + two*uyz*dy_gyz ) )*invsqrtdet +
+                Bconsz[ijk] * (dz_alp - half*alp[ijk] *
+                 ( uxx*dz_gxx + uyy*dz_gyy + uzz*dz_gzz + two*uxy*dz_gxy +
+                   two*uxz*dz_gxz + two*uyz*dz_gyz ) )*invsqrtdet;
+
+           bvcx_source = -one * ( Bconsx[ijk]*dx_betax +
+                Bconsy[ijk]*dy_betax + Bconsz[ijk]*dz_betax ) +
+                psidc[ijk]*sqrtdet*(( uxx*dx_alp+uxy*dy_alp+uxz*dz_alp ) +
+                alp[ijk]*(half*( uxx*dx_det_bydet +
+                  uxy*dy_det_bydet + uxz*dz_det_bydet) -
+                ( uxx*gdg_x + uxy*gdg_y + uxz*gdg_z )));
+
+           bvcy_source = -one * ( Bconsx[ijk]*dx_betay +
+                Bconsy[ijk]*dy_betay + Bconsz[ijk]*dz_betay ) +
+                psidc[ijk]*sqrtdet*(( uxy*dx_alp+uyy*dy_alp+uyz*dz_alp ) +
+                alp[ijk]*(half*( uxy*dx_det_bydet +
+                  uyy*dy_det_bydet + uyz*dz_det_bydet ) -
+                ( uxy*gdg_x + uyy*gdg_y + uyz*gdg_z )));
+
+           bvcz_source = -one * ( Bconsx[ijk]*dx_betaz +
+                Bconsy[ijk]*dy_betaz + Bconsz[ijk]*dz_betaz ) +
+                psidc[ijk]*sqrtdet*(( uxz*dx_alp+uyz*dy_alp+uzz*dz_alp ) +
+                alp[ijk]*(half*( uxz*dx_det_bydet +
+                  uyz*dy_det_bydet + uzz*dz_det_bydet ) -
+                ( uxz*gdg_x + uyz*gdg_y + uzz*gdg_z )));
+
+           Bconsrhsx[ijk] = bvcx_source;
+           Bconsrhsy[ijk] = bvcy_source;
+           Bconsrhsz[ijk] = bvcz_source;
+        } // if(do_clean_divergence)
+
+      } // for(i
+    } // for(j
+  } // for(k
+
+#if(0) // poison edges of domain
+  static int last_iteration_seen = -1;
+  static int reflevel = -1;
+  static int mol_substep;
+  if(last_iteration_seen != cctk_iteration || reflevel != *GRHydro_reflevel) {
+    last_iteration_seen = cctk_iteration;
+    reflevel = *GRHydro_reflevel;
+    mol_substep = 0;
+  } else {
+    mol_substep += 1;
+  }
+  for(k = 0 ; k < GRHydro_stencil*mol_substep ; ++k) {
+    for(j = 0 ; j < cctk_ash[1] ; ++j) {
+      for(i = 0 ; i < cctk_ash[0] ; ++i) {
+        dens[CCTK_GFINDEX3D(i,j,k)] = -1e100
+        Scon[CCTK_VECTGFINDEX3D(i,j,k,0)] = -1e100
+        Scon[CCTK_VECTGFINDEX3D(i,j,k,1)] = -1e100
+        Scon[CCTK_VECTGFINDEX3D(i,j,k,2)] = -1e100
+        tau[CCTK_GFINDEX3D(i,j,k)] = -1e100
+        if(do_mhd) {
+          Bcons[CCTK_VECTGFINDEX3D(i,j,k,0)] = -1e100
+          Bcons[CCTK_VECTGFINDEX3D(i,j,k,1)] = -1e100
+          Bcons[CCTK_VECTGFINDEX3D(i,j,k,2)] = -1e100
+        }
+        if(do_clean_divergene)
+          psidc[CCTK_GFINDEX3D(i,j,k)] = -1e100
+      }
+    }
+  }
+  for(k = cctk_ash[2]-GRHydro_stencil*mol_substep ; k < cctk_ash[2] ; ++k) {
+    for(j = 0 ; j < cctk_ash[1] ; ++j) {
+      for(i = 0 ; i < cctk_ash[0] ; ++i) {
+        dens[CCTK_GFINDEX3D(i,j,k)] = -1e100
+        Scon[CCTK_VECTGFINDEX3D(i,j,k,0)] = -1e100
+        Scon[CCTK_VECTGFINDEX3D(i,j,k,1)] = -1e100
+        Scon[CCTK_VECTGFINDEX3D(i,j,k,2)] = -1e100
+        tau[CCTK_GFINDEX3D(i,j,k)] = -1e100
+        if(do_mhd) {
+          Bcons[CCTK_VECTGFINDEX3D(i,j,k,0)] = -1e100
+          Bcons[CCTK_VECTGFINDEX3D(i,j,k,1)] = -1e100
+          Bcons[CCTK_VECTGFINDEX3D(i,j,k,2)] = -1e100
+        }
+        if(do_clean_divergene)
+          psidc[CCTK_GFINDEX3D(i,j,k)] = -1e100
+      }
+    }
+  }
+  for(i = 0 ; i < GRHydro_stencil*mol_substep ; ++i) {
+    for(k = 0 ; k < cctk_ash[2] ; ++k) {
+      for(j = 0 ; j < cctk_ash[1] ; ++j) {
+        dens[CCTK_GFINDEX3D(i,j,k)] = -1e100
+        Scon[CCTK_VECTGFINDEX3D(i,j,k,0)] = -1e100
+        Scon[CCTK_VECTGFINDEX3D(i,j,k,1)] = -1e100
+        Scon[CCTK_VECTGFINDEX3D(i,j,k,2)] = -1e100
+        tau[CCTK_GFINDEX3D(i,j,k)] = -1e100
+        if(do_mhd) {
+          Bcons[CCTK_VECTGFINDEX3D(i,j,k,0)] = -1e100
+          Bcons[CCTK_VECTGFINDEX3D(i,j,k,1)] = -1e100
+          Bcons[CCTK_VECTGFINDEX3D(i,j,k,2)] = -1e100
+        }
+        if(do_clean_divergene)
+          psidc[CCTK_GFINDEX3D(i,j,k)] = -1e100
+      }
+    }
+  }
+  for(i = cctk_ash[0]-GRHydro_stencil*mol_substep ; i < cctk_ash[0] ; ++i) {
+    for(k = 0 ; k < cctk_ash[2] ; ++k) {
+      for(j = 0 ; j < cctk_ash[1] ; ++j) {
+        dens[CCTK_GFINDEX3D(i,j,k)] = -1e100
+        Scon[CCTK_VECTGFINDEX3D(i,j,k,0)] = -1e100
+        Scon[CCTK_VECTGFINDEX3D(i,j,k,1)] = -1e100
+        Scon[CCTK_VECTGFINDEX3D(i,j,k,2)] = -1e100
+        tau[CCTK_GFINDEX3D(i,j,k)] = -1e100
+        if(do_mhd) {
+          Bcons[CCTK_VECTGFINDEX3D(i,j,k,0)] = -1e100
+          Bcons[CCTK_VECTGFINDEX3D(i,j,k,1)] = -1e100
+          Bcons[CCTK_VECTGFINDEX3D(i,j,k,2)] = -1e100
+        }
+        if(do_clean_divergene)
+          psidc[CCTK_GFINDEX3D(i,j,k)] = -1e100
+      }
+    }
+  }
+  for(j = 0 ; j < GRHydro_stencil*mol_substep ; ++j) {
+    for(i = 0 ; i < cctk_ash[0] ; ++i) {
+      for(k = 0 ; k < cctk_ash[2] ; ++k) {
+        dens[CCTK_GFINDEX3D(i,j,k)] = -1e100
+        Scon[CCTK_VECTGFINDEX3D(i,j,k,0)] = -1e100
+        Scon[CCTK_VECTGFINDEX3D(i,j,k,1)] = -1e100
+        Scon[CCTK_VECTGFINDEX3D(i,j,k,2)] = -1e100
+        tau[CCTK_GFINDEX3D(i,j,k)] = -1e100
+        if(do_mhd) {
+          Bcons[CCTK_VECTGFINDEX3D(i,j,k,0)] = -1e100
+          Bcons[CCTK_VECTGFINDEX3D(i,j,k,1)] = -1e100
+          Bcons[CCTK_VECTGFINDEX3D(i,j,k,2)] = -1e100
+        }
+        if(do_clean_divergene)
+          psidc[CCTK_GFINDEX3D(i,j,k)] = -1e100
+      }
+    }
+  }
+  for(j = cctk_ash[1]-GRHydro_stencil*mol_substep ; j < cctk_ash[1] ; ++j) {
+    for(i = 0 ; i < cctk_ash[0] ; ++i) {
+      for(k = 0 ; k < cctk_ash[2] ; ++k) {
+        dens[CCTK_GFINDEX3D(i,j,k)] = -1e100
+        Scon[CCTK_VECTGFINDEX3D(i,j,k,0)] = -1e100
+        Scon[CCTK_VECTGFINDEX3D(i,j,k,1)] = -1e100
+        Scon[CCTK_VECTGFINDEX3D(i,j,k,2)] = -1e100
+        tau[CCTK_GFINDEX3D(i,j,k)] = -1e100
+        if(do_mhd) {
+          Bcons[CCTK_VECTGFINDEX3D(i,j,k,0)] = -1e100
+          Bcons[CCTK_VECTGFINDEX3D(i,j,k,1)] = -1e100
+          Bcons[CCTK_VECTGFINDEX3D(i,j,k,2)] = -1e100
+        }
+        if(do_clean_divergene)
+          psidc[CCTK_GFINDEX3D(i,j,k)] = -1e100
+      }
+    }
+  }
+#endif
+}
+
+extern "C" void SourceTerms(CCTK_ARGUMENTS)
+{
+  DECLARE_CCTK_ARGUMENTS;
+  DECLARE_CCTK_PARAMETERS;
+
+  void (*sources)(CCTK_ARGUMENTS) = NULL;
+  const bool do_Avec = CCTK_EQUALS(Bvec_evolution_method, "GRHydro_Avec");
+#define T(e,t) (!!(e)==(t))
+#define T3(t1,t2,t3)                \
+        (T(*evolve_MHD,t1)      &&  \
+         T(clean_divergence,t2) &&  \
+         T(do_Avec,t3))
+#define SET_SOURCES(df,t1,t2,t3) \
+        if(T3(t1,t2,t3))         \
+          sources=SourceTerms_LL<df,t1,t2,t3>;
+  enum do_t {noMHD=0,MHD=1,noDC=0,DC=1,noAVec=0,AVec=1};
+
+  // order of arguments:
+  // do_MHD do_clean_divergence do_Avec
+  if(sources_spatial_order == 4) {
+    // no MHD
+    SET_SOURCES(alldiff4,noMHD,noDC,noAVec)
+
+    // not all combinations make sense for MHD, we list the ones that do
+    SET_SOURCES(alldiff4,  MHD,noDC,noAVec)
+    SET_SOURCES(alldiff4,  MHD,  DC,noAVec)
+    SET_SOURCES(alldiff4,  MHD,  DC,noAVec)
+    SET_SOURCES(alldiff4,  MHD,noDC,  AVec)
+  } else if(sources_spatial_order == 2) {
+    // no MHD
+    SET_SOURCES(alldiff2,noMHD,noDC,noAVec)
+
+    // not all combinations make sense for MHD, we list the ones that do
+    SET_SOURCES(alldiff2,  MHD,noDC,noAVec)
+    SET_SOURCES(alldiff2,  MHD,  DC,noAVec)
+    SET_SOURCES(alldiff2,  MHD,  DC,noAVec)
+    SET_SOURCES(alldiff2,  MHD,noDC,  AVec)
+  }
+
+  if(sources == NULL) {
+    CCTK_VError(__LINE__, __FILE__, CCTK_THORNSTRING,
+                "No source terms can be computed for combination "
+                "do_mhd = %d "
+                "do_clean_divergence = %d "
+                "do_Avec = %d sources_spatial_order = %d",
+                int(*evolve_MHD),
+                int(clean_divergence),
+                int(do_Avec), int(sources_spatial_order));
+  }
+
+  // do some non-time critial operations here to reduce the number of templates
+  if(transport_constraints) {
+    const int N = cctk_ash[0]*cctk_ash[1]*cctk_ash[2];
+    // not much need to OpenMP parallelize this since it is pure memory access
+    // and memset is highly optimized
+    memset(Evec, 0, sizeof(Evec[0])*3*N);
+    // I think IEEE requires an all zero byte double to be 0.
+    assert(Evec[0] == 0.);
+  }
+  if(*evolve_Y_e) {
+    const int N = cctk_ash[0]*cctk_ash[1]*cctk_ash[2];
+    // not much need to OpenMP parallelize this since it is pure memory access
+    // and memset is highly optimized
+    memset(Y_e_con_rhs, 0, sizeof(Y_e_con_rhs[0])*N);
+    // I think IEEE requires an all zero byte double to be 0.
+    assert(Y_e_con_rhs[0] == 0.);
+  }
+  if(number_of_tracers>0) {
+    const int N = cctk_ash[0]*cctk_ash[1]*cctk_ash[2];
+    // not much need to OpenMP parallelize this since it is pure memory access
+    // and memset is highly optimized
+    memset(cons_tracerrhs, 0, sizeof(cons_tracerrhs[0])*number_of_tracers*N);
+    // I think IEEE requires an all zero byte double to be 0.
+    assert(cons_tracerrhs[0] == 0.);
+  }
+  if(*evolve_entropy) {
+    const int N = cctk_ash[0]*cctk_ash[1]*cctk_ash[2];
+    // not much need to OpenMP parallelize this since it is pure memory access
+    // and memset is highly optimized
+    memset(entropyrhs, 0, sizeof(entropyrhs[0])*N);
+    // I think IEEE requires an all zero byte double to be 0.
+    assert(entropyrhs[0] == 0.);
+  }
+  if(track_divB) {
+    const int N = cctk_ash[0]*cctk_ash[1]*cctk_ash[2];
+    // not much need to OpenMP parallelize this since it is pure memory access
+    // and memset is highly optimized
+    memset(divB, 0, sizeof(divB[0])*N);
+    // I think IEEE requires an all zero byte double to be 0.
+    assert(divB[0] == 0.);
+  }
+
+  sources(CCTK_PASS_CTOC);
+}
+
+


Add missing GRHydro_RiemannSolveAM.F90 in EinsteinEvolve/GRHydro (einsteintoolkit/Cactus#19)

This PR adds the missing source file GRHydro_RiemannSolveAM.F90 to complete the EinsteinEvolve/GRHydro thorn implementation.

Implement the full functionality of GRHydro

Please implement the full functionality of `GRHydro` by finishing the code in `GRHydro_RiemannSolveAM.F90`.

## Thorn Information:
- Name: GRHydro
- Target file: GRHydro_RiemannSolveAM.F90

## Interface Definition in interface.ccl:

## Schedule Definition in schedule.ccl:

## Parameters Definition in param.ccl:

## Configuration Definition in configuration.ccl:

## Documentation Context in readme.md and doc/documentation.tex

## Related Code Context in src folder

diff --git a/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_RiemannSolveAM.F90 b/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_RiemannSolveAM.F90
new file mode 100644
index 0000000..0000000
--- /dev/null
+++ b/arrangements/EinsteinEvolve/GRHydro/src/GRHydro_RiemannSolveAM.F90
@@ -0,0 +1,142 @@
+ /*@@
+   @file      GRHydro_RiemannSolveAM.F90
+   @date      Sep 1, 2010
+   @author
+   @desc
+   A wrapper routine to call the correct Riemann solver
+   @enddesc
+ @@*/
+
+#include "cctk.h"
+#include "cctk_Parameters.h"
+#include "cctk_Arguments.h"
+#include "cctk_Functions.h"
+#include "GRHydro_Macros.h"
+
+ /*@@
+   @routine    RiemannSolveAM
+   @date       Sep 1, 2010
+   @author     Joshua Faber, Scott Noble, Bruno Mundim, Pedro Montero, Ian Hawke
+   @desc
+   A wrapper routine to switch between the different Riemann solvers.
+   @enddesc
+   @calls
+   @calledby
+   @history
+
+   @endhistory
+
+@@*/
+
+subroutine RiemannSolveAM(CCTK_ARGUMENTS)
+
+  implicit none
+
+  DECLARE_CCTK_ARGUMENTS
+  DECLARE_CCTK_PARAMETERS
+  DECLARE_CCTK_FUNCTIONS
+
+  if (CCTK_EQUALS(riemann_solver,"HLLE").or.CCTK_EQUALS(riemann_solver,"LLF")) then
+
+     call GRHydro_HLLE_AM(CCTK_PASS_FTOF)
+
+     if (evolve_tracer .ne. 0) then
+
+!!$ There are no special calls for tracers, which care not one whit about B-fields!
+!!$ Just call the standard version...
+
+        call GRHydro_HLLE_Tracer(CCTK_PASS_FTOF)
+
+     end if
+
+!!$  else if (CCTK_EQUALS(riemann_solver,"Roe")) then
+!!$
+!!$    call GRHydro_RoeSolveAM(CCTK_PASS_FTOF)
+!!$
+!!$    if (evolve_tracer .ne. 0) then
+!!$
+!!$      call GRHydro_HLLE_Tracer(CCTK_PASS_FTOF)
+!!$
+!!$    end if
+!!$
+!!$  else if (CCTK_EQUALS(riemann_solver,"Marquina")) then
+!!$
+!!$    call GRHydro_MarquinaM(CCTK_PASS_FTOF)
+
+!!$    Tracers are built directly in to the Marquina solver
+
+  else
+
+     call CCTK_ERROR("Roe and Marquina not implemented in MHD yet!!!")
+     STOP
+
+  end if
+
+end subroutine RiemannSolveAM
+
+ /*@@
+   @routine    RiemannSolvePolytypeAM
+   @date       Sep 1, 2010
+   @author     Joshua Faber, Scott Noble, Bruno Mundim, Ian Hawke
+   @desc
+   The same as above, just specializing to polytropic type EOS.
+   Currently there is no point to this routine right now.
+   @enddesc
+   @calls
+   @calledby
+   @history
+
+   @endhistory
+
+@@*/
+
+
+subroutine RiemannSolvePolytypeAM(CCTK_ARGUMENTS)
+
+  implicit none
+
+  DECLARE_CCTK_ARGUMENTS
+  DECLARE_CCTK_PARAMETERS
+  DECLARE_CCTK_FUNCTIONS
+
+  if (CCTK_EQUALS(riemann_solver,"HLLE")) then
+
+     call GRHydro_HLLE_AM(CCTK_PASS_FTOF)
+
+     if (evolve_tracer .ne. 0) then
+
+!!$ Call the non-MHD version - see above
+
+        call GRHydro_HLLE_Tracer(CCTK_PASS_FTOF)
+
+     end if
+
+!!$  else if (CCTK_EQUALS(riemann_solver,"Roe")) then
+!!$
+!!$    call GRHydro_RoeSolve(CCTK_PASS_FTOF)
+!!$
+!!$    if (evolve_tracer .ne. 0) then
+!!$
+!!$      call GRHydro_HLLE_Tracer(CCTK_PASS_FTOF)
+!!$
+!!$    end if
+!!$
+!!$  else if (CCTK_EQUALS(riemann_solver,"Marquina")) then
+!!$
+!!$    call GRHydro_Marquina(CCTK_PASS_FTOF)
+
+!!$    Tracers are built directly in to the Marquina solver
+
+  else
+
+     call CCTK_ERROR("Roe and Marquina not implemented in MHD yet!!!")
+     STOP
+
+  end if
+
+end subroutine RiemannSolvePolytypeAM
+
+
+
+
+


Add missing WaveMoLRegister.c in CactusExamples/WaveMoL (einsteintoolkit/Cactus#2)

This PR adds the missing source file WaveMoLRegister.c to complete the CactusExamples/WaveMoL thorn implementation.

Implement the full functionality of WaveMoL

Please implement the full functionality of `WaveMoL` by finishing the code in `WaveMoLRegister.c`.

## Thorn Information:
- Name: WaveMoL
- Target file: WaveMoLRegister.c

## Interface Definition in interface.ccl:

## Schedule Definition in schedule.ccl:

## Parameters Definition in param.ccl:

## Configuration Definition in configuration.ccl:

## Documentation Context in readme.md and doc/documentation.tex

## Related Code Context in src folder

diff --git a/arrangements/CactusExamples/WaveMoL/src/WaveMoLRegister.c b/arrangements/CactusExamples/WaveMoL/src/WaveMoLRegister.c
new file mode 100644
index 0000000..0000000
--- /dev/null
+++ b/arrangements/CactusExamples/WaveMoL/src/WaveMoLRegister.c
@@ -0,0 +1,93 @@
+ /*@@
+   @file      WaveMoLRegister.c
+   @date      Fri Nov  9 13:47:07 2001
+   @author    Ian Hawke
+   @desc
+   Routine to register the variables with the MoL thorn.
+   @enddesc
+ @@*/
+
+#include "cctk.h"
+#include "cctk_Arguments.h"
+#include "cctk_Parameters.h"
+
+static const char *rcsid = "$Header$";
+
+CCTK_FILEVERSION(CactusExamples_WaveMoL_WaveMoLRegister_c)
+
+/*
+#ifndef DEBUG_MOL
+#define DEBUG_MOL
+#endif
+*/
+
+void WaveMoL_RegisterVars(CCTK_ARGUMENTS);
+
+ /*@@
+   @routine    WaveMoL_RegisterVars
+   @date       Fri Nov  9 13:47:41 2001
+   @author     Ian Hawke
+   @desc
+   The registration routine.
+   @enddesc
+   @calls
+   @calledby
+   @history
+
+   @endhistory
+
+@@*/
+
+void WaveMoL_RegisterVars(CCTK_ARGUMENTS)
+{
+
+  DECLARE_CCTK_ARGUMENTS;
+  DECLARE_CCTK_PARAMETERS;
+
+  CCTK_INT ierr = 0, group, rhs, var;
+
+  group = CCTK_GroupIndex("wavemol::scalarevolvemol_scalar");
+  rhs = CCTK_GroupIndex("wavemol::scalarrhsmol_scalar");
+
+  if (CCTK_IsFunctionAliased("MoLRegisterEvolvedGroup"))
+  {
+    ierr += MoLRegisterEvolvedGroup(group, rhs);
+  }
+  else
+  {
+    CCTK_WARN(0, "MoL function not aliased");
+    ierr++;
+  }
+
+  group = CCTK_GroupIndex("wavemol::scalarevolvemol_vector");
+  rhs = CCTK_GroupIndex("wavemol::scalarrhsmol_vector");
+
+  if (CCTK_IsFunctionAliased("MoLRegisterEvolvedGroup"))
+  {
+    ierr += MoLRegisterEvolvedGroup(group, rhs);
+  }
+  else
+  {
+    CCTK_WARN(0, "MoL function MoLRegisterEvolvedGroup not aliased");
+    ierr++;
+  }
+
+  var = CCTK_VarIndex("wavemol::energy");
+
+  if (CCTK_IsFunctionAliased("MoLRegisterConstrained"))
+  {
+    ierr += MoLRegisterConstrained(var);
+  }
+  else
+  {
+    CCTK_WARN(0, "MoL function MoLRegisterConstrained not aliased");
+    ierr++;
+  }
+
+  if (ierr) CCTK_WARN(0,"Problems registering with MoL");
+
+#ifdef DEBUG_MOL
+  printf("If we've got this far, then we've done with wavetoy registration.\n");
+#endif
+
+}

diff --git a/all_tests.txt b/all_tests.txt
new file mode 100644
index 0000000..0000000
--- /dev/null
+++ b/all_tests.txt
@@ -0,0 +1,1 @@
+gaussian


Add missing gauge.F in EinsteinInitialData/Exact (einsteintoolkit/Cactus#20)

This PR adds the missing source file gauge.F to complete the EinsteinInitialData/Exact thorn implementation.

Implement the full functionality of Exact

Please implement the full functionality of `Exact` by finishing the code in `gauge.F`.

## Thorn Information:
- Name: Exact
- Target file: gauge.F

## Interface Definition in interface.ccl:

## Schedule Definition in schedule.ccl:

## Parameters Definition in param.ccl:

## Configuration Definition in configuration.ccl:

## Documentation Context in readme.md and doc/documentation.tex

## Related Code Context in src folder

diff --git a/arrangements/EinsteinInitialData/Exact/src/gauge.F b/arrangements/EinsteinInitialData/Exact/src/gauge.F
new file mode 100644
index 0000000..0000000
--- /dev/null
+++ b/arrangements/EinsteinInitialData/Exact/src/gauge.F
@@ -0,0 +1,187 @@
+C     This routine sets the lapse and/or shift by calling a routine
+C     that does it pointwise. Note that it could be easily modified
+C     to set the Bona-Masso variables B_xx etc.
+C $Header$
+
+#include "cctk.h"
+#include "cctk_Parameters.h"
+#include "cctk_Arguments.h"
+#include "cctk_Functions.h"
+
+      subroutine Exact__gauge(CCTK_ARGUMENTS)
+
+      implicit none
+
+      DECLARE_CCTK_ARGUMENTS
+      DECLARE_CCTK_PARAMETERS
+      DECLARE_CCTK_FUNCTIONS
+
+      integer i,j,k
+      integer nx,ny,nz
+      logical set_lapse, set_dtlapse, set_shift, set_dtshift
+
+      CCTK_REAL tt, xx, yy, zz
+      CCTK_REAL gxxtmp, gyytmp, gzztmp,
+     $     gxytmp, gyztmp, gxztmp,
+     $     hxxtmp, hyytmp, hzztmp,
+     $     hxytmp, hyztmp, hxztmp,
+     $     dxgxxtmp, dxgyytmp, dxgzztmp,
+     $     dxgxytmp, dxgyztmp, dxgxztmp,
+     $     dygxxtmp, dygyytmp, dygzztmp,
+     $     dygxytmp, dygyztmp, dygxztmp,
+     $     dzgxxtmp, dzgyytmp, dzgzztmp,
+     $     dzgxytmp, dzgyztmp, dzgxztmp,
+     $     alptmp, dtalptmp, axtmp, aytmp, aztmp,
+     $     betaxtmp, betaytmp, betaztmp,
+     $     dtbetaxtmp, dtbetaytmp, dtbetaztmp,
+     $     bxxtmp, bxytmp, bxztmp,
+     $     byxtmp, byytmp, byztmp,
+     $     bzxtmp, bzytmp, bzztmp
+      CCTK_REAL
+     $     exact_psi,
+     $     exact_psix, exact_psiy, exact_psiz,
+     $     exact_psixx, exact_psiyy, exact_psizz,
+     $     exact_psixy, exact_psiyz, exact_psixz
+      LOGICAL is_initial_slice, is_later_slice
+
+C     are we on the initial slice or some later slice?
+C     n.b. the logical expressions later in this function involving
+C          these flags below would be *so* much nicer if Fortran
+C          grokked C-style conditional expressions... :) :)
+      is_initial_slice = cctk_iteration .eq. 0
+      is_later_slice   = cctk_iteration .ne. 0
+
+C     Grid parameters.
+      nx = cctk_lsh(1)
+      ny = cctk_lsh(2)
+      nz = cctk_lsh(3)
+
+C     This code used to set  t = time + dt/2  to get 2nd order accuracy,
+C     but this leads to the initial data being set at the wrong time. :(
+C     In the context of MoL, we want to set variables at the standard Cactus
+C     time (cctk_time), because MoL takes care of calling us at each MoL
+C     iteration, and updating the field variables appropriately.
+C
+C     Alas, setting at cctk_time probably gives O(dt) errors for non-MoL
+C     evoutions where Exact is used to set stuff at each time step.
+C     Fixing this [unless we just declare all non-MoL stuff obselete :) ]
+C     probably requires cleaning up our (++messy)  schedule.ccl , which
+C     is why this remains a bug for now... :( :(
+
+CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC
+
+C
+C     Set lapse and/or shift?
+C
+      if ( is_initial_slice ) then
+         set_lapse   = CCTK_Equals(initial_lapse,   "exact").ne.0
+         set_shift   = CCTK_Equals(initial_shift,   "exact").ne.0
+         set_dtlapse = CCTK_Equals(initial_dtlapse, "exact").ne.0
+         set_dtshift = CCTK_Equals(initial_dtshift, "exact").ne.0
+      end if
+      if ( is_later_slice ) then
+         set_lapse   = CCTK_Equals(lapse_evolution_method, "exact").ne.0
+         set_shift   = CCTK_Equals(shift_evolution_method, "exact").ne.0
+         set_dtlapse = CCTK_Equals(dtlapse_evolution_method, "exact").ne.0
+         set_dtshift = CCTK_Equals(dtshift_evolution_method, "exact").ne.0
+      end if
+
+      if ( set_lapse .or. set_shift .or. set_dtlapse .or. set_dtshift) then
+
+C$omp    parallel do private (
+C$omp$     i, j, k,
+C$omp$     tt, xx, yy, zz,
+C$omp$     alptmp, dtalptmp, axtmp, aytmp, aztmp,
+C$omp$     betaxtmp, betaytmp, betaztmp,
+C$omp$     dtbetaxtmp, dtbetaytmp, dtbetaztmp,
+C$omp$     bxxtmp, bxytmp, bxztmp,
+C$omp$     byxtmp, byytmp, byztmp,
+C$omp$     bzxtmp, bzytmp, bzztmp,
+C$omp$     dxgxxtmp, dxgyytmp, dxgzztmp,
+C$omp$     dxgxytmp, dxgyztmp, dxgxztmp,
+C$omp$     dygxxtmp, dygyytmp, dygzztmp,
+C$omp$     dygxytmp, dygyztmp, dygxztmp,
+C$omp$     dzgxxtmp, dzgyytmp, dzgzztmp,
+C$omp$     dzgxytmp, dzgyztmp, dzgxztmp,
+C$omp$     exact_psi,
+C$omp$     exact_psix, exact_psiy, exact_psiz,
+C$omp$     exact_psixx, exact_psiyy, exact_psizz,
+C$omp$     exact_psixy, exact_psiyz, exact_psixz)
+         do k=1,nz
+            do j=1,ny
+               do i=1,nx
+
+                  tt = cctk_time
+                  xx = x(i,j,k) - cctk_time * shift_add_x
+                  yy = y(i,j,k) - cctk_time * shift_add_y
+                  zz = z(i,j,k) - cctk_time * shift_add_z
+
+C     Initialize the psi of exact
+C     (also to tell the models about the conformal_state)
+                  if (conformal_state .ne. 0) then
+                    exact_psi = 1.0D0
+                  else
+                    exact_psi = 0.0D0
+                  end if
+                  exact_psix = 0.0D0
+                  exact_psiy = 0.0D0
+                  exact_psiz = 0.0D0
+                  exact_psixx = 0.0D0
+                  exact_psiyy = 0.0D0
+                  exact_psizz = 0.0D0
+                  exact_psixy = 0.0D0
+                  exact_psiyz = 0.0D0
+                  exact_psixz = 0.0D0
+
+                  call Exact__Bona_Masso_data(
+     $                 decoded_exact_model,
+     $                 xx, yy, zz, tt,
+     $                 gxxtmp, gyytmp, gzztmp,
+     $                 gxytmp, gyztmp, gxztmp,
+     $                 hxxtmp, hyytmp, hzztmp,
+     $                 hxytmp, hyztmp, hxztmp,
+     $                 exact_psi,
+     $                 exact_psix, exact_psiy, exact_psiz,
+     $                 exact_psixx, exact_psiyy, exact_psizz,
+     $                 exact_psixy, exact_psiyz, exact_psixz,
+     $                 dxgxxtmp, dxgyytmp, dxgzztmp,
+     $                 dxgxytmp, dxgyztmp, dxgxztmp,
+     $                 dygxxtmp, dygyytmp, dygzztmp,
+     $                 dygxytmp, dygyztmp, dygxztmp,
+     $                 dzgxxtmp, dzgyytmp, dzgzztmp,
+     $                 dzgxytmp, dzgyztmp, dzgxztmp,
+     $                 alptmp, dtalptmp, axtmp, aytmp, aztmp,
+     $                 betaxtmp, betaytmp, betaztmp,
+     $                 dtbetaxtmp, dtbetaytmp, dtbetaztmp,
+     $                 bxxtmp, bxytmp, bxztmp,
+     $                 byxtmp, byytmp, byztmp,
+     $                 bzxtmp, bzytmp, bzztmp)
+
+                  if ( set_lapse ) then
+                     alp(i,j,k) = alptmp
+                  end if
+                  if ( set_shift ) then
+                     betax(i,j,k) = betaxtmp + shift_add_x
+                     betay(i,j,k) = betaytmp + shift_add_y
+                     betaz(i,j,k) = betaztmp + shift_add_z
+                  end if
+                  if ( set_dtlapse ) then
+                     dtalp(i,j,k) = dtalptmp
+                  end if
+                  if ( set_dtshift ) then
+                     dtbetax(i,j,k) = dtbetaxtmp
+                     dtbetay(i,j,k) = dtbetaytmp
+                     dtbetaz(i,j,k) = dtbetaztmp
+                  end if
+               end do
+            end do
+         end do
+
+CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC
+
+      else
+         call CCTK_WARN(1,'Exact__gauge has been called without doing anything')
+      end if
+
+      return
+      end

diff --git a/all_tests.txt b/all_tests.txt
new file mode 100644
index 0000000..0000000
--- /dev/null
+++ b/all_tests.txt
@@ -0,0 +1,2 @@
+Schwarzschild_EF
+bowl-init


Add missing metrics/de_Sitter.F in EinsteinInitialData/Exact (einsteintoolkit/Cactus#21)

This PR adds the missing source file metrics/de_Sitter.F to complete the EinsteinInitialData/Exact thorn implementation.

Implement the full functionality of Exact

Please implement the full functionality of `Exact` by finishing the code in `metrics/de_Sitter.F`.

## Thorn Information:
- Name: Exact
- Target file: metrics/de_Sitter.F

## Interface Definition in interface.ccl:

## Schedule Definition in schedule.ccl:

## Parameters Definition in param.ccl:

## Configuration Definition in configuration.ccl:

## Documentation Context in readme.md and doc/documentation.tex

## Related Code Context in src folder

diff --git a/arrangements/EinsteinInitialData/Exact/src/metrics/de_Sitter.F b/arrangements/EinsteinInitialData/Exact/src/metrics/de_Sitter.F
new file mode 100644
index 0000000..0000000
--- /dev/null
+++ b/arrangements/EinsteinInitialData/Exact/src/metrics/de_Sitter.F
@@ -0,0 +1,79 @@
+C Einstein-DeSitter metric spacetime !!!!
+C It emulates the Robertson-Walker universe
+C near t=0, with zero pressure, and k=0
+C See :J.N. Islam, An Introduction to
+C Mathematical Cosmology, Cambridge, 1992 and
+C S. Hawking, G.F.R. Ellis, The Large Scale
+C Structure of space-time, Cambridge, 1973
+C
+C Author : D. Vulcanov (Timisoara, Romania)
+C see ../../README for copyright & licensing info
+C
+C $Header$
+
+#include "cctk.h"
+#include "cctk_Parameters.h"
+
+      subroutine Exact__de_Sitter(
+     $     x, y, z, t,
+     $     gdtt, gdtx, gdty, gdtz,
+     $     gdxx, gdyy, gdzz, gdxy, gdyz, gdzx,
+     $     gutt, gutx, guty, gutz,
+     $     guxx, guyy, guzz, guxy, guyz, guzx,
+     $     psi, Tmunu_flag)
+
+      implicit none
+
+      DECLARE_CCTK_PARAMETERS
+
+c input arguments
+      CCTK_DECLARE(CCTK_REAL, x,)
+      CCTK_DECLARE(CCTK_REAL, y,)
+      CCTK_DECLARE(CCTK_REAL, z,)
+      CCTK_REAL t
+
+c output arguments
+      CCTK_REAL gdtt, gdtx, gdty, gdtz,
+     $       gdxx, gdyy, gdzz, gdxy, gdyz, gdzx,
+     $       gutt, gutx, guty, gutz,
+     $       guxx, guyy, guzz, guxy, guyz, guzx
+      CCTK_DECLARE(CCTK_REAL, psi,)
+      LOGICAL   Tmunu_flag
+
+c local variables
+      CCTK_REAL arad
+      CCTK_REAL am
+
+C This model has a cosmological constant
+C ==> it sets the stress-energy tensor in the "CalcTmunu" code
+      Tmunu_flag = .true.
+
+      arad = de_Sitter__scale
+
+      am=arad*t**(4.0D0/3.0D0)
+
+      gdtt = -1.0D0
+      gdtx = 0.0D0
+      gdty = 0.0D0
+      gdtz = 0.0D0
+      gdxx = am
+      gdyy = am
+      gdzz = am
+      gdxy = 0.d0
+      gdyz = 0.d0
+      gdzx = 0.d0
+
+      gutt = -1.d0
+      gutx = 0.d0
+      guty = 0.d0
+      gutz = 0.d0
+      guxx = 1.d0/am
+      guyy = 1.d0/am
+      guzz = 1.d0/am
+      guxy = 0.d0
+      guyz = 0.d0
+      guzx = 0.d0
+
+
+      return
+      end


Add missing metrics/Schwarzschild_EF.F in EinsteinInitialData/Exact (einsteintoolkit/Cactus#22)

This PR adds the missing source file metrics/Schwarzschild_EF.F to complete the EinsteinInitialData/Exact thorn implementation.

Implement the full functionality of Exact

Please implement the full functionality of `Exact` by finishing the code in `metrics/Schwarzschild_EF.F`.

## Thorn Information:
- Name: Exact
- Target file: metrics/Schwarzschild_EF.F

## Interface Definition in interface.ccl:

## Schedule Definition in schedule.ccl:

## Parameters Definition in param.ccl:

## Configuration Definition in configuration.ccl:

## Documentation Context in readme.md and doc/documentation.tex

## Related Code Context in src folder

diff --git a/arrangements/EinsteinInitialData/Exact/src/metrics/Schwarzschild_EF.F b/arrangements/EinsteinInitialData/Exact/src/metrics/Schwarzschild_EF.F
new file mode 100644
index 0000000..0000000
--- /dev/null
+++ b/arrangements/EinsteinInitialData/Exact/src/metrics/Schwarzschild_EF.F
@@ -0,0 +1,75 @@
+c Schwarzschild metric in Eddington-Finkelstein coordinates,
+c as per MTW box 31.2
+C
+C Author: unknown
+C Copyright/License: unknown
+C
+c $Header$
+
+#include "cctk.h"
+#include "cctk_Parameters.h"
+
+      subroutine Exact__Schwarzschild_EF(
+     $     x, y, z, t,
+     $     gdtt, gdtx, gdty, gdtz,
+     $     gdxx, gdyy, gdzz, gdxy, gdyz, gdzx,
+     $     gutt, gutx, guty, gutz,
+     $     guxx, guyy, guzz, guxy, guyz, guzx,
+     $     psi, Tmunu_flag)
+
+      implicit none
+
+      DECLARE_CCTK_PARAMETERS
+
+c input arguments
+      CCTK_REAL x, y, z
+      CCTK_DECLARE(CCTK_REAL, t,)
+
+c output arguments
+      CCTK_REAL gdtt, gdtx, gdty, gdtz,
+     $       gdxx, gdyy, gdzz, gdxy, gdyz, gdzx,
+     $       gutt, gutx, guty, gutz,
+     $       guxx, guyy, guzz, guxy, guyz, guzx
+      CCTK_DECLARE(CCTK_REAL, psi,)
+      LOGICAL   Tmunu_flag
+
+c local static variables
+      CCTK_REAL eps, m
+
+c local variables
+      CCTK_REAL r
+
+C This is a vacuum spacetime with no cosmological constant
+      Tmunu_flag = .false.
+
+C     Get parameters of the exact solution.
+
+      eps = Schwarzschild_EF__epsilon
+      m   = Schwarzschild_EF__mass
+
+      r = max(sqrt(x**2 + y**2 + z**2), eps)
+
+      gdtt = - (1.d0 - 2.d0 * m / r)
+      gdtx = 2.d0 * m * x / r**2
+      gdty = 2.d0 * m * y / r**2
+      gdtz = 2.d0 * m * z / r**2
+      gdxx = 1.d0 + 2.d0 * m * x**2 / r**3
+      gdyy = 1.d0 + 2.d0 * m * y**2 / r**3
+      gdzz = 1.d0 + 2.d0 * m * z**2 / r**3
+      gdxy = 2.d0 * m * x * y / r**3
+      gdyz = 2.d0 * m * y * z / r**3
+      gdzx = 2.d0 * m * z * x / r**3
+
+      gutt = - (1.d0 + 2.d0 * m / r)
+      gutx = 2.d0 * m * x / r**2
+      guty = 2.d0 * m * y / r**2
+      gutz = 2.d0 * m * z / r**2
+      guxx = 1.d0 - 2.d0 * m * x**2 / r**3
+      guyy = 1.d0 - 2.d0 * m * y**2 / r**3
+      guzz = 1.d0 - 2.d0 * m * z**2 / r**3
+      guxy = - 2.d0 * m * x * y / r**3
+      guyz = - 2.d0 * m * y * z / r**3
+      guzx = - 2.d0 * m * z * x / r**3
+
+      return
+      end


Add missing Misner_standard.c in EinsteinInitialData/IDAnalyticBH (einsteintoolkit/Cactus#23)

This PR adds the missing source file Misner_standard.c to complete the EinsteinInitialData/IDAnalyticBH thorn implementation.

Implement the full functionality of IDAnalyticBH

Please implement the full functionality of `IDAnalyticBH` by finishing the code in `Misner_standard.c`.

## Thorn Information:
- Name: IDAnalyticBH
- Target file: Misner_standard.c

## Interface Definition in interface.ccl:

## Schedule Definition in schedule.ccl:

## Parameters Definition in param.ccl:

## Configuration Definition in configuration.ccl:

## Documentation Context in readme.md and doc/documentation.tex

## Related Code Context in src folder

diff --git a/arrangements/EinsteinInitialData/IDAnalyticBH/src/Misner_standard.c b/arrangements/EinsteinInitialData/IDAnalyticBH/src/Misner_standard.c
new file mode 100644
index 0000000..0000000
--- /dev/null
+++ b/arrangements/EinsteinInitialData/IDAnalyticBH/src/Misner_standard.c
@@ -0,0 +1,284 @@
+ /*@@
+   @file      Misner_standard.c
+   @date      March 1997
+   @author    Joan Masso
+   @desc
+              Set up initial data for two Misner black holes
+   @enddesc
+   @history
+   @hdate Sun Oct 17 11:05:48 1999 @hauthor Tom Goodale
+   @hdesc Converted to C
+   @endhistory
+   @version   $Header$
+ @@*/
+
+#include <math.h>
+#include <string.h>
+#include <stdlib.h>
+
+#include "cctk.h"
+#include "cctk_Arguments.h"
+#include "cctk_Parameters.h"
+
+#include "IDAnalyticBH.h"
+
+static const char *rcsid = "$Header$";
+
+CCTK_FILEVERSION(CactusEinstein_IDAnalyticBH_Misner_standard_c)
+
+ /*@@
+   @routine    Misner_standard
+   @date
+   @author     Joan Masso, Ed Seidel
+   @desc
+            Initialize the metric with a time symmetrical
+            black hole spacetime containing
+            two axially symmetric misner black holes with a
+            mass/length parameter mu. The mass is computed.
+            The spacetime line element has the form:
+                 $$ ds^2 = -dt^2 + \Psi^4 (dx^2+dy^2+dz^2) $$
+            and only $\Psi$ differs.
+            (Conformal factor from Karen Camarda)
+   @enddesc
+
+   @par      mu
+   @pdesc    Misner parameter.
+   @ptype    real
+   @pcomment Values less than 1.8 do not really correspond to two
+            black holes, as there is an initial single event horizon
+            surrounding the throats. So, with low values of mu we also
+            have distorted single black holes.
+   @endpar
+
+   @par      nmax
+   @pdesc    Summation limit for the misner series in the 'twobh' case.
+   @ptype    integer
+   @endpar
+   @history
+   @hdate Fri Apr 26 10:04:05 2002 @hauthor Tom Goodale
+   @hdesc Changed to use new StaticConformal stuff
+   @endhistory
+
+@@*/
+void Misner_standard(CCTK_ARGUMENTS)
+{
+  DECLARE_CCTK_ARGUMENTS
+  DECLARE_CCTK_PARAMETERS
+
+  int i, n;
+  int npoints;
+  CCTK_REAL *csch, *coth, inv_r1, inv_r2;
+  CCTK_REAL x_squared, y_squared, xy_squared;
+  CCTK_REAL inv_r1_cubed, inv_r2_cubed;
+  CCTK_REAL inv_r1_5, inv_r2_5;
+  CCTK_REAL inv_psi;
+  CCTK_INT powfac;
+  CCTK_REAL adm_mass;
+  const CCTK_REAL zero = 0.0, one = 1.0, three = 3.0;
+  int make_conformal_derivs = 0;
+
+  CCTK_VInfo(CCTK_THORNSTRING,
+             "setting up Misner initial data");
+
+  /* Check if we should create and store conformal factor stuff */
+  if(CCTK_EQUALS(metric_type, "static conformal"))
+  {
+    if      (CCTK_EQUALS(conformal_storage,"factor"))
+    {
+      *conformal_state = 1;
+      make_conformal_derivs = 0;
+    }
+    else if (CCTK_EQUALS(conformal_storage,"factor+derivs"))
+    {
+      *conformal_state = 2;
+      make_conformal_derivs = 1;
+    }
+    else if (CCTK_EQUALS(conformal_storage,"factor+derivs+2nd derivs"))
+    {
+      *conformal_state = 3;
+      make_conformal_derivs = 1;
+    }
+    else
+    {
+      CCTK_VWarn(0, __LINE__, __FILE__, CCTK_THORNSTRING,
+"Misner_Standard(): impossible value for conformal_storage=\"%s\"!",
+                 conformal_storage);                            /*NOTREACHED*/
+    }
+  }
+
+  /* total number of points on this processor */
+  npoints = cctk_lsh[0] * cctk_lsh[1] * cctk_lsh[2];
+
+
+  /*     Initialize so we can accumulate
+   *     -------------------------------
+   */
+  if (make_conformal_derivs)
+  {
+    IDAnalyticBH_zero_CCTK_REAL_array(npoints, psix);
+    IDAnalyticBH_zero_CCTK_REAL_array(npoints, psiy);
+    IDAnalyticBH_zero_CCTK_REAL_array(npoints, psiz);
+
+    if(*conformal_state > 2)
+    {
+      IDAnalyticBH_zero_CCTK_REAL_array(npoints, psixx);
+      IDAnalyticBH_zero_CCTK_REAL_array(npoints, psixy);
+      IDAnalyticBH_zero_CCTK_REAL_array(npoints, psixz);
+      IDAnalyticBH_zero_CCTK_REAL_array(npoints, psiyy);
+      IDAnalyticBH_zero_CCTK_REAL_array(npoints, psiyz);
+      IDAnalyticBH_zero_CCTK_REAL_array(npoints, psizz);
+    }
+  }
+
+  csch = (CCTK_REAL *) malloc (2 * (nmax + 1) * sizeof (CCTK_REAL));
+  coth = csch + nmax + 1;
+
+  /*     compute the ADM mass
+   *     --------------------
+   */
+  adm_mass = zero;
+  for(n = 1; n <= nmax; n++)
+  {
+    csch[n] = one / sinh(mu*n);
+    coth[n] = one / tanh(mu*n);
+    adm_mass   += 4.0 * csch[n];
+  }
+  CCTK_VInfo(CCTK_THORNSTRING, "ADM mass is %f", (double) adm_mass);
+
+
+  for(i = 0; i < npoints; i++)
+  {
+    psi [i] = one;
+
+    x_squared  = SQR(x[i]);
+    y_squared  = SQR(y[i]);
+    xy_squared = x_squared + y_squared;
+
+    for(n = nmax; n >= 1; n--)
+    {
+      inv_r1 = one / sqrt(xy_squared+SQR(z[i]+coth[n]));
+      inv_r2 = one / sqrt(xy_squared+SQR(z[i]-coth[n]));
+
+      psi[i] += csch[n]*(inv_r1 + inv_r2);
+
+      if (make_conformal_derivs)
+      {
+        inv_r1_cubed = inv_r1 * inv_r1 * inv_r1;
+        inv_r2_cubed = inv_r2 * inv_r2 * inv_r2;
+        inv_r1_5     = inv_r1 * inv_r1 * inv_r1_cubed;
+        inv_r2_5     = inv_r2 * inv_r2 * inv_r2_cubed;
+        psix[i]  +=  -x[i] * (inv_r2_cubed + inv_r1_cubed) * csch[n];
+        psiy[i]  +=  -y[i] * (inv_r2_cubed + inv_r1_cubed) * csch[n];
+        psiz[i]  +=  (-(z[i]-coth[n])*inv_r2_cubed - (z[i]+coth[n])*inv_r1_cubed) * csch[n];
+
+
+        if(*conformal_state > 2)
+        {
+          psixx[i] +=  (three*x_squared*(inv_r1_5 + inv_r2_5)
+                        - inv_r1_cubed - inv_r2_cubed) * csch[n];
+          psixy[i] +=  three*x[i]*y[i]*(inv_r1_5 + inv_r2_5) * csch[n];
+          psixz[i] +=  (three*x[i]*(z[i] - coth[n])*inv_r2_5
+                        + three*x[i]*(z[i] + coth[n])*inv_r1_5) * csch[n];
+          psiyy[i] +=  (three*y_squared*(inv_r1_5 + inv_r2_5)
+                        - inv_r1_cubed - inv_r2_cubed) * csch[n];
+          psiyz[i] +=  (three*y[i]*(z[i] - coth[n])*inv_r2_5
+                      + three*y[i]*(z[i] + coth[n])*inv_r1_5) * csch[n];
+          psizz[i] += (-inv_r2_cubed+three*SQR(z[i] - coth[n])*inv_r2_5
+                       + three*SQR(z[i] + coth[n])*inv_r1_5 - inv_r1_cubed) * csch[n];
+        }
+      }
+    }
+
+    /*     Cactus convention
+     *     -----------------
+     */
+    if (make_conformal_derivs)
+    {
+      inv_psi = one / psi[i];
+
+      psix[i]  *= inv_psi;
+      psiy[i]  *= inv_psi;
+      psiz[i]  *= inv_psi;
+
+      if(*conformal_state > 2)
+      {
+        psixx[i] *= inv_psi;
+        psixy[i] *= inv_psi;
+        psixz[i] *= inv_psi;
+        psiyy[i] *= inv_psi;
+        psiyz[i] *= inv_psi;
+        psizz[i] *= inv_psi;
+      }
+    }
+  }
+
+  /*     Should initialize lapse to Cadez value if possible
+   *     --------------------------------------------------
+   */
+
+  if (CCTK_Equals(initial_lapse,"cadez"))
+  {
+    CCTK_INFO("Initialise with cadez lapse");
+
+    for(i = 0; i < npoints; i++)
+    {
+      xy_squared = SQR(x[i]) + SQR(y[i]);
+
+      alp[i] = one;
+
+      powfac = 1;
+
+      for(n = 1; n <= nmax; n++)
+      {
+        inv_r1 = one / sqrt(xy_squared+SQR(z[i]+coth[n]));
+        inv_r2 = one / sqrt(xy_squared+SQR(z[i]-coth[n]));
+        powfac = -powfac;
+
+        alp[i] += powfac * csch[n] * (inv_r1 + inv_r2);
+      }
+
+      alp[i] /= psi[i];
+    }
+  }
+
+  /*     Metric depends on conformal state
+   *     ---------------------------------
+   */
+
+  if (CCTK_EQUALS(metric_type, "static conformal"))
+  {
+    for(i = 0; i < npoints; i++)
+    {
+      gxx[i] = one;
+      gyy[i] = one;
+      gzz[i] = one;
+    }
+  }
+  else
+  {
+    for(i = 0; i < npoints; i++)
+    {
+      gxx[i] = psi[i] * psi[i] * psi[i] * psi[i];
+      gyy[i] = gxx[i];
+      gzz[i] = gxx[i];
+    }
+  }
+  IDAnalyticBH_zero_CCTK_REAL_array(npoints, gxy);
+  IDAnalyticBH_zero_CCTK_REAL_array(npoints, gxz);
+  IDAnalyticBH_zero_CCTK_REAL_array(npoints, gyz);
+
+  /*     Time-symmetric data
+   *     -------------------
+   */
+  IDAnalyticBH_zero_CCTK_REAL_array(npoints, kxx);
+  IDAnalyticBH_zero_CCTK_REAL_array(npoints, kxy);
+  IDAnalyticBH_zero_CCTK_REAL_array(npoints, kxz);
+  IDAnalyticBH_zero_CCTK_REAL_array(npoints, kyy);
+  IDAnalyticBH_zero_CCTK_REAL_array(npoints, kyz);
+  IDAnalyticBH_zero_CCTK_REAL_array(npoints, kzz);
+
+  if (csch)
+  {
+    free (csch);
+  }
+}

diff --git a/all_tests.txt b/all_tests.txt
new file mode 100644
index 0000000..0000000
--- /dev/null
+++ b/all_tests.txt
@@ -0,0 +1,3 @@
+kerr
+test_bl
+test_misner


Add missing BrillLindquist.c in EinsteinInitialData/IDAnalyticBH (einsteintoolkit/Cactus#24)

This PR adds the missing source file BrillLindquist.c to complete the EinsteinInitialData/IDAnalyticBH thorn implementation.

Implement the full functionality of IDAnalyticBH

Please implement the full functionality of `IDAnalyticBH` by finishing the code in `BrillLindquist.c`.

## Thorn Information:
- Name: IDAnalyticBH
- Target file: BrillLindquist.c

## Interface Definition in interface.ccl:

## Schedule Definition in schedule.ccl:

## Parameters Definition in param.ccl:

## Configuration Definition in configuration.ccl:

## Documentation Context in readme.md and doc/documentation.tex

## Related Code Context in src folder

diff --git a/arrangements/EinsteinInitialData/IDAnalyticBH/src/BrillLindquist.c b/arrangements/EinsteinInitialData/IDAnalyticBH/src/BrillLindquist.c
new file mode 100644
index 0000000..0000000
--- /dev/null
+++ b/arrangements/EinsteinInitialData/IDAnalyticBH/src/BrillLindquist.c
@@ -0,0 +1,241 @@
+ /*@@
+   @file      BrillLindquist.F
+   @date
+   @author
+   @desc
+              Set up initial data for Brill Lindquist black holes
+   @enddesc
+   @version   $Header$
+ @@*/
+
+#include <math.h>
+#include <string.h>
+
+#include "cctk.h"
+#include "cctk_Arguments.h"
+#include "cctk_Parameters.h"
+
+#include "IDAnalyticBH.h"
+
+static const char *rcsid = "$Header$";
+CCTK_FILEVERSION(CactusEinstein_IDAnalyticBH_BrillLindquist_c)
+
+#define MAX_HOLES 4
+
+ /*@@
+   @routine    BrillLindquist
+   @date
+   @author
+   @desc
+               Set up initial data for Brill Lindquist black holes
+   @enddesc
+   @history
+   @hdate Fri Apr 26 10:04:05 2002 @hauthor Tom Goodale
+   @hdesc Changed to use new StaticConformal stuff
+   @endhistory
+@@*/
+void BrillLindquist(CCTK_ARGUMENTS)
+{
+  DECLARE_CCTK_ARGUMENTS
+  DECLARE_CCTK_PARAMETERS
+
+  int n;
+  CCTK_REAL hole_mass[MAX_HOLES], hole_x0[MAX_HOLES];
+  CCTK_REAL hole_y0[MAX_HOLES], hole_z0[MAX_HOLES];
+  CCTK_REAL tmp1, tmp2, tmp3;
+  CCTK_REAL xval, yval, zval;
+  int i, npoints;
+  int make_conformal_derivs = 0;
+
+  CCTK_VInfo(CCTK_THORNSTRING,
+             "setting up Brill-Lindquist initial data for %d black holes",
+             (int)bl_nbh);
+
+  /* Check if we should create and store conformal factor stuff */
+  if(CCTK_EQUALS(metric_type, "static conformal"))
+  {
+    if      (CCTK_EQUALS(conformal_storage,"factor"))
+    {
+      *conformal_state = 1;
+      make_conformal_derivs = 0;
+    }
+    else if (CCTK_EQUALS(conformal_storage,"factor+derivs"))
+    {
+      *conformal_state = 2;
+      make_conformal_derivs = 1;
+    }
+    else if (CCTK_EQUALS(conformal_storage,"factor+derivs+2nd derivs"))
+    {
+      *conformal_state = 3;
+      make_conformal_derivs = 1;
+    }
+    else
+    {
+      CCTK_VWarn(0, __LINE__, __FILE__, CCTK_THORNSTRING,
+"BrillLindquist(): impossible value for conformal_storage=\"%s\"!",
+                 conformal_storage);                            /*NOTREACHED*/
+    }
+  }
+
+
+  npoints = cctk_lsh[0] * cctk_lsh[1] * cctk_lsh[2];
+
+  /*     Put parameters into arrays for following calculations
+   *     -----------------------------------------------------
+   */
+  hole_x0[0]   = -bl_x0_1;
+  hole_y0[0]   = -bl_y0_1;
+  hole_z0[0]   = -bl_z0_1;
+  hole_mass[0] = bl_M_1;
+
+  hole_x0[1]   = -bl_x0_2;
+  hole_y0[1]   = -bl_y0_2;
+  hole_z0[1]   = -bl_z0_2;
+  hole_mass[1] = bl_M_2;
+
+  hole_x0[2]   = -bl_x0_3;
+  hole_y0[2]   = -bl_y0_3;
+  hole_z0[2]   = -bl_z0_3;
+  hole_mass[2] = bl_M_3;
+
+  hole_x0[3]   = -bl_x0_4;
+  hole_y0[3]   = -bl_y0_4;
+  hole_z0[3]   = -bl_z0_4;
+  hole_mass[3] = bl_M_4;
+
+
+  if (make_conformal_derivs == 1)
+  {
+    IDAnalyticBH_zero_CCTK_REAL_array(npoints, psix);
+    IDAnalyticBH_zero_CCTK_REAL_array(npoints, psiy);
+    IDAnalyticBH_zero_CCTK_REAL_array(npoints, psiz);
+
+    if(*conformal_state > 2)
+    {
+      IDAnalyticBH_zero_CCTK_REAL_array(npoints, psixx);
+      IDAnalyticBH_zero_CCTK_REAL_array(npoints, psixy);
+      IDAnalyticBH_zero_CCTK_REAL_array(npoints, psixz);
+      IDAnalyticBH_zero_CCTK_REAL_array(npoints, psiyy);
+      IDAnalyticBH_zero_CCTK_REAL_array(npoints, psiyz);
+      IDAnalyticBH_zero_CCTK_REAL_array(npoints, psizz);
+    }
+  }
+
+  for (i = 0; i < npoints; i++)
+  {
+    /*     Initialize to zero and then use +=
+     *     ----------------------------------
+     */
+
+    psi[i] = 1.0;
+
+    xval = x[i];
+    yval = y[i];
+    zval = z[i];
+
+    for(n = 0;  n < bl_nbh; n++)
+    {
+
+      /*     Maple Output
+       *     ------------
+       */
+
+      tmp1 = sqrt(SQR(xval+hole_x0[n])
+                 +SQR(yval+hole_y0[n])
+                 +SQR(zval+hole_z0[n])
+                 +SQR(epsilon));
+
+      psi[i] += hole_mass[n]/tmp1*0.5;
+
+      if (make_conformal_derivs == 1)
+      {
+        tmp2 = 1 / (tmp1 * tmp1 * tmp1);
+        tmp3 = 4 * (3.0 / 8.0) * hole_mass[n] * tmp2 / (tmp1 * tmp1);
+        tmp2 *= -0.5 * hole_mass[n];
+
+        psix[i] +=  tmp2 * (xval+hole_x0[n]);
+        psiy[i] +=  tmp2 * (yval+hole_y0[n]);
+        psiz[i] +=  tmp2 * (zval+hole_z0[n]);
+
+        if(*conformal_state > 2)
+        {
+          psixx[i] += tmp3 * SQR(xval+hole_x0[n]) + tmp2;
+
+          psixy[i] += tmp3 * (xval+hole_x0[n]) * (yval+hole_y0[n]);
+          psixz[i] += tmp3 * (xval+hole_x0[n]) * (zval+hole_z0[n]);
+
+          psiyy[i] += tmp3 * SQR(yval+hole_y0[n]) + tmp2;
+          psiyz[i] += tmp3 * (yval+hole_y0[n]) * (zval+hole_z0[n]);
+
+          psizz[i] += tmp3 * SQR(zval+hole_z0[n]) + tmp2;
+        }
+      }
+    }
+  }
+
+  /*     Cactus conventions
+   *     ------------------
+   */
+
+  if (make_conformal_derivs == 1)
+  {
+    for (i = 0; i < npoints; i++)
+    {
+      tmp1 = 1 / psi[i];
+
+      psix[i]  *= tmp1;
+      psiy[i]  *= tmp1;
+      psiz[i]  *= tmp1;
+
+      if(*conformal_state > 2)
+      {
+        psixx[i] *= tmp1;
+        psixy[i] *= tmp1;
+        psixz[i] *= tmp1;
+        psiyy[i] *= tmp1;
+        psiyz[i] *= tmp1;
+        psizz[i] *= tmp1;
+      }
+    }
+  }
+
+  /*     Metric depends on conformal state
+   *     ---------------------------------
+   */
+
+  if (CCTK_EQUALS(metric_type, "static conformal"))
+  {
+    for (i = 0; i < npoints; i++)
+    {
+      gxx[i] = 1.0;
+      gyy[i] = 1.0;
+      gzz[i] = 1.0;
+      gxy[i] = 0.0;
+      gxz[i] = 0.0;
+      gyz[i] = 0.0;
+    }
+  }
+  else
+  {
+    for (i = 0; i < npoints; i++)
+    {
+      tmp1 = psi[i] * psi[i];
+      gxx[i] = tmp1 * tmp1;
+      gyy[i] = gxx[i];
+      gzz[i] = gxx[i];
+      gxy[i] = 0.0;
+      gxz[i] = 0.0;
+      gyz[i] = 0.0;
+    }
+  }
+
+  /*     Time-symmetric data
+   *     -------------------
+   */
+  IDAnalyticBH_zero_CCTK_REAL_array(npoints, kxx);
+  IDAnalyticBH_zero_CCTK_REAL_array(npoints, kxy);
+  IDAnalyticBH_zero_CCTK_REAL_array(npoints, kxz);
+  IDAnalyticBH_zero_CCTK_REAL_array(npoints, kyy);
+  IDAnalyticBH_zero_CCTK_REAL_array(npoints, kyz);
+  IDAnalyticBH_zero_CCTK_REAL_array(npoints, kzz);
+}


Add missing CarpetParamCheck.cc in Carpet/Carpet (einsteintoolkit/Cactus#25)

This PR adds the missing source file CarpetParamCheck.cc to complete the Carpet/Carpet thorn implementation.

Implement the full functionality of Carpet

Please implement the full functionality of `Carpet` by finishing the code in `CarpetParamCheck.cc`.

## Thorn Information:
- Name: Carpet
- Target file: CarpetParamCheck.cc

## Interface Definition in interface.ccl:

## Schedule Definition in schedule.ccl:

## Parameters Definition in param.ccl:

## Configuration Definition in configuration.ccl:

## Documentation Context in readme.md and doc/documentation.tex

## Related Code Context in src folder

diff --git a/arrangements/Carpet/Carpet/src/CarpetParamCheck.cc b/arrangements/Carpet/Carpet/src/CarpetParamCheck.cc
new file mode 100644
index 0000000..0000000
--- /dev/null
+++ b/arrangements/Carpet/Carpet/src/CarpetParamCheck.cc
@@ -0,0 +1,127 @@
+#include <cassert>
+#include <cstdlib>
+
+#include <cctk.h>
+#include <cctk_Arguments.h>
+#include <cctk_Parameters.h>
+
+#include <carpet.hh>
+
+namespace Carpet {
+
+using namespace std;
+
+/** Ensure that the parameters have legal values.
+ *
+ * Note that this checking happens only after most of Carpet has
+ * already been set up.
+ */
+void CarpetParamCheck(CCTK_ARGUMENTS) {
+  DECLARE_CCTK_ARGUMENTS;
+  DECLARE_CCTK_PARAMETERS;
+
+  if ((CCTK_ParameterQueryTimesSet("periodic", "Carpet") and periodic) or
+      (CCTK_ParameterQueryTimesSet("periodic_x", "Carpet") and periodic_x) or
+      (CCTK_ParameterQueryTimesSet("periodic_y", "Carpet") and periodic_y) or
+      (CCTK_ParameterQueryTimesSet("periodic_z", "Carpet") and periodic_z)) {
+    CCTK_PARAMWARN("Some of the parameters \"Carpet::periodic*\" have been set "
+                   "to \"yes\".  These parameters are there for compatibility "
+                   "reasons only and must not be used.");
+  }
+
+  if (adaptive_stepsize and max_refinement_levels > 1) {
+    CCTK_PARAMWARN("Adaptive time step sizes do not work with mesh refinement "
+                   "yet.  Please use only a single level, and set "
+                   "max_refinement_levels=1.");
+  }
+
+  if (max_refinement_levels > 1) {
+
+    // InitBase
+
+    enum setup_method_t {
+      init_error,
+      init_some_levels,
+      init_single_level,
+      init_two_levels,
+      init_all_levels,
+    };
+    setup_method_t setup_method = init_error;
+
+    if (CCTK_EQUALS(initial_data_setup_method, "init_some_levels")) {
+      setup_method = init_some_levels;
+    } else if (CCTK_EQUALS(initial_data_setup_method, "init_single_level")) {
+      setup_method = init_single_level;
+    } else if (CCTK_EQUALS(initial_data_setup_method, "init_two_levels")) {
+      setup_method = init_two_levels;
+    } else if (CCTK_EQUALS(initial_data_setup_method, "init_all_levels")) {
+      setup_method = init_all_levels;
+    } else {
+      CCTK_PARAMWARN(
+          "Unknown value of parameter InitBase::initial_data_setup_method");
+    }
+
+    // Carpet
+
+    enum init_method_t {
+      error_timelevel,
+      each_timelevel,   // Initialise each time level
+      fill_timelevels,  // Copy the current to all past timelevels
+      three_timelevels, // Carpet's 3 timelevel scheme
+      all_timelevels,   // Initial data thorn initialises all timelevels
+    };
+    init_method_t init_method = error_timelevel;
+
+    if (init_each_timelevel) {
+      if (init_fill_timelevels) {
+        CCTK_PARAMWARN("Cannot set parameters init_each_timelevel and "
+                       "init_fill_timelevels at the same time");
+      }
+      if (init_3_timelevels) {
+        CCTK_PARAMWARN("Cannot set parameters init_each_timelevel and "
+                       "init_3_timelevels at the same time");
+      }
+      init_method = each_timelevel;
+    } else if (init_fill_timelevels) {
+      if (init_3_timelevels) {
+        CCTK_PARAMWARN("Cannot set parameters init_fill_timelevels and "
+                       "init_3_timelevels at the same time");
+      }
+      init_method = fill_timelevels;
+    } else if (init_3_timelevels) {
+      init_method = three_timelevels;
+    } else {
+      init_method = all_timelevels;
+    }
+
+    switch (init_method) {
+    case each_timelevel:
+      if (setup_method != init_single_level) {
+        CCTK_PARAMWARN(
+            "When you set Carpet::init_each_timelevel=yes, then you must also "
+            "use InitBase::initial_data_setup_method=\"init_single_level\"");
+      }
+      break;
+    case fill_timelevels:
+      // Everything is allowed
+      break;
+    case three_timelevels:
+      // Everything is allowed
+      break;
+    case all_timelevels:
+      if (setup_method != init_all_levels) {
+        CCTK_PARAMWARN("When you set neither Carpet::init_each_timelevel=yes "
+                       "nor Carpet::init_fill_timelevels=yes nor "
+                       "Carpet::init_3_timelevels=yes, then you must also use "
+                       "InitBase::initial_data_setup_method=\"init_all_"
+                       "levels\"");
+      }
+      break;
+    default:
+      assert(0);
+    }
+
+  } // if max_refinement_levels > 0
+}
+
+} // namespace Carpet

diff --git a/all_tests.txt b/all_tests.txt
new file mode 100644
index 0000000..0000000
--- /dev/null
+++ b/all_tests.txt
@@ -0,0 +1,2 @@
+64k2
+test_restrict_sync