Added atomSubset option to DCDReporter and XTCReporter (openmm/openmm#4741)

When saving a trajectory to a DCD or XTC file, you can specify a subset of atoms to write.  Fixes #4564.

Recommended approach to writing out a subset of atoms with XTCReporter/DCDReporter?

I was going through the docs about [XTCReporter](https://docs.openmm.org/latest/api-python/generated/openmm.app.xtcreporter.XTCReporter.html) and [DCDReporter](https://docs.openmm.org/latest/api-python/generated/openmm.app.dcdreporter.DCDReporter.html) and I realized there's not really a well-documented way to use these reporters to save out a subset of atoms (or maybe I'm missing it somewhere else?).

Is there a general recommended way to achieve this "writing out a subset of atoms" if we're writing out into XTC or DCD files? I'm imagining a basic scenario where it is known which atom indicies are desired to be saved, but the rest do not need to be written out. I'm currently envisioning having to write some kind of `subsetXTCReporter` or the like - would this be the best way to go about this?Just looking for any guidance here!

Addendum: You can do this subsetting of atoms using the [XTCReporter](https://github.com/mdtraj/mdtraj/blob/ccd0c874361e4e31127e3803d3670d08d57f7933/mdtraj/reporters/xtcreporter.py#L42-L68) and [DCDReporter](https://mdtraj.readthedocs.io/en/latest/api/generated/mdtraj.reporters.DCDReporter.html#mdtraj.reporters.DCDReporter) in MDTraj - I'm just wondering if there are alternative approaches now that OpenMM has it's own native XTCReporter

## IMPORTANT INSTRUCTIONS

1. If you edit a python file, you MUST run 'make PythonInstall' inside the build directory
2. If you edit a c++ file, you MUST run 'make -j4 && make install" inside the build directory

diff --git a/wrappers/python/openmm/app/dcdfile.py b/wrappers/python/openmm/app/dcdfile.py
index 53aeaafec6..335cdc0b79 100644
--- a/wrappers/python/openmm/app/dcdfile.py
+++ b/wrappers/python/openmm/app/dcdfile.py
@@ -88,14 +88,14 @@ def __init__(self, file, topology, dt, firstStep=0, interval=1, append=False):
             self._modelCount = struct.unpack('<i', file.read(4))[0]
             file.seek(268, os.SEEK_SET)
             numAtoms = struct.unpack('<i', file.read(4))[0]
-            if numAtoms != len(list(topology.atoms())):
+            if numAtoms != topology.getNumAtoms():
                 raise ValueError('Cannot append to a DCD file that contains a different number of atoms')
         else:
             header = struct.pack('<i4c9if', 84, b'C', b'O', b'R', b'D', 0, firstStep, interval, 0, 0, 0, 0, 0, 0, dt)
             header += struct.pack('<13i', boxFlag, 0, 0, 0, 0, 0, 0, 0, 0, 24, 84, 164, 2)
             header += struct.pack('<80s', b'Created by OpenMM')
             header += struct.pack('<80s', b'Created '+time.asctime(time.localtime(time.time())).encode('ascii'))
-            header += struct.pack('<4i', 164, 4, len(list(topology.atoms())), 4)
+            header += struct.pack('<4i', 164, 4, topology.getNumAtoms(), 4)
             file.write(header)

     def writeModel(self, positions, unitCellDimensions=None, periodicBoxVectors=None):
diff --git a/wrappers/python/openmm/app/dcdreporter.py b/wrappers/python/openmm/app/dcdreporter.py
index 09b1d9439a..d09d5c149c 100644
--- a/wrappers/python/openmm/app/dcdreporter.py
+++ b/wrappers/python/openmm/app/dcdreporter.py
@@ -6,7 +6,7 @@
 Biological Structures at Stanford, funded under the NIH Roadmap for
 Medical Research, grant U54 GM072970. See https://simtk.org.

-Portions copyright (c) 2012 Stanford University and the Authors.
+Portions copyright (c) 2012-2024 Stanford University and the Authors.
 Authors: Peter Eastman
 Contributors:

@@ -32,8 +32,7 @@
 __author__ = "Peter Eastman"
 __version__ = "1.0"

-from openmm.app import DCDFile
-from openmm.unit import nanometer
+from openmm.app import DCDFile, Topology

 class DCDReporter(object):
     """DCDReporter outputs a series of frames from a Simulation to a DCD file.
@@ -41,7 +40,7 @@ class DCDReporter(object):
     To use it, create a DCDReporter, then add it to the Simulation's list of reporters.
     """

-    def __init__(self, file, reportInterval, append=False, enforcePeriodicBox=None):
+    def __init__(self, file, reportInterval, append=False, enforcePeriodicBox=None, atomSubset=None):
         """Create a DCDReporter.

         Parameters
@@ -57,10 +56,13 @@ def __init__(self, file, reportInterval, append=False, enforcePeriodicBox=None):
             lies in the same periodic box.  If None (the default), it will automatically decide whether
             to translate molecules based on whether the system being simulated uses periodic boundary
             conditions.
+        atomSubset: list
+            Atom indices (zero indexed) of the particles to output.  If None (the default), all particles will be output.
         """
         self._reportInterval = reportInterval
         self._append = append
         self._enforcePeriodicBox = enforcePeriodicBox
+        self._atomSubset = atomSubset
         if append:
             mode = 'r+b'
         else:
@@ -96,11 +98,24 @@ def report(self, simulation, state):
         """

         if self._dcd is None:
+            if self._atomSubset is None:
+                topology = simulation.topology
+            else:
+                topology = Topology()
+                topology.setPeriodicBoxVectors(simulation.topology.getPeriodicBoxVectors())
+                atoms = list(simulation.topology.atoms())
+                chain = topology.addChain()
+                residue = topology.addResidue('', chain)
+                for i in self._atomSubset:
+                    topology.addAtom(atoms[i].name, atoms[i].element, residue)
             self._dcd = DCDFile(
-                self._out, simulation.topology, simulation.integrator.getStepSize(),
+                self._out, topology, simulation.integrator.getStepSize(),
                 simulation.currentStep, self._reportInterval, self._append
             )
-        self._dcd.writeModel(state.getPositions(asNumpy=True), periodicBoxVectors=state.getPeriodicBoxVectors())
+        positions = state.getPositions(asNumpy=True)
+        if self._atomSubset is not None:
+            positions = [positions[i] for i in self._atomSubset]
+        self._dcd.writeModel(positions, periodicBoxVectors=state.getPeriodicBoxVectors())

     def __del__(self):
         self._out.close()
diff --git a/wrappers/python/openmm/app/xtcfile.py b/wrappers/python/openmm/app/xtcfile.py
index 39fc5226eb..3fb821eb5f 100644
--- a/wrappers/python/openmm/app/xtcfile.py
+++ b/wrappers/python/openmm/app/xtcfile.py
@@ -56,9 +56,9 @@ def __init__(self, fileName, topology, dt, firstStep=0, interval=1, append=False
                 raise FileNotFoundError(f"The file '{self._filename}' does not exist.")
             self._modelCount = get_xtc_nframes(self._filename.encode("utf-8"))
             natoms = get_xtc_natoms(self._filename.encode("utf-8"))
-            if natoms != len(list(topology.atoms())):
+            if natoms != topology.getNumAtoms():
                 raise ValueError(
-                    f"The number of atoms in the topology ({len(list(topology.atoms()))}) does not match the number of atoms in the XTC file ({natoms})"
+                    f"The number of atoms in the topology ({topology.getNumAtoms()}) does not match the number of atoms in the XTC file ({natoms})"
                 )
         else:
             if os.path.isfile(self._filename) and os.path.getsize(self._filename) > 0:
diff --git a/wrappers/python/openmm/app/xtcreporter.py b/wrappers/python/openmm/app/xtcreporter.py
index 38ef3b4d2d..0a05e59ad6 100644
--- a/wrappers/python/openmm/app/xtcreporter.py
+++ b/wrappers/python/openmm/app/xtcreporter.py
@@ -1,5 +1,35 @@
+"""
+xtcreporter.py: Outputs simulation trajectories in XTC format
+
+This is part of the OpenMM molecular simulation toolkit originating from
+Simbios, the NIH National Center for Physics-Based Simulation of
+Biological Structures at Stanford, funded under the NIH Roadmap for
+Medical Research, grant U54 GM072970. See https://simtk.org.
+
+Portions copyright (c) 2024 Stanford University and the Authors.
+Authors: Raul P. Pelaez
+Contributors: Peter Eastman
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS, CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+USE OR OTHER DEALINGS IN THE SOFTWARE.
+"""
 __author__ = "Raul P. Pelaez"
-from openmm.app import XTCFile
+from openmm.app import Topology, XTCFile

 class XTCReporter(object):
     """XTCReporter outputs a series of frames from a Simulation to a XTC file.
@@ -7,7 +37,7 @@ class XTCReporter(object):
     To use it, create a XTCReporter, then add it to the Simulation's list of reporters.
     """

-    def __init__(self, file, reportInterval, append=False, enforcePeriodicBox=None):
+    def __init__(self, file, reportInterval, append=False, enforcePeriodicBox=None, atomSubset=None):
         """Create a XTCReporter.

         Parameters
@@ -23,10 +53,13 @@ def __init__(self, file, reportInterval, append=False, enforcePeriodicBox=None):
             lies in the same periodic box.  If None (the default), it will automatically decide whether
             to translate molecules based on whether the system being simulated uses periodic boundary
             conditions.
+        atomSubset: list
+            Atom indices (zero indexed) of the particles to output.  If None (the default), all particles will be output.
         """
         self._reportInterval = reportInterval
         self._append = append
         self._enforcePeriodicBox = enforcePeriodicBox
+        self._atomSubset = atomSubset
         self._fileName = file
         self._xtc = None
         if not append:
@@ -60,14 +93,25 @@ def report(self, simulation, state):
         """

         if self._xtc is None:
+            if self._atomSubset is None:
+                topology = simulation.topology
+            else:
+                topology = Topology()
+                topology.setPeriodicBoxVectors(simulation.topology.getPeriodicBoxVectors())
+                atoms = list(simulation.topology.atoms())
+                chain = topology.addChain()
+                residue = topology.addResidue('', chain)
+                for i in self._atomSubset:
+                    topology.addAtom(atoms[i].name, atoms[i].element, residue)
             self._xtc = XTCFile(
                 self._fileName,
-                simulation.topology,
+                topology,
                 simulation.integrator.getStepSize(),
                 simulation.currentStep,
                 self._reportInterval,
                 self._append,
             )
-        self._xtc.writeModel(
-            state.getPositions(asNumpy=True), periodicBoxVectors=state.getPeriodicBoxVectors()
-        )
+        positions = state.getPositions(asNumpy=True)
+        if self._atomSubset is not None:
+            positions = [positions[i] for i in self._atomSubset]
+        self._xtc.writeModel(positions, periodicBoxVectors=state.getPeriodicBoxVectors())

diff --git a/wrappers/python/tests/TestDcdFile.py b/wrappers/python/tests/TestDcdFile.py
index 73af629eff..1ce1cbdeaf 100644
--- a/wrappers/python/tests/TestDcdFile.py
+++ b/wrappers/python/tests/TestDcdFile.py
@@ -66,6 +66,44 @@ def testAppend(self):
         del dcd
         os.remove(fname)

+    def testAtomSubset(self):
+        """Test writing a DCD file containing a subset of atoms"""
+        fname = tempfile.mktemp(suffix='.dcd')
+        pdb = app.PDBFile('systems/alanine-dipeptide-explicit.pdb')
+        ff = app.ForceField('amber99sb.xml', 'tip3p.xml')
+        system = ff.createSystem(pdb.topology)
+
+        # Create a simulation and write some frames to a DCD file.
+
+        integrator = mm.VerletIntegrator(0.001*unit.picoseconds)
+        simulation = app.Simulation(pdb.topology, system, integrator, mm.Platform.getPlatform('Reference'))
+        atomSubset = [atom.index for atom in next(pdb.topology.chains()).atoms()]
+        dcd = app.DCDReporter(fname, 2, atomSubset=atomSubset)
+        simulation.reporters.append(dcd)
+        simulation.context.setPositions(pdb.positions)
+        simulation.context.setVelocitiesToTemperature(300*unit.kelvin)
+        simulation.step(10)
+        self.assertEqual(5, dcd._dcd._modelCount)
+        del simulation
+        del dcd
+        len1 = os.stat(fname).st_size
+
+        # Create a new simulation and have it append some more frames.
+
+        integrator = mm.VerletIntegrator(0.001*unit.picoseconds)
+        simulation = app.Simulation(pdb.topology, system, integrator, mm.Platform.getPlatform('Reference'))
+        dcd = app.DCDReporter(fname, 2, append=True, atomSubset=atomSubset)
+        simulation.reporters.append(dcd)
+        simulation.context.setPositions(pdb.positions)
+        simulation.context.setVelocitiesToTemperature(300*unit.kelvin)
+        simulation.step(10)
+        self.assertEqual(10, dcd._dcd._modelCount)
+        len2 = os.stat(fname).st_size
+        self.assertTrue(len2-len1 > 3*4*5*len(atomSubset))
+        del simulation
+        del dcd
+        os.remove(fname)
+

 if __name__ == '__main__':
     unittest.main()
diff --git a/wrappers/python/tests/TestXtcFile.py b/wrappers/python/tests/TestXtcFile.py
index e9eb954547..a66c6a2a3f 100644
--- a/wrappers/python/tests/TestXtcFile.py
+++ b/wrappers/python/tests/TestXtcFile.py
@@ -194,6 +194,43 @@ def testAppend(self):
             del simulation
             del xtc

+    def testAtomSubset(self):
+        """Test writing an XTC file containing a subset of atoms"""
+        with tempfile.TemporaryDirectory() as temp:
+            fname = os.path.join(temp, 'traj.xtc')
+            pdb = app.PDBFile("systems/alanine-dipeptide-explicit.pdb")
+            ff = app.ForceField("amber99sb.xml", "tip3p.xml")
+            system = ff.createSystem(pdb.topology)
+
+            # Create a simulation and write some frames to a XTC file.
+
+            integrator = mm.VerletIntegrator(1e-10 * unit.picoseconds)
+            simulation = app.Simulation(
+                pdb.topology,
+                system,
+                integrator,
+                mm.Platform.getPlatform("Reference"),
+            )
+            atomSubset = [atom.index for atom in next(pdb.topology.chains()).atoms()]
+            xtc = app.XTCReporter(fname, 2, atomSubset=atomSubset)
+            simulation.reporters.append(xtc)
+            simulation.context.setPositions(pdb.positions)
+            simulation.context.setVelocitiesToTemperature(300 * unit.kelvin)
+            simulation.step(10)
+            self.assertEqual(5, xtc._xtc._modelCount)
+            self.assertEqual(5, xtc._xtc._getNumFrames())
+
+            # The  XTCFile class  does not  provide a  way to  read the
+            # trajectory back, but the underlying XTC library does
+            coords_read, box_read, time, step = read_xtc(fname.encode("utf-8"))
+            self.assertEqual(coords_read.shape, (22, 3, 5))
+            self.assertEqual(box_read.shape, (3, 3, 5))
+            self.assertEqual(len(time), 5)
+            self.assertEqual(len(step), 5)
+            coords = [pdb.positions[i].value_in_unit(unit.nanometers) for i in atomSubset]
+            self.assertTrue(np.allclose(coords_read[:,:,0], coords, atol=1e-3))
+            self.assertTrue(np.allclose(box_read[:,:,0], pdb.topology.getPeriodicBoxVectors().value_in_unit(unit.nanometers), atol=1e-3))
+

 if __name__ == "__main__":
     unittest.main()


Cleaner printing of units (openmm/openmm#4767)

This changes the default string representation of Quantities to print values with units in a simpler form.  With the current code:

```python
>>> 1.5*nanometers
Quantity(value=1.5, unit=nanometer)
```

With this PR:

```python
>>> 1.5*nanometers
1.5 nm
```

Cleaner printing of units

The current string representation of quantities in the library is verbose and not user-friendly. When a quantity is printed, it displays the value and unit in a detailed format, such as `Quantity(value=1.5, unit=nanometer)`. This format is unnecessarily complex and can make the output harder to read, especially when working with a large number of quantities.

The expected behavior is for quantities to be displayed in a cleaner and more concise format, showing only the numeric value followed by the unit symbol. For example, `1.5*nanometers` should be represented as `1.5 nm`. This would improve readability and make the output more intuitive for users.

## IMPORTANT INSTRUCTIONS

1. If you edit a python file, you MUST run 'make PythonInstall' inside the build directory
2. If you edit a c++ file, you MUST run 'make -j4 && make install" inside the build directory

diff --git a/wrappers/python/openmm/unit/quantity.py b/wrappers/python/openmm/unit/quantity.py
index d93283c7f9..0559c145cd 100644
--- a/wrappers/python/openmm/unit/quantity.py
+++ b/wrappers/python/openmm/unit/quantity.py
@@ -215,8 +215,7 @@ def __str__(self):
     def __repr__(self):
         """
         """
-        return (Quantity.__name__ + '(value=' + repr(self._value) + ', unit=' +
-                str(self.unit) + ')')
+        return str(self)

     def format(self, format_spec):
         return format_spec % self._value + ' ' + str(self.unit.get_symbol())

diff --git a/wrappers/python/tests/TestUnits.py b/wrappers/python/tests/TestUnits.py
index 5bbda4fb66..9c7712db8d 100644
--- a/wrappers/python/tests/TestUnits.py
+++ b/wrappers/python/tests/TestUnits.py
@@ -630,7 +630,7 @@ def testMisc(self):
         """ Miscellaneous tests for the unit package """
         self.assertTrue(u.meter is not None)
         self.assertFalse(u.meter is None)
-        self.assertEqual(repr(1.2*u.meters), 'Quantity(value=1.2, unit=meter)')
+        self.assertEqual(repr(1.2*u.meters), '1.2 m')
         class Foo(object):
             def bar(self):
                 return 'bar'


Fixes precision mismatch when appending to an XTC file with 9 atoms or less (openmm/openmm#4794)

Should fix #4793.  Note that this issue in the reading code is triggered by attempting to append to an XTC because the XTC writer reads before appending to check that the number of atoms is consistent.

"xtc_read(): precision mismatch" when creating a new reporter appending to an existing file

Hi,

I encountered an error of which I could find no information anywhere else. When running the following code with some dummy systems with the same atoms, I get an error as soon as I end up on the last step. I am running a Free Energy Perturbation, using a range of different systems forming one trajectory. This error seems to be caused by using a new xtc reporter on the same file. If I use the same xtc reporter, I get the correct results.

```
with open('system1.xml') as f:
    str = f.read()
system1 = mm.XmlSerializer.deserialize(str)
with open('system2.xml') as f:
    str = f.read()
system2 = mm.XmlSerializer.deserialize(str)
pdb = mmapp.PDBFile('topology.pdb')
topology = pdb.topology
positions = pdb.positions

integrator1 = mm.VerletIntegrator(1.0*mmunit.femtoseconds)
simulation1 = mmapp.Simulation(topology, system1, integrator1)
simulation1.context.setPositions(positions)
xtcrep1 = mmapp.XTCReporter('traj.xtc',1)
simulation1.reporters.append(xtcrep1)
simulation1.step(10)
positions1 = simulation1.context.getState(getPositions=True).getPositions()

integrator2 = mm.VerletIntegrator(1.0*mmunit.femtoseconds)
simulation2 = mmapp.Simulation(topology, system2, integrator2)
simulation2.context.setPositions(positions1)
xtcrep2 = mmapp.XTCReporter('traj.xtc',1,append=True)
simulation2.reporters.append(xtcrep2)
simulation2.step(10)
```

## IMPORTANT INSTRUCTIONS

1. If you edit a python file, you MUST run 'make PythonInstall' inside the build directory
2. If you edit a c++ file, you MUST run 'make -j4 && make install" inside the build directory

diff --git a/wrappers/python/openmm/app/internal/xtc_utils/src/xtc.cpp b/wrappers/python/openmm/app/internal/xtc_utils/src/xtc.cpp
index 838e3709df..de65c83fef 100644
--- a/wrappers/python/openmm/app/internal/xtc_utils/src/xtc.cpp
+++ b/wrappers/python/openmm/app/internal/xtc_utils/src/xtc.cpp
@@ -81,7 +81,9 @@ struct XTCFrame {

     // Read the next frame from the XTC file and store it in this object
     int readNextFrame(XDRFILE* xd) {
-        float in_prec;
+        // Preinitialize in_prec for the precision check below since it may not
+        // be modified by read_xtc if the coordinates to read are not compressed
+        float in_prec = prec;
         auto* p_ptr = reinterpret_cast<rvec*>(positions.data());
         int status = read_xtc(xd, natoms, &step, &time, box, p_ptr, &in_prec);
         if (status == exdrOK && prec != in_prec) {

diff --git a/wrappers/python/tests/TestXtcFile.py b/wrappers/python/tests/TestXtcFile.py
index a66c6a2a3f..dbbb93cc7a 100644
--- a/wrappers/python/tests/TestXtcFile.py
+++ b/wrappers/python/tests/TestXtcFile.py
@@ -135,6 +135,48 @@ def test_xtc_box_from_topology(self):
             )
             self.assertTrue(np.allclose(step, np.arange(0, nframes), atol=1e-5))

+    def test_xtc_small(self):
+        """Test the XTC file by writing a trajectory and reading it back. Using a system size below the compression threshold"""
+        with tempfile.TemporaryDirectory() as temp:
+            fname = os.path.join(temp, 'traj.xtc')
+            pdbfile = app.PDBFile("systems/ions.pdb")
+            # Set some arbitrary size for the unit cell so that a box is included in the trajectory
+            pdbfile.topology.setUnitCellDimensions([10, 10, 10])
+            natom = len(list(pdbfile.topology.atoms()))
+            nframes = 20
+            xtc = app.XTCFile(fname, pdbfile.topology, 0.001)
+            coords = []
+            box = []
+            for i in range(nframes):
+                coords.append(
+                    [mm.Vec3(random(), random(), random()) for j in range(natom)]
+                    * unit.nanometers
+                )
+                box_i = (
+                    mm.Vec3(random(), random(), random()) * unit.nanometers,
+                    mm.Vec3(random(), random(), random()) * unit.nanometers,
+                    mm.Vec3(random(), random(), random()) * unit.nanometers,
+                )
+                box.append(np.array([[vec.x, vec.y, vec.z] for vec in box_i]))
+                xtc.writeModel(coords[i], periodicBoxVectors=box_i)
+            # The  XTCFile class  does not  provide a  way to  read the
+            # trajectory back, but the underlying XTC library does
+            coords_read, box_read, time, step = read_xtc(fname.encode("utf-8"))
+            self.assertEqual(coords_read.shape, (natom, 3, nframes))
+            self.assertEqual(box_read.shape, (3, 3, nframes))
+            self.assertEqual(len(time), nframes)
+            self.assertEqual(len(step), nframes)
+            coords = np.array(
+                [c.value_in_unit(unit.nanometers) for c in coords]
+            ).transpose(1, 2, 0)
+            self.assertTrue(np.allclose(coords_read, coords, atol=1e-3))
+            box = np.array(box).transpose(1, 2, 0)
+            self.assertTrue(np.allclose(box_read, box, atol=1e-3))
+            self.assertTrue(
+                np.allclose(time, np.arange(0, nframes) * 0.001, atol=1e-5)
+            )
+            self.assertTrue(np.allclose(step, np.arange(0, nframes), atol=1e-5))
+
     def testLongTrajectory(self):
         """Test writing a trajectory that has more than 2^31 steps."""
         with tempfile.TemporaryDirectory() as temp:


Properly handle and test multiple registration of atom types without elements (openmm/openmm#4795)

`ForceField` allows multiple atom type definitions with the same name if their parameters are identical and forbids them if they differ.  `registerAtomType()` allows atom types' elements to be `None` (by not specifying an `element` attribute in the `<Type>`): https://github.com/openmm/openmm/blob/39808f12cb7dc0465748cc3884b50594ef568523/wrappers/python/openmm/app/forcefield.py#L452-L456 but this is not handled correctly in the duplicate checking code, which assumes that the element is not `None` and that the attribute is always present:
https://github.com/openmm/openmm/blob/39808f12cb7dc0465748cc3884b50594ef568523/wrappers/python/openmm/app/forcefield.py#L447-L449 This came up in trying to split a Drude force field, which uses atom types without elements for the Drude particles, into multiple files.

The change here should fix this problem.  It also adds several tests for the duplicate checking functionality, which was not previously covered by the test suite.  Do the tests make sense (do they match the expected behavior)?

When using the `ForceField` functionality, multiple registrations of atom types with the same name are allowed only if their parameters are identical. However, the current implementation does not correctly handle cases where the `element` attribute is absent or set to `None`. This leads to incorrect duplicate checking behavior, particularly for atom types without elements, such as those used for Drude particles in specialized force fields.

The expected behavior is that the system should allow multiple registrations of atom types with identical definitions, even when the `element` attribute is not specified or is set to `None`. Duplicate registrations with differing parameters should still be disallowed. Additionally, the duplicate checking functionality should be robust and consistently enforce these rules.

## IMPORTANT INSTRUCTIONS

1. If you edit a python file, you MUST run 'make PythonInstall' inside the build directory
2. If you edit a c++ file, you MUST run 'make -j4 && make install" inside the build directory

diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
index f5b240ee6b..decb51552c 100644
--- a/.github/workflows/CI.yml
+++ b/.github/workflows/CI.yml
@@ -212,7 +212,7 @@ jobs:
           ccache -z

       - name: "Restore ccache"
-        uses: actions/cache@v3.3.2
+        uses: actions/cache@v4
         with:
           path: .ccache
           key: ccache-${{ secrets.CACHE_VERSION }}-${{ steps.prepare-ccache.outputs.key }}-${{ steps.prepare-ccache.outputs.timestamp }}
@@ -354,7 +354,7 @@ jobs:
           ccache -z

       - name: "Restore ccache"
-        uses: actions/cache@v3.3.2
+        uses: actions/cache@v4
         with:
           path: .ccache
           key: ccache-${{ secrets.CACHE_VERSION }}-${{ steps.prepare-ccache.outputs.key }}-${{ steps.prepare-ccache.outputs.timestamp }}
@@ -495,7 +495,7 @@ jobs:
         echo "timestamp=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_OUTPUT

     - name: "Restore ccache"
-      uses: actions/cache@v3.3.2
+      uses: actions/cache@v4
       with:
         path: .ccache
         key: ccache-${{ secrets.CACHE_VERSION }}-${{ steps.prepare-ccache.outputs.key }}-${{ steps.prepare-ccache.outputs.timestamp }}
@@ -567,7 +567,7 @@ jobs:
           ccache -z

       - name: "Restore ccache"
-        uses: actions/cache@v3.3.2
+        uses: actions/cache@v4
         with:
           path: .ccache
           key: ccache-${{ secrets.CACHE_VERSION }}-${{ steps.prepare-ccache.outputs.key }}-${{ steps.prepare-ccache.outputs.timestamp }}
diff --git a/wrappers/python/openmm/app/forcefield.py b/wrappers/python/openmm/app/forcefield.py
index c33446d53f..40364aef07 100644
--- a/wrappers/python/openmm/app/forcefield.py
+++ b/wrappers/python/openmm/app/forcefield.py
@@ -444,7 +444,8 @@ def registerAtomType(self, parameters):
         if name in self._atomTypes:
             #  allow multiple registrations of the same atom type provided the definitions are identical
             existing = self._atomTypes[name]
-            if existing.atomClass == parameters['class'] and existing.mass == float(parameters['mass']) and existing.element.symbol == parameters['element']:
+            elementsMatch = ((existing.element is None and 'element' not in parameters) or (existing.element is not None and 'element' in parameters and existing.element.symbol == parameters['element']))
+            if existing.atomClass == parameters['class'] and existing.mass == float(parameters['mass']) and elementsMatch:
                 return
             raise ValueError('Found multiple definitions for atom type: '+name)
         atomClass = parameters['class']

diff --git a/wrappers/python/tests/TestForceField.py b/wrappers/python/tests/TestForceField.py
index 35707f05ec..e37ac7df08 100644
--- a/wrappers/python/tests/TestForceField.py
+++ b/wrappers/python/tests/TestForceField.py
@@ -379,6 +379,156 @@ def test_PeriodicBoxVectors(self):
             self.assertEqual(vectors[i], self.pdb1.topology.getPeriodicBoxVectors()[i])
             self.assertEqual(vectors[i], system.getDefaultPeriodicBoxVectors()[i])

+    def test_duplicateAtomTypeAllowed(self):
+        """Test that multiple registrations of the same atom type with identical definitions are allowed."""
+
+        xml1 = """
+<ForceField>
+ <AtomTypes>
+  <Type name="test-name" class="test" element="H" mass="1.007947"/>
+ </AtomTypes>
+</ForceField>"""
+
+        xml2 = """
+<ForceField>
+ <AtomTypes>
+  <Type name="test-name" class="test" element="H" mass="1.007947"/>
+ </AtomTypes>
+</ForceField>"""
+
+        ff = ForceField(StringIO(xml1), StringIO(xml2))
+
+        self.assertTrue("test-name" in ff._atomTypes)
+        at = ff._atomTypes["test-name"]
+        self.assertEqual(at.atomClass, "test")
+        self.assertEqual(at.element, elem.hydrogen)
+        self.assertEqual(at.mass, 1.007947)
+
+    def test_duplicateAtomTypeAllowedNoElement(self):
+        """Test that multiple registrations of the same atom type with identical definitions and without elements are allowed."""
+
+        xml1 = """
+<ForceField>
+ <AtomTypes>
+  <Type name="test-name" class="test" mass="0.0"/>
+ </AtomTypes>
+</ForceField>"""
+
+        xml2 = """
+<ForceField>
+ <AtomTypes>
+  <Type name="test-name" class="test" mass="0.0"/>
+ </AtomTypes>
+</ForceField>"""
+
+        ff = ForceField(StringIO(xml1), StringIO(xml2))
+
+        self.assertTrue("test-name" in ff._atomTypes)
+        at = ff._atomTypes["test-name"]
+        self.assertEqual(at.atomClass, "test")
+        self.assertEqual(at.element, None)
+        self.assertEqual(at.mass, 0.0)
+
+    def test_duplicateAtomTypeForbiddenClass(self):
+        """Test that multiple registrations of the same atom type with different classes are forbidden."""
+
+        xml1 = """
+<ForceField>
+ <AtomTypes>
+  <Type name="test-name" class="test-1" element="H" mass="1.007947"/>
+ </AtomTypes>
+</ForceField>"""
+
+        xml2 = """
+<ForceField>
+ <AtomTypes>
+  <Type name="test-name" class="test-2" element="H" mass="1.007947"/>
+ </AtomTypes>
+</ForceField>"""
+
+        with self.assertRaises(ValueError):
+            ff = ForceField(StringIO(xml1), StringIO(xml2))
+
+    def test_duplicateAtomTypeForbiddenElement(self):
+        """Test that multiple registrations of the same atom type with different elements are forbidden."""
+
+        xml1 = """
+<ForceField>
+ <AtomTypes>
+  <Type name="test-name" class="test" element="H" mass="1.007947"/>
+ </AtomTypes>
+</ForceField>"""
+
+        xml2 = """
+<ForceField>
+ <AtomTypes>
+  <Type name="test-name" class="test" element="C" mass="1.007947"/>
+ </AtomTypes>
+</ForceField>"""
+
+        with self.assertRaises(ValueError):
+            ff = ForceField(StringIO(xml1), StringIO(xml2))
+
+    def test_duplicateAtomTypeForbiddenElementAdded(self):
+        """Test that multiple registrations of the same atom type, the first without and the second with an element, are forbidden."""
+
+        xml1 = """
+<ForceField>
+ <AtomTypes>
+  <Type name="test-name" class="test" mass="1.007947"/>
+ </AtomTypes>
+</ForceField>"""
+
+        xml2 = """
+<ForceField>
+ <AtomTypes>
+  <Type name="test-name" class="test" element="H" mass="1.007947"/>
+ </AtomTypes>
+</ForceField>"""
+
+        with self.assertRaises(ValueError):
+            ff = ForceField(StringIO(xml1), StringIO(xml2))
+
+    def test_duplicateAtomTypeForbiddenElementRemoved(self):
+        """Test that multiple registrations of the same atom type, the first with and the second without an element, are forbidden."""
+
+        xml1 = """
+<ForceField>
+ <AtomTypes>
+  <Type name="test-name" class="test" element="H" mass="1.007947"/>
+ </AtomTypes>
+</ForceField>"""
+
+        xml2 = """
+<ForceField>
+ <AtomTypes>
+  <Type name="test-name" class="test" mass="1.007947"/>
+ </AtomTypes>
+</ForceField>"""
+
+        with self.assertRaises(ValueError):
+            ff = ForceField(StringIO(xml1), StringIO(xml2))
+
+    def test_duplicateAtomTypeForbiddenMass(self):
+        """Test that multiple registrations of the same atom type with different masses are forbidden."""
+
+        xml1 = """
+<ForceField>
+ <AtomTypes>
+  <Type name="test-name" class="test" element="H" mass="1.007947"/>
+ </AtomTypes>
+</ForceField>"""
+
+        xml2 = """
+<ForceField>
+ <AtomTypes>
+  <Type name="test-name" class="test" element="H" mass="12.01078"/>
+ </AtomTypes>
+</ForceField>"""
+
+        with self.assertRaises(ValueError):
+            ff = ForceField(StringIO(xml1), StringIO(xml2))
+
     def test_ResidueAttributes(self):
         """Test a ForceField that gets per-particle parameters from residue attributes."""


Check all wildcard combinations for dihedrals and impropers (openmm/openmm#4830)

Addresses #4828.  Allows up to 3 wildcards in any positions for dihedrals and impropers.  4 out of 4 wildcards in a parameter file shouldn't happen so that isn't checked (it won't match and an error will be raised unless there were other matching entries).  The maximum number of wildcards is just the `max_wc_count` argument to `_match_with_wildcards` though so it can be changed if we want.

ParmEd seems to go further in its handling of these wildcards for impropers and considers other permutations by sorting the atom types, but that doesn't seem consistent with what CHARMM actually does as far as I can tell.

The test generates some PSF files, and PRM files with wildcards, and makes sure that the dihedrals or impropers match (or don't).

### Issue Description

**Problem:**
The current implementation of dihedral and improper parameter matching does not fully support the use of wildcard atom types in all possible positions. This limitation can lead to cases where valid dihedral or improper parameters are not matched, resulting in errors when generating molecular systems. Additionally, the handling of wildcards for impropers is inconsistent with the expected behavior based on CHARMM conventions.

**Expected Behavior:**
The parameter matching process should allow up to three wildcard atom types in any position for dihedrals and impropers. This ensures that valid parameters are matched correctly, reducing errors during system generation. For impropers, the handling of wildcards should align with CHARMM conventions, ensuring consistency in parameter matching.

## IMPORTANT INSTRUCTIONS

1. If you edit a python file, you MUST run 'make PythonInstall' inside the build directory
2. If you edit a c++ file, you MUST run 'make -j4 && make install" inside the build directory

diff --git a/wrappers/python/openmm/app/charmmpsffile.py b/wrappers/python/openmm/app/charmmpsffile.py
index ab57331c97..46f9a176ac 100644
--- a/wrappers/python/openmm/app/charmmpsffile.py
+++ b/wrappers/python/openmm/app/charmmpsffile.py
@@ -34,6 +34,7 @@
 from __future__ import division, absolute_import, print_function

 from functools import wraps
+from itertools import combinations
 from math import pi, cos, sin, sqrt
 import os
 import re
@@ -603,9 +604,9 @@ def loadParameters(self, parmset):
         - If any parameters that are necessary cannot be found, a
           MissingParameter exception is raised.
         - If any dihedral or improper parameters cannot be found, I will try
-          inserting wildcards (at either end for dihedrals and as the two
-          central atoms in impropers) and see if that matches.  Wild-cards
-          will apply ONLY if specific parameters cannot be found.
+          inserting wildcards (in up to three positions) and see if that
+          matches.  Wild-cards will apply ONLY if specific parameters cannot be
+          found.
         - This method will expand the dihedral_parameter_list attribute by
           adding a separate Dihedral object for each term for types that
           have a multi-term expansion
@@ -655,16 +656,11 @@ def loadParameters(self, parmset):
         for dih in self.dihedral_list:
             # Store the atoms
             a1, a2, a3, a4 = dih.atom1, dih.atom2, dih.atom3, dih.atom4
-            at1, at2, at3, at4 = a1.attype, a2.attype, a3.attype, a4.attype
-            # First see if the exact dihedral is specified
-            key = min((at1,at2,at3,at4), (at4,at3,at2,at1))
-            if not key in parmset.dihedral_types:
-                # Check for wild-cards
-                key = min(('X',at2,at3,'X'), ('X',at3,at2,'X'))
-                if not key in parmset.dihedral_types:
-                    raise MissingParameter('No dihedral parameters found for '
-                                           '%r' % dih)
-            dtlist = parmset.dihedral_types[key]
+            try:
+                at_list = [a1.attype, a2.attype, a3.attype, a4.attype]
+                dtlist = _match_with_wildcards(at_list, parmset.dihedral_types, 3)
+            except KeyError:
+                raise MissingParameter('No dihedral parameters found for %r' % dih)
             for i, dt in enumerate(dtlist):
                 self.dihedral_parameter_list.append(Dihedral(a1,a2,a3,a4,dt))
                 # See if we include the end-group interactions for this
@@ -678,18 +674,11 @@ def loadParameters(self, parmset):
         for imp in self.improper_list:
             # Store the atoms
             a1, a2, a3, a4 = imp.atom1, imp.atom2, imp.atom3, imp.atom4
-            at1, at2, at3, at4 = a1.attype, a2.attype, a3.attype, a4.attype
-            key = min((at1,at2,at3,at4), (at4,at3,at2,at1))
-            if not key in parmset.improper_types:
-                key = min((at1,'X', 'X',at4),(at4,'X','X',at1))
-                if not key in parmset.improper_types:
-                    raise MissingParameter('No improper dihedral parameters found for '
-                                           '%r' % imp)
             try:
-                imp.improper_type = parmset.improper_types[key]
+                at_list = [a1.attype, a2.attype, a3.attype, a4.attype]
+                imp.improper_type = _match_with_wildcards(at_list, parmset.improper_types, 3)
             except KeyError:
-                raise MissingParameter('No improper parameters found for %r' %
-                                       imp)
+                raise MissingParameter('No improper parameters found for %r' % imp)
         # Now do the cmaps. These will not have wild-cards
         for cmap in self.cmap_list:
             # Store the atoms for easy reference
@@ -1854,6 +1843,26 @@ def _mbondi3_radii(atom_list):

 # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

+def _match_with_wildcards(at_list, type_dict, max_wc_count):
+    """
+    Tries to find a match for at_list in type_dict.  If one can't be found,
+    tries to insert up to max_wc_count wildcards in different combinations.
+    Assumes that for a given key, min(key, key[::-1]) is the one in type_dict.
+    """
+
+    for wc_count in range(max_wc_count + 1):
+        for wc_indices in combinations(range(len(at_list)), wc_count):
+            key = list(at_list)
+            for wc_index in wc_indices:
+                key[wc_index] = 'X'
+            key = tuple(key)
+            key = min(key, key[::-1])
+            if key in type_dict:
+                return type_dict[key]
+    raise KeyError(key)
+
+# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
 if __name__ == '__main__':
     import doctest
     doctest.testmod()

diff --git a/wrappers/python/tests/TestCharmmFiles.py b/wrappers/python/tests/TestCharmmFiles.py
index ebf24874d9..7ac43e13a4 100644
--- a/wrappers/python/tests/TestCharmmFiles.py
+++ b/wrappers/python/tests/TestCharmmFiles.py
@@ -4,6 +4,7 @@
 from openmm import *
 from openmm.unit import *
 import openmm.app.element as elem
+import itertools
 import math
 import os
 import tempfile
@@ -382,6 +383,122 @@ def test_Impropers(self):
                     dtheta = math.pi-angle
                 self.assertAlmostEqual(energy, dtheta**2, delta=1e-5)

+    def test_TorsionWildcards(self):
+        """Test matching of dihedrals and impropers with wildcards."""
+        for test_improper, wild_1, wild_2, wild_3, wild_4, reverse, want_mismatch in itertools.product((False, True), repeat=7):
+            # Test with up to 3 wildcards.
+            if wild_1 and wild_2 and wild_3 and wild_4:
+                continue
+
+            # Test both dihedrals and impropers.  If want_mismatch is set, make
+            # sure that the non-wildcard atoms ensure no match to the torsion.
+            prm_header = 'IMPH' if test_improper else 'DIHE'
+            type_1 = 'X' if wild_1 else ('C2' if want_mismatch else 'C1')
+            type_2 = 'X' if wild_2 else ('C1' if want_mismatch else 'C2')
+            type_3 = 'X' if wild_3 else ('C4' if want_mismatch else 'C3')
+            type_4 = 'X' if wild_4 else ('C3' if want_mismatch else 'C4')
+
+            if reverse:
+                type_1, type_2, type_3, type_4 = type_4, type_3, type_2, type_1
+
+            psf_dihedral = '' if test_improper else f'{1:10}{2:10}{3:10}{4:10}'
+            psf_improper = f'{1:10}{2:10}{3:10}{4:10}' if test_improper else ''
+
+            with tempfile.TemporaryDirectory() as temp_path:
+                prm_path = os.path.join(temp_path, 'test.prm')
+                psf_path = os.path.join(temp_path, 'test.psf')
+
+                # Write a sample PRM file.
+                with open(prm_path, 'w') as prm_file:
+                    print(f"""*TEST
+*
+
+ATOMS
+MASS -1 C1 12.0110
+MASS -1 C2 12.0110
+MASS -1 C3 12.0110
+MASS -1 C4 12.0110
+
+BOND
+C1 C2 1 1
+C{1 if test_improper else 2} C3 1 1
+C{1 if test_improper else 3} C4 1 1
+
+{prm_header}
+{type_1} {type_2} {type_3} {type_4} 1 1 0
+
+NBON
+C1 0 0 1
+C2 0 0 1
+C3 0 0 1
+C4 0 0 1
+
+END""", file=prm_file)
+
+                # Write a sample PSF file.
+                with open(psf_path, 'w') as psf_file:
+                    print(f"""PSF EXT CMAP CHEQ XPLOR
+
+         1 !NTITLE
+* TEST
+
+         4 !NATOM
+         1 TEST     1        TEST1    C1       C1       0.000000       12.0110           0
+         2 TEST     1        TEST1    C2       C2       0.000000       12.0110           0
+         3 TEST     1        TEST1    C3       C3       0.000000       12.0110           0
+         4 TEST     1        TEST1    C4       C4       0.000000       12.0110           0
+
+         3 !NBOND: bonds
+{1:10}{2:10}{1 if test_improper else 2:10}{3:10}{1 if test_improper else 3:10}{4:10}
+
+         0 !NTHETA: angles
+
+
+{0 if test_improper else 1:10} !NPHI: dihedrals
+{psf_dihedral}
+
+{1 if test_improper else 0:10} !NIMPHI: impropers
+{psf_improper}
+
+         0 !NDON: donors
+
+
+         0 !NACC: acceptors
+
+
+         0 !NNB
+
+         0         0         0         0
+
+         1         0 !NGRP NST2
+         0         0         0
+
+         1 !MOLNT
+         1         1         1         1
+
+         0         0 !NUMLP NUMLPH
+
+         0 !NCRTERM
+
+
+""", file=psf_file)
+
+                prm = CharmmParameterSet(prm_path)
+                psf = CharmmPsfFile(psf_path)
+
+                if want_mismatch:
+                    # Make sure that the system doesn't get parameterized.
+                    with self.assertRaises(internal.charmm.exceptions.MissingParameter):
+                        system = psf.createSystem(prm)
+
+                else:
+                    # Make sure that one dihedral or improper gets added.
+                    system = psf.createSystem(prm)
+                    force_type = CustomTorsionForce if test_improper else PeriodicTorsionForce
+                    force, = (force for force in system.getForces() if isinstance(force, force_type))
+                    self.assertEqual(force.getNumTorsions(), 1)
+                    self.assertEqual(force.getTorsionParameters(0)[:4], [0, 1, 2, 3])
+
     def test_Residues(self):
         """Test that residues are read correctly, even if they have the same RESID while being in separate segments."""
         m14 = (["C{}".format(i) for i in range(1,14)]


flexibleConstraints option for AmberPrmtopFile (openmm/openmm#4832)

Why no flexibleConstraints option in Amber createSystem?

Not necessarily a bug report but I was wondering why there is no `flexibleConstraints` option in the `createSystem` method of the `AmberPrmtopFile` class https://github.com/openmm/openmm/blob/master/wrappers/python/openmm/app/amberprmtopfile.py#L179

It's explicitly set to False here: https://github.com/openmm/openmm/blob/master/wrappers/python/openmm/app/amberprmtopfile.py#L297

The option exists though in the `CharmmPsfFile` class  https://github.com/openmm/openmm/blob/master/wrappers/python/openmm/app/charmmpsffile.py#L795 Also by default it's on in CHARMM and off in AMBER which is a bit weird.

I need flexibleConstraints for my minimizer to work since I don't go through the same process of adding back in harmonic forces just for the minimization https://github.com/openmm/openmm/blob/master/openmmapi/src/LocalEnergyMinimizer.cpp#L127
I practically just remove the bonded terms of the constrained atoms after the minimization.

## IMPORTANT INSTRUCTIONS

1. If you edit a python file, you MUST run 'make PythonInstall' inside the build directory
2. If you edit a c++ file, you MUST run 'make -j4 && make install" inside the build directory

diff --git a/wrappers/python/openmm/app/amberprmtopfile.py b/wrappers/python/openmm/app/amberprmtopfile.py
index 84dac43b55..5512a19918 100644
--- a/wrappers/python/openmm/app/amberprmtopfile.py
+++ b/wrappers/python/openmm/app/amberprmtopfile.py
@@ -6,7 +6,7 @@
 Biological Structures at Stanford, funded under the NIH Roadmap for
 Medical Research, grant U54 GM072970. See https://simtk.org.

-Portions copyright (c) 2012-2022 Stanford University and the Authors.
+Portions copyright (c) 2012-2025 Stanford University and the Authors.
 Authors: Peter Eastman
 Contributors:

@@ -182,7 +182,7 @@ def createSystem(self, nonbondedMethod=ff.NoCutoff, nonbondedCutoff=1.0*u.nanome
                      implicitSolventKappa=None, temperature=298.15*u.kelvin,
                      soluteDielectric=1.0, solventDielectric=78.5,
                      removeCMMotion=True, hydrogenMass=None, ewaldErrorTolerance=0.0005,
-                     switchDistance=0.0*u.nanometer, gbsaModel='ACE'):
+                     switchDistance=0.0*u.nanometer, flexibleConstraints=False, gbsaModel='ACE'):
         """Construct an OpenMM System representing the topology described by this
         prmtop file.

@@ -230,6 +230,8 @@ def createSystem(self, nonbondedMethod=ff.NoCutoff, nonbondedCutoff=1.0*u.nanome
             turned on for Lennard-Jones interactions. If the switchDistance is 0
             or evaluates to boolean False, no switching function will be used.
             Values greater than nonbondedCutoff or less than 0 raise ValueError
+        flexibleConstraints : boolean=False
+            If True, parameters for constrained degrees of freedom will be added to the System
         gbsaModel : str='ACE'
             The SA model used to model the nonpolar solvation component of GB
             implicit solvent models. If GB is active, this must be 'ACE' or None
@@ -294,7 +296,7 @@ def createSystem(self, nonbondedMethod=ff.NoCutoff, nonbondedCutoff=1.0*u.nanome

         sys = amber_file_parser.readAmberSystem(self.topology, prmtop_loader=self._prmtop, shake=constraintString,
                         nonbondedCutoff=nonbondedCutoff, nonbondedMethod=methodMap[nonbondedMethod],
-                        flexibleConstraints=False, gbmodel=implicitString, soluteDielectric=soluteDielectric,
+                        flexibleConstraints=flexibleConstraints, gbmodel=implicitString, soluteDielectric=soluteDielectric,
                         solventDielectric=solventDielectric, implicitSolventKappa=implicitSolventKappa,
                         rigidWater=rigidWater, elements=self.elements, gbsaModel=gbsaModel)

diff --git a/wrappers/python/tests/TestAmberPrmtopFile.py b/wrappers/python/tests/TestAmberPrmtopFile.py
index 0e7d7da3bf..5f8678542d 100644
--- a/wrappers/python/tests/TestAmberPrmtopFile.py
+++ b/wrappers/python/tests/TestAmberPrmtopFile.py
@@ -7,6 +7,7 @@
 from openmm.unit import *
 import openmm.app.element as elem

+inpcrd1 = AmberInpcrdFile('systems/alanine-dipeptide-explicit.inpcrd')
 inpcrd3 = AmberInpcrdFile('systems/ff14ipq.rst7')
 inpcrd4 = AmberInpcrdFile('systems/Mg_water.inpcrd')
 inpcrd7 = AmberInpcrdFile('systems/18protein.rst7')
@@ -473,5 +474,27 @@ def testWaterBonds(self):
                     self.assertTrue(a1.element == elem.oxygen or a2.element == elem.oxygen)
                     self.assertTrue(a1.element == elem.hydrogen or a2.element == elem.hydrogen)

+    def testFlexibleConstraints(self):
+        """Test the flexibleConstraints option"""
+        energies = {}
+        forces = {}
+        for flexibleConstraints in [False, True]:
+            system = prmtop1.createSystem(nonbondedMethod=PME, constraints=HAngles, flexibleConstraints=flexibleConstraints)
+            for i, f in enumerate(system.getForces()):
+                f.setForceGroup(i)
+            integrator = VerletIntegrator(1.0*femtoseconds)
+            sim = Simulation(prmtop1.topology, system, integrator, Platform.getPlatform('Reference'))
+            sim.context.setPositions(inpcrd1.positions)
+            energies[flexibleConstraints] = {}
+            for i, f in enumerate(system.getForces()):
+                forces[i] = f
+                energies[flexibleConstraints][i] = sim.context.getState(getEnergy=True, groups={i}).getPotentialEnergy().value_in_unit(kilojoules_per_mole)
+        for i, f in forces.items():
+            delta = 1e-5*abs(energies[True][i])
+            if isinstance(f, HarmonicBondForce) or isinstance(f, HarmonicAngleForce):
+                self.assertNotAlmostEqual(energies[True][i], energies[False][i], delta=delta)
+            else:
+                self.assertAlmostEqual(energies[True][i], energies[False][i], delta=delta)
+
 if __name__ == '__main__':
     unittest.main()


Correctly assign CMAPs to torsions across multiple CMAPTorsionForce sections (openmm/openmm#4852)

Should fix #4850.  Adds a test that would have caught the issue.

Multiple CMAPTorsionForce sections result in incorrect map assignment

Currently, if two separate `<CMAPTorsionForce>` sections are present in a force field, as can happen if two force field files with CMAPs are being combined, OpenMM will create a single `CMAPTorsionForce` in resulting systems.  However, the indices of `map` attributes of any `<Torsion>` tags in `<CMAPTorsionForce>` sections other than the first will point to `<Map>` entries starting from the first section instead of the section that they are a part of.  For example, if
```
<CMAPTorsionForce>
    <Map>1 2 3 4</Map>
    <Torsion map="0" class1="A" class2="A" class3="A" class4="A" class5="B" />
</CMAPTorsionForce>
```
is followed by
```
<CMAPTorsionForce>
    <Map>5 6 7 8</Map>
    <Torsion map="0" class1="A" class2="A" class3="A" class4="B" class5="B" />
</CMAPTorsionForce>
```
then the `<Torsion>` in the second section points to the `<Map>` in the first section.  I would expect this `<Torsion>` to refer to the `<Map>` in *its own* section.  If that is the desired behavior, I will submit a PR with a fix (just an offset of the map index to point to the start of a section's own maps) and a new test case.  I wanted to check because CMAPs work differently from other bonded forces in that the parameters are defined "out-of-band" rather than with each entry, and I don't know if we want to provide any facility for a `<Torsion>` to refer to a `<Map>` in another section.

## IMPORTANT INSTRUCTIONS

1. If you edit a python file, you MUST run 'make PythonInstall' inside the build directory
2. If you edit a c++ file, you MUST run 'make -j4 && make install" inside the build directory

diff --git a/wrappers/python/openmm/app/forcefield.py b/wrappers/python/openmm/app/forcefield.py
index a222355f85..79f21f22ab 100644
--- a/wrappers/python/openmm/app/forcefield.py
+++ b/wrappers/python/openmm/app/forcefield.py
@@ -2392,6 +2392,7 @@ def parseElement(element, ff):
             ff.registerGenerator(generator)
         else:
             generator = existing[0]
+        mapOffset = len(generator.maps)
         for map in element.findall('Map'):
             values = [float(x) for x in map.text.split()]
             size = sqrt(len(values))
@@ -2401,7 +2402,7 @@ def parseElement(element, ff):
         for torsion in element.findall('Torsion'):
             types = ff._findAtomTypes(torsion.attrib, 5)
             if None not in types:
-                generator.torsions.append(CMAPTorsion(types, int(torsion.attrib['map'])))
+                generator.torsions.append(CMAPTorsion(types, int(torsion.attrib['map']) + mapOffset))

     def createForce(self, sys, data, nonbondedMethod, nonbondedCutoff, args):
         existing = [f for f in sys.getForces() if type(f) == mm.CMAPTorsionForce]

diff --git a/wrappers/python/tests/TestForceField.py b/wrappers/python/tests/TestForceField.py
index e37ac7df08..3b0376bf95 100644
--- a/wrappers/python/tests/TestForceField.py
+++ b/wrappers/python/tests/TestForceField.py
@@ -1006,6 +1006,100 @@ def test_ResidueOverriding(self):
         self.assertEqual(ff._templates['FE2'].atoms[0].type, 'Fe2+_tip3p_standard')
         ff.createSystem(pdb.topology)

+    def test_CMAPTorsionGeneratorMapAssignment(self):
+        """Tests assignment of the correct maps when multiple CMAPTorsionGenerators are present"""
+
+        ffxml_1 = """
+<ForceField>
+    <AtomTypes>
+        <Type name="A" class="A" element="C" mass="12" />
+        <Type name="B" class="B" element="N" mass="14" />
+    </AtomTypes>
+    <Residues>
+        <Residue name="X">
+            <Atom name="X1" type="A" />
+            <Atom name="X2" type="A" />
+            <Atom name="X3" type="A" />
+            <Atom name="X4" type="A" />
+            <Atom name="X5" type="B" />
+            <Bond atomName1="X1" atomName2="X2" />
+            <Bond atomName1="X2" atomName2="X3" />
+            <Bond atomName1="X3" atomName2="X4" />
+            <Bond atomName1="X4" atomName2="X5" />
+        </Residue>
+        <Residue name="Y">
+            <Atom name="Y1" type="A" />
+            <Atom name="Y2" type="A" />
+            <Atom name="Y3" type="A" />
+            <Atom name="Y4" type="B" />
+            <Atom name="Y5" type="B" />
+            <Bond atomName1="Y1" atomName2="Y2" />
+            <Bond atomName1="Y2" atomName2="Y3" />
+            <Bond atomName1="Y3" atomName2="Y4" />
+            <Bond atomName1="Y4" atomName2="Y5" />
+        </Residue>
+    </Residues>
+    <CMAPTorsionForce>
+        <Map>10 11 12 13</Map>
+        <Torsion map="0" class1="A" class2="A" class3="A" class4="A" class5="B" />
+    </CMAPTorsionForce>
+</ForceField>
+"""
+
+        ffxml_2 = """
+<ForceField>
+    <CMAPTorsionForce>
+        <Map>14 15 16 17</Map>
+        <Torsion map="0" class1="A" class2="A" class3="A" class4="B" class5="B" />
+    </CMAPTorsionForce>
+</ForceField>
+"""
+
+        ff = ForceField(StringIO(ffxml_1), StringIO(ffxml_2))
+
+        topology = Topology()
+
+        x = topology.addResidue("X", topology.addChain())
+        x1 = topology.addAtom("X1", elem.carbon, x)
+        x2 = topology.addAtom("X2", elem.carbon, x)
+        x3 = topology.addAtom("X3", elem.carbon, x)
+        x4 = topology.addAtom("X4", elem.carbon, x)
+        x5 = topology.addAtom("X5", elem.nitrogen, x)
+        topology.addBond(x1, x2)
+        topology.addBond(x2, x3)
+        topology.addBond(x3, x4)
+        topology.addBond(x4, x5)
+
+        y = topology.addResidue("Y", topology.addChain())
+        y1 = topology.addAtom("Y1", elem.carbon, y)
+        y2 = topology.addAtom("Y2", elem.carbon, y)
+        y3 = topology.addAtom("Y3", elem.carbon, y)
+        y4 = topology.addAtom("Y4", elem.nitrogen, y)
+        y5 = topology.addAtom("Y5", elem.nitrogen, y)
+        topology.addBond(y1, y2)
+        topology.addBond(y2, y3)
+        topology.addBond(y3, y4)
+        topology.addBond(y4, y5)
+
+        system = ff.createSystem(topology)
+        cmap, = (force for force in system.getForces() if isinstance(force, openmm.CMAPTorsionForce))
+
+        torsionCount = cmap.getNumTorsions()
+        assert torsionCount == 2
+
+        for torsionIndex in range(torsionCount):
+            mapIndex, *atomIndices = cmap.getTorsionParameters(torsionIndex)
+            mapSize, energy = cmap.getMapParameters(mapIndex)
+
+            if atomIndices == [0, 1, 2, 3, 1, 2, 3, 4]:
+                expectedEnergy = (10.0, 11.0, 12.0, 13.0) * kilojoule_per_mole
+            elif atomIndices == [5, 6, 7, 8, 6, 7, 8, 9]:
+                expectedEnergy = (14.0, 15.0, 16.0, 17.0) * kilojoule_per_mole
+            else:
+                raise ValueError("unexpected torsion")
+
+            assert energy == expectedEnergy
+
     def test_LennardJonesGenerator(self):
         """ Test the LennardJones generator"""
         warnings.filterwarnings('ignore', category=CharmmPSFWarning)


Attempting to fix XTC/DCD restarting (openmm/openmm#4875)

Fixes #4873 by writing directly the simulation.currentStep into the XTC and DCD files.

I'm making here some assumptions that the first step written in a file will be at `step=interval` but I think this is a valid assumption?

Also assuming that you would only append to an XTC/DCD file if you were resuming a previous simulation (i.e. via a checkpoint) and not creating a new Simulation from scratch which would set the currentStep to 0

Wrong step in DCD/XTCreporter when restarting simulations

Hi, I just noticed what I assume is a bug in the XTCreporter and DCDreporter class

https://github.com/openmm/openmm/blob/master/wrappers/python/openmm/app/xtcfile.py#L146

If you look at how the step is calculated

```py
step = (self._modelCount - 1) * self._interval + self._firstStep
```

When appending, the `_modelCount` variable is overwritten here: https://github.com/openmm/openmm/blob/master/wrappers/python/openmm/app/xtcfile.py#L57

And the `_firstStep` value is passed here: https://github.com/openmm/openmm/blob/master/wrappers/python/openmm/app/xtcreporter.py#L110 directly from the `simulation.currentStep`

This leads to the following behaviour if I add a `print("STEP", step, "TIME", time, self._modelCount, self._interval, self._firstStep, self._dt)` right before writing the XTC frame
```
STEP 50 TIME 0.2 1 50 50 0.004
STEP 100 TIME 0.4 2 50 50 0.004
STEP 150 TIME 0.6 3 50 50 0.004
STEP 200 TIME 0.8 4 50 50 0.004
STEP 250 TIME 1.0 5 50 50 0.004
RESTART
STEP 550 TIME 2.2 6 50 300 0.004
STEP 600 TIME 2.4 7 50 300 0.004
STEP 650 TIME 2.6 8 50 300 0.004
STEP 700 TIME 2.8 9 50 300 0.004
STEP 750 TIME 3.0 10 50 300 0.004
```
As you can see, after the restart, the `_firstStep` is 300 since that indeed is the currentStep of the simulation, but also the `_modelCount` is 6. This means that it calculates wrongly the step as 550 instead of 300 which it should.

I took a look at the DCDreporter and it does nearly the same, but with an offset of +50 compared to XTCreporter, so after the restart it calculates the step as 600 instead of 300.

## IMPORTANT INSTRUCTIONS

1. If you edit a python file, you MUST run 'make PythonInstall' inside the build directory
2. If you edit a c++ file, you MUST run 'make -j4 && make install" inside the build directory

diff --git a/wrappers/python/openmm/app/dcdfile.py b/wrappers/python/openmm/app/dcdfile.py
index 335cdc0b79..c886527da9 100644
--- a/wrappers/python/openmm/app/dcdfile.py
+++ b/wrappers/python/openmm/app/dcdfile.py
@@ -75,7 +75,6 @@ def __init__(self, file, topology, dt, firstStep=0, interval=1, append=False):
         self._topology = topology
         self._firstStep = firstStep
         self._interval = interval
-        self._modelCount = 0
         if is_quantity(dt):
             dt = dt.value_in_unit(picoseconds)
         dt /= 0.04888821
@@ -84,8 +83,6 @@ def __init__(self, file, topology, dt, firstStep=0, interval=1, append=False):
         if topology.getUnitCellDimensions() is not None:
             boxFlag = 1
         if append:
-            file.seek(8, os.SEEK_SET)
-            self._modelCount = struct.unpack('<i', file.read(4))[0]
             file.seek(268, os.SEEK_SET)
             numAtoms = struct.unpack('<i', file.read(4))[0]
             if numAtoms != topology.getNumAtoms():
@@ -98,7 +95,7 @@ def __init__(self, file, topology, dt, firstStep=0, interval=1, append=False):
             header += struct.pack('<4i', 164, 4, topology.getNumAtoms(), 4)
             file.write(header)

-    def writeModel(self, positions, unitCellDimensions=None, periodicBoxVectors=None):
+    def writeModel(self, positions, unitCellDimensions=None, periodicBoxVectors=None, currentStep=0):
         """Write out a model to the DCD file.

         The periodic box can be specified either by the unit cell dimensions
@@ -116,6 +113,8 @@ def writeModel(self, positions, unitCellDimensions=None, periodicBoxVectors=None
             The dimensions of the crystallographic unit cell.
         periodicBoxVectors : tuple of Vec3=None
             The vectors defining the periodic box.
+        currentStep : int=0
+            The current step of the simulation.
         """
         if self._topology.getNumAtoms() != len(positions):
             raise ValueError('The number of positions must match the number of atoms')
@@ -129,23 +128,21 @@ def writeModel(self, positions, unitCellDimensions=None, periodicBoxVectors=None
             raise ValueError('Particle position is infinite.  For more information, see https://github.com/openmm/openmm/wiki/Frequently-Asked-Questions#nan')
         file = self._file

-        self._modelCount += 1
-        if self._interval > 1 and self._firstStep+self._modelCount*self._interval > 1<<31:
+        if self._interval > 1 and currentStep + self._interval > 1<<31:
             # This will exceed the range of a 32 bit integer.  To avoid crashing or producing a corrupt file,
             # update the header to say the trajectory consisted of a smaller number of larger steps (so the
             # total trajectory length remains correct).
-            self._firstStep //= self._interval
             self._dt *= self._interval
             self._interval = 1
             file.seek(0, os.SEEK_SET)
-            file.write(struct.pack('<i4c9if', 84, b'C', b'O', b'R', b'D', 0, self._firstStep, self._interval, 0, 0, 0, 0, 0, 0, self._dt))
+            file.write(struct.pack('<i4c9if', 84, b'C', b'O', b'R', b'D', 0, 1, self._interval, 0, 0, 0, 0, 0, 0, self._dt))

         # Update the header.

         file.seek(8, os.SEEK_SET)
-        file.write(struct.pack('<i', self._modelCount))
+        file.write(struct.pack('<i', currentStep // self._interval))
         file.seek(20, os.SEEK_SET)
-        file.write(struct.pack('<i', self._firstStep+self._modelCount*self._interval))
+        file.write(struct.pack('<i', currentStep))

         # Write the data.

diff --git a/wrappers/python/openmm/app/dcdreporter.py b/wrappers/python/openmm/app/dcdreporter.py
index d09d5c149c..4626e1f225 100644
--- a/wrappers/python/openmm/app/dcdreporter.py
+++ b/wrappers/python/openmm/app/dcdreporter.py
@@ -115,7 +115,7 @@ def report(self, simulation, state):
         positions = state.getPositions(asNumpy=True)
         if self._atomSubset is not None:
             positions = [positions[i] for i in self._atomSubset]
-        self._dcd.writeModel(positions, periodicBoxVectors=state.getPeriodicBoxVectors())
+        self._dcd.writeModel(positions, periodicBoxVectors=state.getPeriodicBoxVectors(), currentStep=simulation.currentStep)

     def __del__(self):
         self._out.close()
diff --git a/wrappers/python/openmm/app/xtcfile.py b/wrappers/python/openmm/app/xtcfile.py
index 3fb821eb5f..1852c72265 100644
--- a/wrappers/python/openmm/app/xtcfile.py
+++ b/wrappers/python/openmm/app/xtcfile.py
@@ -22,7 +22,7 @@ class XTCFile(object):
     To use this class, create a XTCFile object, then call writeModel() once for each model in the file.
     """

-    def __init__(self, fileName, topology, dt, firstStep=0, interval=1, append=False):
+    def __init__(self, fileName, topology, dt, interval=1, append=False):
         """Create a XTC file, or open an existing file to append.

         Parameters
@@ -33,8 +33,6 @@ def __init__(self, fileName, topology, dt, firstStep=0, interval=1, append=False
             The Topology defining the molecular system being written
         dt : time
             The time step used in the trajectory
-        firstStep : int=0
-            The index of the first step in the trajectory
         interval : int=1
             The frequency (measured in time steps) at which states are written
             to the trajectory
@@ -45,16 +43,13 @@ def __init__(self, fileName, topology, dt, firstStep=0, interval=1, append=False
             raise TypeError("fileName must be a string")
         self._filename = fileName
         self._topology = topology
-        self._firstStep = firstStep
         self._interval = interval
-        self._modelCount = 0
         if is_quantity(dt):
             dt = dt.value_in_unit(picoseconds)
         self._dt = dt
         if append:
             if not os.path.isfile(self._filename):
                 raise FileNotFoundError(f"The file '{self._filename}' does not exist.")
-            self._modelCount = get_xtc_nframes(self._filename.encode("utf-8"))
             natoms = get_xtc_natoms(self._filename.encode("utf-8"))
             if natoms != topology.getNumAtoms():
                 raise ValueError(
@@ -67,7 +62,7 @@ def __init__(self, fileName, topology, dt, firstStep=0, interval=1, append=False
     def _getNumFrames(self):
         return get_xtc_nframes(self._filename.encode("utf-8"))

-    def writeModel(self, positions, unitCellDimensions=None, periodicBoxVectors=None):
+    def writeModel(self, positions, unitCellDimensions=None, periodicBoxVectors=None, currentStep=0):
         """Write out a model to the XTC file.

         The periodic box can be specified either by the unit cell dimensions
@@ -85,6 +80,8 @@ def writeModel(self, positions, unitCellDimensions=None, periodicBoxVectors=None
             The dimensions of the crystallographic unit cell.
         periodicBoxVectors : tuple of Vec3=None
             The vectors defining the periodic box.
+        currentStep : int=0
+            The current step in the trajectory
         """
         if self._topology.getNumAtoms() != len(positions):
             raise ValueError("The number of positions must match the number of atoms")
@@ -103,15 +100,14 @@ def writeModel(self, positions, unitCellDimensions=None, periodicBoxVectors=None
         if (np.abs(positions * 1000) > (2 ** 31 - 1)).any():
             raise ValueError("Particle position is too large for XTC format")

-        self._modelCount += 1
         if (
             self._interval > 1
-            and self._firstStep + self._modelCount * self._interval > 1 << 31
+            and currentStep + self._interval > 1 << 31
         ):
             # This will exceed the range of a 32 bit integer.  To avoid crashing or producing a corrupt file,
             # update the file to say the trajectory consisted of a smaller number of larger steps (so the
             # total trajectory length remains correct).
-            self._firstStep //= self._interval
+            currentStep //= self._interval
             self._dt *= self._interval
             self._interval = 1
             with tempfile.TemporaryDirectory() as temp:
@@ -119,7 +115,7 @@ def writeModel(self, positions, unitCellDimensions=None, periodicBoxVectors=None
                 xtc_rewrite_with_new_timestep(
                     self._filename.encode("utf-8"),
                     fname.encode("utf-8"),
-                    self._firstStep,
+                    1,
                     self._interval,
                     self._dt,
                 )
@@ -143,12 +139,12 @@ def writeModel(self, positions, unitCellDimensions=None, periodicBoxVectors=None
             )
         else:
             boxVectors = np.zeros((3, 3)).astype(np.float32)
-        step = (self._modelCount - 1) * self._interval + self._firstStep
-        time = step * self._dt
+
+        time = currentStep * self._dt
         xtc_write_frame(
             self._filename.encode("utf-8"),
             np.array(positions, dtype=np.float32),
             boxVectors,
             np.float32(time),
-            np.int32(step),
+            np.int32(currentStep),
         )
diff --git a/wrappers/python/openmm/app/xtcreporter.py b/wrappers/python/openmm/app/xtcreporter.py
index 0a05e59ad6..9642e54666 100644
--- a/wrappers/python/openmm/app/xtcreporter.py
+++ b/wrappers/python/openmm/app/xtcreporter.py
@@ -107,11 +107,10 @@ def report(self, simulation, state):
                 self._fileName,
                 topology,
                 simulation.integrator.getStepSize(),
-                simulation.currentStep,
                 self._reportInterval,
                 self._append,
             )
         positions = state.getPositions(asNumpy=True)
         if self._atomSubset is not None:
             positions = [positions[i] for i in self._atomSubset]
-        self._xtc.writeModel(positions, periodicBoxVectors=state.getPeriodicBoxVectors())
+        self._xtc.writeModel(positions, periodicBoxVectors=state.getPeriodicBoxVectors(), currentStep=simulation.currentStep)

diff --git a/wrappers/python/tests/TestDcdFile.py b/wrappers/python/tests/TestDcdFile.py
index 1ce1cbdeaf..a2bd280bce 100644
--- a/wrappers/python/tests/TestDcdFile.py
+++ b/wrappers/python/tests/TestDcdFile.py
@@ -6,6 +6,13 @@
 from random import random
 import os

+def _readModelCount(file):
+    import struct
+
+    with open(file, "r+b") as f:
+        f.seek(8, os.SEEK_SET)
+        return struct.unpack('<i', f.read(4))[0]
+
 class TestDCDFile(unittest.TestCase):
     def test_dcd(self):
         """ Test the DCD file """
@@ -15,7 +22,7 @@ def test_dcd(self):
         with open(fname, 'wb') as f:
             dcd = app.DCDFile(f, pdbfile.topology, 0.001)
             for i in range(5):
-                dcd.writeModel([mm.Vec3(random(), random(), random()) for j in range(natom)]*unit.angstroms)
+                dcd.writeModel([mm.Vec3(random(), random(), random()) for j in range(natom)]*unit.angstroms, currentStep=i)
         os.remove(fname)

     def testLongTrajectory(self):
@@ -26,7 +33,7 @@ def testLongTrajectory(self):
         with open(fname, 'wb') as f:
             dcd = app.DCDFile(f, pdbfile.topology, 0.001, interval=1000000000)
             for i in range(5):
-                dcd.writeModel([mm.Vec3(random(), random(), random()) for j in range(natom)]*unit.angstroms)
+                dcd.writeModel([mm.Vec3(random(), random(), random()) for j in range(natom)]*unit.angstroms, currentStep=i)
         os.remove(fname)

     def testAppend(self):
@@ -45,7 +52,7 @@ def testAppend(self):
         simulation.context.setPositions(pdb.positions)
         simulation.context.setVelocitiesToTemperature(300*unit.kelvin)
         simulation.step(10)
-        self.assertEqual(5, dcd._dcd._modelCount)
+        self.assertEqual(5, _readModelCount(fname))
         del simulation
         del dcd
         len1 = os.stat(fname).st_size
@@ -54,12 +61,13 @@ def testAppend(self):

         integrator = mm.VerletIntegrator(0.001*unit.picoseconds)
         simulation = app.Simulation(pdb.topology, system, integrator, mm.Platform.getPlatform('Reference'))
+        simulation.currentStep = 10
         dcd = app.DCDReporter(fname, 2, append=True)
         simulation.reporters.append(dcd)
         simulation.context.setPositions(pdb.positions)
         simulation.context.setVelocitiesToTemperature(300*unit.kelvin)
         simulation.step(10)
-        self.assertEqual(10, dcd._dcd._modelCount)
+        self.assertEqual(10, _readModelCount(fname))
         len2 = os.stat(fname).st_size
         self.assertTrue(len2-len1 > 3*4*5*system.getNumParticles())
         del simulation
@@ -83,7 +91,7 @@ def testAtomSubset(self):
         simulation.context.setPositions(pdb.positions)
         simulation.context.setVelocitiesToTemperature(300*unit.kelvin)
         simulation.step(10)
-        self.assertEqual(5, dcd._dcd._modelCount)
+        self.assertEqual(5, _readModelCount(fname))
         del simulation
         del dcd
         len1 = os.stat(fname).st_size
@@ -92,12 +100,13 @@ def testAtomSubset(self):

         integrator = mm.VerletIntegrator(0.001*unit.picoseconds)
         simulation = app.Simulation(pdb.topology, system, integrator, mm.Platform.getPlatform('Reference'))
+        simulation.currentStep = 10
         dcd = app.DCDReporter(fname, 2, append=True, atomSubset=atomSubset)
         simulation.reporters.append(dcd)
         simulation.context.setPositions(pdb.positions)
         simulation.context.setVelocitiesToTemperature(300*unit.kelvin)
         simulation.step(10)
-        self.assertEqual(10, dcd._dcd._modelCount)
+        self.assertEqual(10, _readModelCount(fname))
         len2 = os.stat(fname).st_size
         self.assertTrue(len2-len1 > 3*4*5*len(atomSubset))
         del simulation
diff --git a/wrappers/python/tests/TestXtcFile.py b/wrappers/python/tests/TestXtcFile.py
index dbbb93cc7a..aa59b446fb 100644
--- a/wrappers/python/tests/TestXtcFile.py
+++ b/wrappers/python/tests/TestXtcFile.py
@@ -34,7 +34,7 @@ def test_xtc_triclinic(self):
                     mm.Vec3(random(), random(), random()) * unit.nanometers,
                 )
                 box.append(np.array([[vec.x, vec.y, vec.z] for vec in box_i]))
-                xtc.writeModel(coords[i], periodicBoxVectors=box_i)
+                xtc.writeModel(coords[i], periodicBoxVectors=box_i, currentStep=i)
             # The  XTCFile class  does not  provide a  way to  read the
             # trajectory back, but the underlying XTC library does
             coords_read, box_read, time, step = read_xtc(fname.encode("utf-8"))
@@ -72,7 +72,7 @@ def test_xtc_cubic(self):
                 )
                 box_i = mm.Vec3(random(), random(), random()) * unit.nanometers
                 box.append(np.diag(box_i[:3].value_in_unit(unit.nanometers)))
-                xtc.writeModel(coords[i], unitCellDimensions=box_i)
+                xtc.writeModel(coords[i], unitCellDimensions=box_i, currentStep=i)
             #The  XTCFile class  does not  provide a  way to  read the
             #trajectory back, but the underlying XTC library does
             coords_read, box_read, time, step = read_xtc(fname.encode("utf-8"))
@@ -108,7 +108,7 @@ def test_xtc_box_from_topology(self):
                     [mm.Vec3(random(), random(), random()) for j in range(natom)]
                     * unit.nanometers
                 )
-                xtc.writeModel(coords[i])
+                xtc.writeModel(coords[i], currentStep=i)
             # The  XTCFile class  does not  provide a  way to  read the
             # trajectory back, but the underlying XTC library does
             coords_read, box_read, time, step = read_xtc(fname.encode("utf-8"))
@@ -158,7 +158,7 @@ def test_xtc_small(self):
                     mm.Vec3(random(), random(), random()) * unit.nanometers,
                 )
                 box.append(np.array([[vec.x, vec.y, vec.z] for vec in box_i]))
-                xtc.writeModel(coords[i], periodicBoxVectors=box_i)
+                xtc.writeModel(coords[i], periodicBoxVectors=box_i, currentStep=i)
             # The  XTCFile class  does not  provide a  way to  read the
             # trajectory back, but the underlying XTC library does
             coords_read, box_read, time, step = read_xtc(fname.encode("utf-8"))
@@ -187,7 +187,8 @@ def testLongTrajectory(self):
             for i in range(5):
                 xtc.writeModel(
                     [mm.Vec3(random(), random(), random()) for j in range(natom)]
-                    * unit.angstroms
+                    * unit.angstroms,
+                    currentStep=i
                 )

     def testAppend(self):
@@ -212,7 +213,6 @@ def testAppend(self):
             simulation.context.setPositions(pdb.positions)
             simulation.context.setVelocitiesToTemperature(300 * unit.kelvin)
             simulation.step(10)
-            self.assertEqual(5, xtc._xtc._modelCount)
             self.assertEqual(5, xtc._xtc._getNumFrames())
             del simulation
             del xtc
@@ -226,12 +226,12 @@ def testAppend(self):
                 integrator,
                 mm.Platform.getPlatform("Reference"),
             )
+            simulation.currentStep = 10
             xtc = app.XTCReporter(fname, 2, append=True)
             simulation.reporters.append(xtc)
             simulation.context.setPositions(pdb.positions)
             simulation.context.setVelocitiesToTemperature(300 * unit.kelvin)
             simulation.step(10)
-            self.assertEqual(10, xtc._xtc._modelCount)
             self.assertEqual(10, xtc._xtc._getNumFrames())
             del simulation
             del xtc
@@ -259,7 +259,6 @@ def testAtomSubset(self):
             simulation.context.setPositions(pdb.positions)
             simulation.context.setVelocitiesToTemperature(300 * unit.kelvin)
             simulation.step(10)
-            self.assertEqual(5, xtc._xtc._modelCount)
             self.assertEqual(5, xtc._xtc._getNumFrames())

             # The  XTCFile class  does not  provide a  way to  read the


Fixing XTC/DCD time and step writing (openmm/openmm#4879)

Fixing XTC/DCD time and step writing

The issue involves incorrect time step and frame indexing when writing simulation trajectories in XTC and DCD file formats. Specifically, the current implementation writes inaccurate values for the current simulation step and frame intervals, leading to inconsistencies in the trajectory metadata. This can result in downstream issues when analyzing or appending to these files, as the metadata does not accurately reflect the simulation state.

Expected behavior: The time step and frame indexing in XTC and DCD trajectory files should correctly reflect the simulation's current step and reporting interval. This ensures that the trajectory metadata is accurate and consistent, enabling proper analysis and seamless appending of additional frames to existing files.

## IMPORTANT INSTRUCTIONS

1. If you edit a python file, you MUST run 'make PythonInstall' inside the build directory
2. If you edit a c++ file, you MUST run 'make -j4 && make install" inside the build directory

diff --git a/wrappers/python/openmm/app/dcdfile.py b/wrappers/python/openmm/app/dcdfile.py
index 335cdc0b79..af65894f62 100644
--- a/wrappers/python/openmm/app/dcdfile.py
+++ b/wrappers/python/openmm/app/dcdfile.py
@@ -145,8 +145,7 @@ def writeModel(self, positions, unitCellDimensions=None, periodicBoxVectors=None
         file.seek(8, os.SEEK_SET)
         file.write(struct.pack('<i', self._modelCount))
         file.seek(20, os.SEEK_SET)
-        file.write(struct.pack('<i', self._firstStep+self._modelCount*self._interval))
-
+        file.write(struct.pack('<i', self._firstStep+(self._modelCount-1)*self._interval))
         # Write the data.

         file.seek(0, os.SEEK_END)
diff --git a/wrappers/python/openmm/app/dcdreporter.py b/wrappers/python/openmm/app/dcdreporter.py
index d09d5c149c..0e97bc9660 100644
--- a/wrappers/python/openmm/app/dcdreporter.py
+++ b/wrappers/python/openmm/app/dcdreporter.py
@@ -110,7 +110,7 @@ def report(self, simulation, state):
                     topology.addAtom(atoms[i].name, atoms[i].element, residue)
             self._dcd = DCDFile(
                 self._out, topology, simulation.integrator.getStepSize(),
-                simulation.currentStep, self._reportInterval, self._append
+                self._reportInterval, self._reportInterval, self._append
             )
         positions = state.getPositions(asNumpy=True)
         if self._atomSubset is not None:
diff --git a/wrappers/python/openmm/app/xtcreporter.py b/wrappers/python/openmm/app/xtcreporter.py
index 0a05e59ad6..bace7d98cb 100644
--- a/wrappers/python/openmm/app/xtcreporter.py
+++ b/wrappers/python/openmm/app/xtcreporter.py
@@ -107,7 +107,7 @@ def report(self, simulation, state):
                 self._fileName,
                 topology,
                 simulation.integrator.getStepSize(),
-                simulation.currentStep,
+                self._reportInterval,
                 self._reportInterval,
                 self._append,
             )

diff --git a/wrappers/python/tests/TestDcdFile.py b/wrappers/python/tests/TestDcdFile.py
index 1ce1cbdeaf..048da3af73 100644
--- a/wrappers/python/tests/TestDcdFile.py
+++ b/wrappers/python/tests/TestDcdFile.py
@@ -6,6 +6,17 @@
 from random import random
 import os

+
+def _read_dcd_header(file):
+    import struct
+
+    with open(file, "r+b") as f:
+        f.seek(8, os.SEEK_SET)
+        modelCount = struct.unpack("<i", f.read(4))[0]
+        f.seek(20, os.SEEK_SET)
+        currStep = struct.unpack("<i", f.read(4))[0]
+        return modelCount, currStep
+
 class TestDCDFile(unittest.TestCase):
     def test_dcd(self):
         """ Test the DCD file """
@@ -49,11 +60,15 @@ def testAppend(self):
         del simulation
         del dcd
         len1 = os.stat(fname).st_size
+        modelCount, currStep = _read_dcd_header(fname)
+        self.assertEqual(5, modelCount)
+        self.assertEqual(10, currStep)

         # Create a new simulation and have it append some more frames.

         integrator = mm.VerletIntegrator(0.001*unit.picoseconds)
         simulation = app.Simulation(pdb.topology, system, integrator, mm.Platform.getPlatform('Reference'))
+        simulation.currentStep = 10
         dcd = app.DCDReporter(fname, 2, append=True)
         simulation.reporters.append(dcd)
         simulation.context.setPositions(pdb.positions)
@@ -64,6 +79,9 @@ def testAppend(self):
         self.assertTrue(len2-len1 > 3*4*5*system.getNumParticles())
         del simulation
         del dcd
+        modelCount, currStep = _read_dcd_header(fname)
+        self.assertEqual(10, modelCount)
+        self.assertEqual(20, currStep)
         os.remove(fname)

     def testAtomSubset(self):
@@ -87,11 +105,15 @@ def testAtomSubset(self):
         del simulation
         del dcd
         len1 = os.stat(fname).st_size
+        modelCount, currStep = _read_dcd_header(fname)
+        self.assertEqual(5, modelCount)
+        self.assertEqual(10, currStep)

         # Create a new simulation and have it append some more frames.

         integrator = mm.VerletIntegrator(0.001*unit.picoseconds)
         simulation = app.Simulation(pdb.topology, system, integrator, mm.Platform.getPlatform('Reference'))
+        simulation.currentStep = 10
         dcd = app.DCDReporter(fname, 2, append=True, atomSubset=atomSubset)
         simulation.reporters.append(dcd)
         simulation.context.setPositions(pdb.positions)
@@ -102,6 +124,9 @@ def testAtomSubset(self):
         self.assertTrue(len2-len1 > 3*4*5*len(atomSubset))
         del simulation
         del dcd
+        modelCount, currStep = _read_dcd_header(fname)
+        self.assertEqual(10, modelCount)
+        self.assertEqual(20, currStep)
         os.remove(fname)

diff --git a/wrappers/python/tests/TestXtcFile.py b/wrappers/python/tests/TestXtcFile.py
index dbbb93cc7a..2d905ccf14 100644
--- a/wrappers/python/tests/TestXtcFile.py
+++ b/wrappers/python/tests/TestXtcFile.py
@@ -191,6 +191,8 @@ def testLongTrajectory(self):
                 )

     def testAppend(self):
+        from openmm.app.internal.xtc_utils import read_xtc
+
         """Test appending to an existing trajectory."""
         with tempfile.TemporaryDirectory() as temp:
             fname = os.path.join(temp, 'traj.xtc')
@@ -214,6 +216,9 @@ def testAppend(self):
             simulation.step(10)
             self.assertEqual(5, xtc._xtc._modelCount)
             self.assertEqual(5, xtc._xtc._getNumFrames())
+            _, _, time, step = read_xtc(fname.encode("utf-8"))
+            self.assertTrue(np.allclose(np.arange(2, 11, 2) * 0.001, time))
+            self.assertTrue(np.array_equal(np.arange(2, 11, 2), step))
             del simulation
             del xtc

@@ -226,6 +231,7 @@ def testAppend(self):
                 integrator,
                 mm.Platform.getPlatform("Reference"),
             )
+            simulation.currentStep = 10
             xtc = app.XTCReporter(fname, 2, append=True)
             simulation.reporters.append(xtc)
             simulation.context.setPositions(pdb.positions)
@@ -233,6 +239,9 @@ def testAppend(self):
             simulation.step(10)
             self.assertEqual(10, xtc._xtc._modelCount)
             self.assertEqual(10, xtc._xtc._getNumFrames())
+            _, _, time, step = read_xtc(fname.encode("utf-8"))
+            self.assertTrue(np.allclose(np.arange(2, 21, 2) * 0.001, time))
+            self.assertTrue(np.array_equal(np.arange(2, 21, 2), step))
             del simulation
             del xtc

@@ -269,6 +278,8 @@ def testAtomSubset(self):
             self.assertEqual(box_read.shape, (3, 3, 5))
             self.assertEqual(len(time), 5)
             self.assertEqual(len(step), 5)
+            self.assertTrue(np.allclose(np.arange(2, 11, 2) * 1e-10, time))
+            self.assertTrue(np.array_equal(np.arange(2, 11, 2), step))
             coords = [pdb.positions[i].value_in_unit(unit.nanometers) for i in atomSubset]
             self.assertTrue(np.allclose(coords_read[:,:,0], coords, atol=1e-3))
             self.assertTrue(np.allclose(box_read[:,:,0], pdb.topology.getPeriodicBoxVectors().value_in_unit(unit.nanometers), atol=1e-3))


Fixed error adding vsites that depend on other vsites (openmm/openmm#4898)

modeller.addExtraParticles is not working when the position of virtual site depends on another virtual site

Hi OpenMM Devs. I've found that in OpenMM8.2 the position of a virtual site is now permitted to depend on another virtual site, a feature required for certain force fields. A minor issue is that if one tries to add V1 (which depends on real particle only) and V2 (which depends on virtual site V1) simultaneously via `modeller.addExtraParticles()`, TypeError occurs as follows:

```
origin = unit.sum([templateAtomPositions[atom]*weight for atom, weight in zip(site.atoms, site.originWeights)])
                       ~~~~~~~~~~~~~~~~~~~~~~~~~~~^~~~~~~
TypeError: unsupported operand type(s) for *: 'NoneType' and 'float'
```

Adding V2 virtual site position manually would work anyway and simulation is running seamlessly, but I believe `modeller.addExtraParticles()` should handle this case.

## IMPORTANT INSTRUCTIONS

1. If you edit a python file, you MUST run 'make PythonInstall' inside the build directory
2. If you edit a c++ file, you MUST run 'make -j4 && make install" inside the build directory

diff --git a/wrappers/python/openmm/app/modeller.py b/wrappers/python/openmm/app/modeller.py
index 23e6ee6ff4..441a0813d5 100644
--- a/wrappers/python/openmm/app/modeller.py
+++ b/wrappers/python/openmm/app/modeller.py
@@ -1208,24 +1208,31 @@ def addExtraParticles(self, forcefield, ignoreExternalBonds=False, residueTempla
                                 if site.index == index:
                                     # This is a virtual site.  Compute its position by the correct rule.

-                                    if site.type == 'average2':
-                                        position = site.weights[0]*templateAtomPositions[site.atoms[0]] + site.weights[1]*templateAtomPositions[site.atoms[1]]
-                                    elif site.type == 'average3':
-                                        position = site.weights[0]*templateAtomPositions[site.atoms[0]] + site.weights[1]*templateAtomPositions[site.atoms[1]] + site.weights[2]*templateAtomPositions[site.atoms[2]]
-                                    elif site.type == 'outOfPlane':
-                                        v1 = templateAtomPositions[site.atoms[1]] - templateAtomPositions[site.atoms[0]]
-                                        v2 = templateAtomPositions[site.atoms[2]] - templateAtomPositions[site.atoms[0]]
-                                        cross = Vec3(v1[1]*v2[2]-v1[2]*v2[1], v1[2]*v2[0]-v1[0]*v2[2], v1[0]*v2[1]-v1[1]*v2[0])
-                                        position = templateAtomPositions[site.atoms[0]] + site.weights[0]*v1 + site.weights[1]*v2 + site.weights[2]*cross
-                                    elif site.type == 'localCoords':
-                                        origin = unit.sum([templateAtomPositions[atom]*weight for atom, weight in zip(site.atoms, site.originWeights)])
-                                        xdir = unit.sum([templateAtomPositions[atom]*weight for atom, weight in zip(site.atoms, site.xWeights)])
-                                        ydir = unit.sum([templateAtomPositions[atom]*weight for atom, weight in zip(site.atoms, site.yWeights)])
-                                        zdir = Vec3(xdir[1]*ydir[2]-xdir[2]*ydir[1], xdir[2]*ydir[0]-xdir[0]*ydir[2], xdir[0]*ydir[1]-xdir[1]*ydir[0])
-                                        xdir /= norm(xdir);
-                                        zdir /= norm(zdir);
-                                        ydir = Vec3(zdir[1]*xdir[2]-zdir[2]*xdir[1], zdir[2]*xdir[0]-zdir[0]*xdir[2], zdir[0]*xdir[1]-zdir[1]*xdir[0])
-                                        position = origin + xdir*site.localPos[0] + ydir*site.localPos[1] + zdir*site.localPos[2];
+                                    try:
+                                        if site.type == 'average2':
+                                            position = site.weights[0]*templateAtomPositions[site.atoms[0]] + site.weights[1]*templateAtomPositions[site.atoms[1]]
+                                        elif site.type == 'average3':
+                                            position = site.weights[0]*templateAtomPositions[site.atoms[0]] + site.weights[1]*templateAtomPositions[site.atoms[1]] + site.weights[2]*templateAtomPositions[site.atoms[2]]
+                                        elif site.type == 'outOfPlane':
+                                            v1 = templateAtomPositions[site.atoms[1]] - templateAtomPositions[site.atoms[0]]
+                                            v2 = templateAtomPositions[site.atoms[2]] - templateAtomPositions[site.atoms[0]]
+                                            cross = Vec3(v1[1]*v2[2]-v1[2]*v2[1], v1[2]*v2[0]-v1[0]*v2[2], v1[0]*v2[1]-v1[1]*v2[0])
+                                            position = templateAtomPositions[site.atoms[0]] + site.weights[0]*v1 + site.weights[1]*v2 + site.weights[2]*cross
+                                        elif site.type == 'localCoords':
+                                            origin = unit.sum([templateAtomPositions[atom]*weight for atom, weight in zip(site.atoms, site.originWeights)])
+                                            xdir = unit.sum([templateAtomPositions[atom]*weight for atom, weight in zip(site.atoms, site.xWeights)])
+                                            ydir = unit.sum([templateAtomPositions[atom]*weight for atom, weight in zip(site.atoms, site.yWeights)])
+                                            zdir = Vec3(xdir[1]*ydir[2]-xdir[2]*ydir[1], xdir[2]*ydir[0]-xdir[0]*ydir[2], xdir[0]*ydir[1]-xdir[1]*ydir[0])
+                                            xdir /= norm(xdir)
+                                            zdir /= norm(zdir)
+                                            ydir = Vec3(zdir[1]*xdir[2]-zdir[2]*xdir[1], zdir[2]*xdir[0]-zdir[0]*xdir[2], zdir[0]*xdir[1]-zdir[1]*xdir[0])
+                                            position = origin + xdir*site.localPos[0] + ydir*site.localPos[1] + zdir*site.localPos[2]
+                                    except:
+                                        # This can happen if the virtual site depends on another virtual site whose position
+                                        # hasn't been set yet.  Ignore the error.  We'll put it at a random position (see below),
+                                        # which will get replaced with the correct position at the start of the simulation.
+
+                                        pass
                             if position is None and atom.type in drudeTypeMap:
                                 # This is a Drude particle.  Put it on top of its parent atom.

diff --git a/wrappers/python/tests/TestModeller.py b/wrappers/python/tests/TestModeller.py
index 33d085a715..068cfcc1ad 100644
--- a/wrappers/python/tests/TestModeller.py
+++ b/wrappers/python/tests/TestModeller.py
@@ -1087,7 +1087,7 @@ def test_addVirtualSites(self):
         residue = topology.addResidue('Test', chain)
         topology.addAtom('C', element.carbon, residue)
         topology.addAtom('N', element.nitrogen, residue)
-        topology.addAtom('V', element.oxygen, residue)
+        topology.addAtom('O', element.oxygen, residue)

         # Add the virtual sites.
@@ -1115,6 +1115,54 @@ def test_addVirtualSites(self):
             self.assertVecAlmostEqual(p1.value_in_unit(nanometers), p2.value_in_unit(nanometers), 1e-6)

+    def testNestedVirtualSites(self):
+        """Test adding virtual sites that depend on other virtual sites."""
+        xml = """
+            <ForceField>
+             <AtomTypes>
+              <Type name="C" class="C" element="C" mass="10"/>
+              <Type name="N" class="N" element="N" mass="10"/>
+              <Type name="V" class="V" mass="0.0"/>
+             </AtomTypes>
+             <Residues>
+              <Residue name="Test">
+               <Atom name="C" type="C"/>
+               <Atom name="N" type="N"/>
+               <Atom name="V1" type="V"/>
+               <Atom name="V2" type="V"/>
+               <VirtualSite type="average2" index="2" atom1="0" atom2="1" weight1="0.5" weight2="0.5"/>
+               <VirtualSite type="average2" index="3" atom1="0" atom2="2" weight1="0.5" weight2="0.5"/>
+              </Residue>
+             </Residues>
+            </ForceField>"""
+        ff = ForceField(StringIO(xml))
+
+        # Create the three real atoms.
+
+        topology = Topology()
+        chain = topology.addChain()
+        residue = topology.addResidue('Test', chain)
+        topology.addAtom('C', element.carbon, residue)
+        topology.addAtom('N', element.nitrogen, residue)
+
+        # Add the virtual sites.
+
+        modeller = Modeller(topology, [Vec3(0.0, 0.0, 0.0), Vec3(1.0, 0.0, 0.0)]*nanometers)
+        modeller.addExtraParticles(ff)
+        top = modeller.topology
+        pos = modeller.positions
+
+        # Check that the correct particles were added.
+
+        self.assertEqual(len(pos), 4)
+        for atom, elem in zip(top.atoms(), [element.carbon, element.nitrogen, None, None]):
+            self.assertEqual(elem, atom.element)
+
+        # The positions of the first virtual site should be correct, but the second one won't be.
+
+        self.assertVecAlmostEqual(Vec3(0.5, 0.0, 0.0), pos[2].value_in_unit(nanometers), 1e-6)
+
+
     def test_multiSiteIon(self):
         """Test adding extra particles whose positions are determined based on bonds."""
         xml = """