Anirudh/s7 runtime and heap fixes

.gitignore

@@ -10,3 +10,7 @@
 *.ipr
 *.iws
 .Rproj.user
+
+# local working artifacts (not part of the repo)
+/archived-runs/
+/_to_delete/

ees/pom.xml

@@ -304,6 +304,8 @@
           <!-- to skip unit test but run integration test use -DskipTests -DskipITs=false -->
           <skipTests>false</skipTests> <!-- don't let -DskipTests influence integration tests -->
           <skipITs>${skipITs}</skipITs><!-- default is to skip; to run use -DskipITs=false -->
+          <!-- avoid out of memory errors (IT tests were previously forked with the JVM default heap, not this project's -Xmx): -->
+          <argLine>-Xmx10g -Xms3g -Djava.awt.headless=true -Dmatsim.preferLocalDtds=true</argLine>
         </configuration>
         <executions>
           <execution>
@@ -327,7 +329,7 @@
           <forkCount>1</forkCount>
           <reuseForks>false</reuseForks>
           <!-- avoid out of memory errors: -->
-          <argLine>-Xmx8g -Xms8g -Djava.awt.headless=true -Dmatsim.preferLocalDtds=true</argLine>
+          <argLine>-Xmx10g -Xms3g -Djava.awt.headless=true -Dmatsim.preferLocalDtds=true</argLine>
         </configuration>
       </plugin>
       <plugin>

ees/scenarios/surf-coast-shire/midweek-in-jan/matsim-config.xml

@@ -17,7 +17,7 @@
 	</module>
 
 	<module name="plans">
-		<param name="inputPlansFile" value="demand-seasonal-40k-50it-epsg32754-7886c91.xml.gz" />
+		<param name="inputPlansFile" value="demand-seasonal-10k-50it-epsg32754-7886c91.xml.gz" />
 	</module>
 
 	<module name="controler">
@@ -35,7 +35,7 @@
 	<module name="qsim">
 		<!-- "start/endTime" of MobSim (00:00:00 == take earliest activity time/ run as long as active vehicles exist) -->
 		<param name="startTime" value="00:00:00" />
-		<param name="endTime" value="23:59:59" />
+		<param name="endTime" value="16:00:00" /> <!-- cut from 23:59:59 per Dhirendra, 2026-08-03: get close to 1min runtime -->
 
 		<param name = "snapshotperiod"	value = "00:00:00"/> <!-- 00:00:00 means NO snapshot writing, 00:00:30 means every 30 secs -->
 

ees/scripts/network_to_geojson.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+"""
+Convert a MATSim network XML (nodes + links) into GeoJSON, for loading
+directly into QGIS or any other GIS tool.
+
+Usage:
+  python3 network_to_geojson.py \
+      --input surf-coast-shire-network-2021-epsg32754.xml.gz \
+      --output-links network-links.geojson \
+      --output-nodes network-nodes.geojson \
+      --epsg 32754
+"""
+import argparse
+import gzip
+import json
+import re
+import sys
+
+NODE_RE = re.compile(r'<node id="([^"]+)" x="([^"]+)" y="([^"]+)"')
+LINK_RE = re.compile(
+    r'<link id="([^"]+)" from="([^"]+)" to="([^"]+)" length="([^"]+)" '
+    r'freespeed="([^"]+)" capacity="([^"]+)" permlanes="([^"]+)"(?: oneway="([^"]+)")?(?: modes="([^"]+)")?'
+)
+
+
+def convert(src, links_out, nodes_out, epsg):
+    nodes = {}
+    links = []
+
+    with gzip.open(src, "rt", encoding="utf-8") as f:
+        for line in f:
+            nm = NODE_RE.search(line)
+            if nm:
+                nid, x, y = nm.groups()
+                nodes[nid] = (float(x), float(y))
+                continue
+            lm = LINK_RE.search(line)
+            if lm:
+                lid, frm, to, length, freespeed, capacity, permlanes, oneway, modes = lm.groups()
+                links.append({
+                    "id": lid, "from": frm, "to": to, "length": float(length),
+                    "freespeed": float(freespeed), "capacity": float(capacity),
+                    "permlanes": float(permlanes), "modes": modes or "",
+                })
+
+    print(f"nodes={len(nodes)} links={len(links)}", file=sys.stderr)
+    crs = {"type": "name", "properties": {"name": f"urn:ogc:def:crs:EPSG::{epsg}"}}
+
+    node_features = [
+        {"type": "Feature", "geometry": {"type": "Point", "coordinates": [x, y]}, "properties": {"id": nid}}
+        for nid, (x, y) in nodes.items()
+    ]
+    with open(nodes_out, "w") as f:
+        json.dump({"type": "FeatureCollection", "name": "network-nodes", "crs": crs, "features": node_features}, f)
+
+    link_features = []
+    skipped = 0
+    for l in links:
+        if l["from"] not in nodes or l["to"] not in nodes:
+            skipped += 1
+            continue
+        x1, y1 = nodes[l["from"]]
+        x2, y2 = nodes[l["to"]]
+        link_features.append({
+            "type": "Feature",
+            "geometry": {"type": "LineString", "coordinates": [[x1, y1], [x2, y2]]},
+            "properties": {
+                "id": l["id"], "from": l["from"], "to": l["to"],
+                "length_m": l["length"], "freespeed_mps": l["freespeed"],
+                "capacity_vph": l["capacity"], "lanes": l["permlanes"], "modes": l["modes"],
+            },
+        })
+    print(f"link features={len(link_features)} skipped(missing node)={skipped}", file=sys.stderr)
+
+    with open(links_out, "w") as f:
+        json.dump({"type": "FeatureCollection", "name": "network-links", "crs": crs, "features": link_features}, f)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--input", required=True, help="source MATSim network .xml.gz")
+    parser.add_argument("--output-links", required=True, help="destination links GeoJSON")
+    parser.add_argument("--output-nodes", required=True, help="destination nodes GeoJSON")
+    parser.add_argument("--epsg", default="32754", help="EPSG code of the network's coordinates (default: 32754)")
+    args = parser.parse_args()
+    convert(args.input, args.output_links, args.output_nodes, args.epsg)
+
+
+if __name__ == "__main__":
+    main()

ees/scripts/reduce_population.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+"""
+Reduce and re-order a MATSim population XML.
+
+Motivation (Aug 2026, S7 debugging with Dhirendra):
+  - The original demand file has person ids sorted alphabetically as
+    strings ("0", "1", "10", "100", "1000", ...) rather than numerically.
+  - For faster IT-test / dev-loop runs we also want a smaller population.
+
+This script keeps only persons with numeric id < --max-agents, and writes
+them back out in ascending numeric id order (which also happens to fix the
+ordering issue above, since ids in this dataset are a contiguous 0..N-1
+range).
+
+Usage:
+  python3 reduce_population.py \
+      --input demand-seasonal-40k-50it-epsg32754-7886c91.xml.gz \
+      --output demand-seasonal-20k-50it-epsg32754-7886c91.xml.gz \
+      --max-agents 20000
+"""
+import argparse
+import gzip
+import re
+import sys
+
+PERSON_START_RE = re.compile(r'<person id="(\d+)"')
+
+
+def reduce_population(src, dst, keep_max):
+    header_lines = []
+    footer_line = None
+    kept = {}
+    state = "header"
+    current_id = None
+    current_lines = []
+    total_persons = 0
+
+    with gzip.open(src, "rt", encoding="utf-8") as f:
+        for line in f:
+            if state == "header":
+                m = PERSON_START_RE.search(line)
+                if m:
+                    state = "body"
+                    current_id = int(m.group(1))
+                    current_lines = [line]
+                else:
+                    header_lines.append(line)
+                continue
+
+            if state == "body":
+                current_lines.append(line)
+                if "</person>" in line:
+                    total_persons += 1
+                    if current_id < keep_max:
+                        kept[current_id] = current_lines
+                    current_lines = []
+                    current_id = None
+                    state = "between"
+                continue
+
+            if state == "between":
+                m = PERSON_START_RE.search(line)
+                if m:
+                    state = "body"
+                    current_id = int(m.group(1))
+                    current_lines = [line]
+                elif "</population>" in line:
+                    footer_line = line
+                    state = "footer"
+                continue
+
+    print(f"Total persons seen: {total_persons}", file=sys.stderr)
+    print(f"Kept (id < {keep_max}): {len(kept)}", file=sys.stderr)
+
+    if footer_line is None:
+        footer_line = "</population>\n"
+
+    with gzip.open(dst, "wt", encoding="utf-8") as out:
+        out.writelines(header_lines)
+        for pid in sorted(kept.keys()):
+            out.writelines(kept[pid])
+            out.write("\n")
+        out.write(footer_line)
+
+    print(f"Wrote {dst}", file=sys.stderr)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--input", required=True, help="source population .xml.gz")
+    parser.add_argument("--output", required=True, help="destination population .xml.gz")
+    parser.add_argument("--max-agents", type=int, default=20000, help="keep persons with numeric id below this value (default: 20000)")
+    args = parser.parse_args()
+    reduce_population(args.input, args.output, args.max_agents)
+
+
+if __name__ == "__main__":
+    main()