Add benchmarks and runner script for CI

docs/developing.rst

@@ -135,6 +135,38 @@ commonly-used features are:
 	* the :rst:role:`term` role for glossary terms is extended so that the cross-reference will
 	  work even if the link is plural but the glossary entry is singular or vice versa.
 
+Benchmarking and Profiling
+--------------------------
+
+The :file:`tools/benchmarking` folder contains tools to help measure Scenic's performance
+and resource usage. In particular, the :file:`tools/benchmarking/ci` folder contains a set
+of benchmarks available to run as a CI workflow to assess the performance impact of a PR.
+Those with write or triage permissions in the Scenic repository can trigger benchmarking
+by leaving a comment on the PR starting with ``!benchmark``. You can also run benchmarks
+locally using the :file:`run_benchmarking.py` script: use its ``-h`` option for
+instructions. The script uses `pyperf <https://pyperf.readthedocs.io/>`_, which runs the
+benchmarks many times to improve the stability of the results, so benchmarking is slow.
+You may want to use the ``--fast`` option for quick-and-dirty results. Note that
+even with the slow default options, many of the Scenic benchmarks have substantial
+variability, so speed differences on the order of 10% or less are probably not
+significant.
+
+When working on optimizing a particular part of Scenic, it is much faster to focus on one
+or a few suitable Scenic programs rather than constantly re-running the whole benchmark
+suite. Scenic's :option:`--gather-stats` option is convenient for measuring the time and
+sampling iterations required for a single Scenic program. For example:
+
+.. code:: console
+
+	scenic examples/webots/vacuum/vacuum_simple.scenic -v 0 --gather-stats 100
+
+This option is also ideal for running Scenic in a profiler to investigate where time is
+being spent. Using `Pyinstrument <https://github.com/joerick/pyinstrument>`_, for example:
+
+.. code:: console
+
+	pyinstrument -r html -m scenic examples/webots/vacuum/vacuum_simple.scenic -v 0 --gather-stats 100
+
 .. rubric:: Footnotes
 
 .. [#f1] To run the formatters on *all* files, changed or otherwise, use :command:`make format` in the root directory of the repository. But this should not be necessary if you installed the pre-commit hooks and so all files already committed are clean.

docs/options.rst

@@ -134,3 +134,10 @@ Debugging
 	Implies the :option:`-b` option.
 
 	This option can be enabled from the Python API using `scenic.setDebuggingOptions`.
+
+.. option:: --gather-stats
+
+	Collect timing statistics over a specified number of scenes, rather than rendering
+	diagrams. If the number is negative, it is considered a number of rejection sampling
+	iterations rather than scenes (useful to reduce variability, and if the number of
+	iterations required to generate a scene is very large).

tools/benchmarking/ci/distributions.scenic

@@ -0,0 +1,20 @@
+a = Range(0, 1)
+b = Uniform(-a, a)
+c = Normal(b, 1)
+d = TruncatedNormal(min(max(c, -5), 5), a+1, -10, 10)
+e = (Orientation.fromEuler(1, 2, 3) + a).yaw
+
+x = DiscreteRange(0, 2)
+y = Discrete({x: 3, x*x: 5})
+
+l = Uniform([a, b, c], [a, b, c, d], [a, b, c, d, e])
+elem = Uniform(*l[x:])
+
+vf = VectorField("foo", lambda pos: pos.x)
+
+ego = new Object in RectangularRegion((d, y), elem, 10, 10),
+    facing a + (e relative to vf)
+
+vecs = [ego.position, ego.position.rotatedBy(a)]
+shuf = Uniform(vecs, vecs[::-1])
+param p = Uniform(*shuf).x + shuf[0].distanceTo(shuf[1])

tools/benchmarking/ci/driving.scenic

@@ -0,0 +1,19 @@
+
+from utils import getAssetPath
+
+param map = getAssetPath("maps/CARLA/Town01.xodr")
+param use2DMap = True
+param map_options = dict(useCache=False, writeCache=False)
+model scenic.domains.driving.model
+
+selected_road = Uniform(*network.roads)
+selected_lane = Uniform(*selected_road.lanes)
+ego = new Car on selected_lane.centerline
+
+new Pedestrian on visible sidewalk
+
+rightCurb = ego.laneGroup.curb
+spot = new OrientedPoint on visible rightCurb
+badAngle = Uniform(1.0, -1.0) * Range(10, 20) deg
+parkedCar = new Car left of spot by 0.5,
+                facing badAngle relative to roadDirection

tools/benchmarking/ci/driving_2d.scenic

@@ -0,0 +1,20 @@
+# option: mode2D=True
+
+from utils import getAssetPath
+
+param map = getAssetPath("maps/CARLA/Town01.xodr")
+param use2DMap = True
+param map_options = dict(useCache=False, writeCache=False)
+model scenic.domains.driving.model
+
+selected_road = Uniform(*network.roads)
+selected_lane = Uniform(*selected_road.lanes)
+ego = new Car on selected_lane.centerline
+
+new Pedestrian on visible sidewalk
+
+rightCurb = ego.laneGroup.curb
+spot = new OrientedPoint on visible rightCurb
+badAngle = Uniform(1.0, -1.0) * Range(10, 20) deg
+parkedCar = new Car left of spot by 0.5,
+                facing badAngle relative to roadDirection

tools/benchmarking/ci/many_boxes_2d.scenic

@@ -0,0 +1,5 @@
+
+workspace = Workspace(RectangularRegion((0, 0), 0, 100, 100))
+
+for _ in range(10):
+    new Object in workspace, facing toward (0, 0, 0)

tools/benchmarking/ci/many_boxes_3d.scenic

@@ -0,0 +1,5 @@
+
+workspace = Workspace(BoxRegion(dimensions=(100, 100, 100)))
+
+for _ in range(10):
+    new Object in workspace, facing toward (0, 0, 0)

tools/benchmarking/ci/many_planes.scenic

@@ -0,0 +1,11 @@
+from utils import getAssetPath
+
+workspace = Workspace(RectangularRegion(0@0, 0, 100, 100))
+
+plane_shape = MeshShape.fromFile(
+    getAssetPath("meshes/classic_plane.obj.bz2"),
+    initial_rotation=(-90 deg, 0, -10 deg)
+)
+
+for i in range(4):
+    new Object at Range(-40, 40) @ Range(-40, 40), with shape plane_shape

tools/benchmarking/ci/mars.scenic

@@ -0,0 +1,98 @@
+"""Reduced version of the Webots Mars rover example."""
+
+model scenic.simulators.webots.model
+
+from utils import getAssetPath
+
+# Set up workspace
+width = 10
+length = 10
+workspace = Workspace(RectangularRegion(0 @ 0, 0, width, length))
+
+# types of objects
+
+class MarsGround(Ground):
+    width: width
+    length: length
+    color: (0.863 , 0.447, 0.0353)
+    gridSize: 20
+
+class MarsHill(Hill):
+    position: new Point in workspace
+    width: Range(1,2)
+    length: Range(1,2)
+    height: Range(0.1, 0.3)
+    spread: Range(0.2, 0.3)
+    regionContainedIn: everywhere
+
+class Goal(WebotsObject):
+    """Flag indicating the goal location."""
+    width: 0.1
+    length: 0.1
+    webotsType: 'GOAL'
+    color: (0.035 , 0.639, 0.784)
+
+class Rover(WebotsObject):
+    """Mars rover."""
+    width: 0.5
+    length: 0.7
+    height: 0.4
+    webotsType: 'ROVER'
+    rotationOffset: (90 deg, 0, 0)
+
+class Debris(WebotsObject):
+    """Abstract class for debris scattered randomly in the workspace."""
+    # Recess things into the ground slightly by default
+    baseOffset: (0, 0, -self.height/3)
+
+class BigRock(Debris):
+    """Large rock."""
+    shape: MeshShape.fromFile(getAssetPath("meshes/webots_rock_large.obj.bz2"))
+    yaw: Range(0, 360 deg)
+    webotsType: 'ROCK_BIG'
+    positionOffset: Vector(0,0, -self.height/2)
+
+class Rock(Debris):
+    """Small rock."""
+    shape: MeshShape.fromFile(getAssetPath("meshes/webots_rock_small.obj.bz2"))
+    yaw: Range(0, 360 deg)
+    webotsType: 'ROCK_SMALL'
+    positionOffset: Vector(0,0, -self.height/2)
+
+class Pipe(Debris):
+    """Pipe with variable length."""
+    width: 0.2
+    length: Range(0.5, 1.5)
+    height: self.width
+    shape: CylinderShape(initial_rotation=(90 deg, 0, 90 deg))
+    yaw: Range(0, 360 deg)
+    webotsType: 'PIPE'
+    rotationOffset: (90 deg, 0, 90 deg)
+
+    def startDynamicSimulation(self):
+        # Apply variable length
+        self.webotsObject.getField('height').setSFFloat(self.length)
+
+
+# Ground with random gaussian hills
+ground = new MarsGround on (0,0,0), with terrain [new MarsHill for _ in range(15)]
+
+# Ego and goal on ground
+ego = new Rover at (0, -3), on ground, with controller 'sojourner'
+goal = new Goal at (Range(-1, 1), Range(2, 3)), on ground, facing (0,0,0)
+
+# Bottleneck made of two pipes with a rock in between
+bottleneck = new OrientedPoint at ego offset by Range(-0.5, 0.5) @ Range(0.5, 1.5), facing Range(-30, 30) deg
+require abs((angle to goal) - (angle to bottleneck)) <= 10 deg
+new BigRock at bottleneck, on ground
+
+gap = 1.2 * ego.width
+halfGap = gap / 2
+
+leftEdge = new OrientedPoint left of bottleneck by halfGap,
+    facing Range(60, 120) deg relative to bottleneck.heading
+rightEdge = new OrientedPoint right of bottleneck by halfGap,
+    facing Range(-120, -60) deg relative to bottleneck.heading
+
+new Pipe ahead of leftEdge, with length Range(1, 2), on ground, facing leftEdge, with parentOrientation 0
+new Pipe ahead of rightEdge, with length Range(1, 2), on ground, facing rightEdge, with parentOrientation 0

tools/benchmarking/ci/nearby_nonconvex.scenic

@@ -0,0 +1,4 @@
+region = BoxRegion().difference(BoxRegion(dimensions=(0.1, 0.1, 0.1)))
+shape = MeshShape(region.mesh)
+ego = new Object with shape shape
+other = new Object at (Range(0.9, 1.1), 0), with shape shape