Add a LeafSystem for dragging objects from meshcat

bindings/generated_docstrings/geometry.h

@@ -3752,6 +3752,23 @@ your gamepad is working), see
           const char* doc =
 R"""((Advanced) Returns the number of currently-open websocket connections.)""";
         } GetNumActiveConnections;
+        // Symbol: drake::geometry::Meshcat::GetObjectDrag
+        struct /* GetObjectDrag */ {
+          // Source: drake/geometry/meshcat.h
+          const char* doc =
+R"""(Returns the current mouse-drag state if a user is presently dragging
+an object in a connected Meshcat browser, or std∷nullopt otherwise.
+
+A drag is initiated in the browser by holding the <kbd>Ctrl</kbd> key
+and pressing the left mouse button on an object, then moving the
+mouse; releasing the mouse button ends the drag. A downstream system
+(see multibody∷meshcat∷MeshcatMouseSpring) can read this state and
+convert it into a force applied to a simulated body, letting users
+drag objects with the cursor.
+
+If multiple browsers report drags concurrently, the returned value
+reflects the most recently received message.)""";
+        } GetObjectDrag;
         // Symbol: drake::geometry::Meshcat::GetPackedObject
         struct /* GetPackedObject */ {
           // Source: drake/geometry/meshcat.h
@@ -3863,6 +3880,39 @@ it will listen on the first available port starting at 7000 (up to
           const char* doc_1args_params =
 R"""(Constructs the Meshcat instance using the given ``params``.)""";
         } ctor;
+        // Symbol: drake::geometry::Meshcat::ObjectDrag
+        struct /* ObjectDrag */ {
+          // Source: drake/geometry/meshcat.h
+          const char* doc =
+R"""(The state of an in-progress mouse drag of a scene object, as reported
+by a Meshcat browser. See GetObjectDrag().)""";
+          // Symbol: drake::geometry::Meshcat::ObjectDrag::anchor_in_world
+          struct /* anchor_in_world */ {
+            // Source: drake/geometry/meshcat.h
+            const char* doc =
+R"""(The current position of the drag's *attachment point* -- the point on
+the object where the drag began. The browser keeps this point rigidly
+attached to the object, so as the object moves (e.g., under simulated
+physics) this value tracks the world-frame location of that material
+point. Expressed in Drake's z-up world frame (p_WA).)""";
+          } anchor_in_world;
+          // Symbol: drake::geometry::Meshcat::ObjectDrag::path
+          struct /* path */ {
+            // Source: drake/geometry/meshcat.h
+            const char* doc =
+R"""(The "/"-delimited Meshcat path of the object being dragged (e.g.,
+"/drake/visualizer/my_model/my_body/my_geometry").)""";
+          } path;
+          // Symbol: drake::geometry::Meshcat::ObjectDrag::target_in_world
+          struct /* target_in_world */ {
+            // Source: drake/geometry/meshcat.h
+            const char* doc =
+R"""(The current position of the cursor's drag *target*. As the user moves
+the mouse, the cursor is projected into the scene to form this point.
+A virtual spring should pull ``anchor_in_world`` toward this point.
+Expressed in Drake's z-up world frame (p_WT).)""";
+          } target_in_world;
+        } ObjectDrag;
         // Symbol: drake::geometry::Meshcat::OrthographicCamera
         struct /* OrthographicCamera */ {
           // Source: drake/geometry/meshcat.h

bindings/generated_docstrings/multibody_meshcat.h

@@ -16,6 +16,7 @@
 // #include "drake/multibody/meshcat/contact_visualizer_params.h"
 // #include "drake/multibody/meshcat/hydroelastic_contact_visualizer.h"
 // #include "drake/multibody/meshcat/joint_sliders.h"
+// #include "drake/multibody/meshcat/meshcat_mouse_spring.h"
 // #include "drake/multibody/meshcat/point_contact_visualizer.h"
 
 // Symbol: pydrake_doc_multibody_meshcat
@@ -378,6 +379,112 @@ Parameter ``q``:
     MultibodyPlant∷num_positions().)""";
           } SetPositions;
         } JointSliders;
+        // Symbol: drake::multibody::meshcat::MeshcatMouseSpring
+        struct /* MeshcatMouseSpring */ {
+          // Source: drake/multibody/meshcat/meshcat_mouse_spring.h
+          const char* doc =
+R"""(MeshcatMouseSpring lets a user drag the bodies of a MultibodyPlant
+with the mouse in a Meshcat browser: holding Ctrl and dragging a body
+with the left mouse button applies a virtual spring force that pulls
+the grabbed point toward the cursor.
+
+This system reads the drag state from Meshcat (see
+geometry∷Meshcat∷GetObjectDrag()) and outputs a corresponding
+ExternallyAppliedSpatialForce on the dragged body. Connecting that
+output to MultibodyPlant∷get_applied_spatial_force_input_port()
+applies the force; AddToBuilder() performs that connection along with
+the input connections.
+
+.. pydrake_system::
+
+    name: MeshcatMouseSpring
+    input_ports:
+    - body_poses
+    - body_spatial_velocities
+    output_ports:
+    - spatial_forces
+
+The ``body_poses`` and ``body_spatial_velocities`` inputs come from
+the same-named MultibodyPlant output ports.
+
+With ``m`` the dragged body's mass, the applied force (in the world
+frame) is ``m * stiffness * (target - anchor) - m * sqrt(stiffness) *
+v_anchor``, where ``anchor`` is the grabbed point on the body,
+``target`` is the cursor position, and ``v_anchor`` is the world
+velocity of the grabbed point. Scaling by ``m`` makes the
+translational response frequency ``sqrt(stiffness)`` and damping ratio
+independent of the body's mass.
+
+When no drag is in progress the output is empty. Any body with
+geometry published to Meshcat by a geometry∷MeshcatVisualizer can be
+dragged; the world body cannot.
+
+This system is ``double``-only, because Meshcat reports drag state as
+plain doubles and mouse interaction is not meaningful for other scalar
+types.)""";
+          // Symbol: drake::multibody::meshcat::MeshcatMouseSpring::AddToBuilder
+          struct /* AddToBuilder */ {
+            // Source: drake/multibody/meshcat/meshcat_mouse_spring.h
+            const char* doc =
+R"""(Adds a MeshcatMouseSpring to ``builder`` and connects it to `plant`'s
+body-pose and body-spatial-velocity output ports and its
+applied-spatial-force input port. Returns a reference to the
+newly-added system.
+
+Precondition:
+    plant is part of builder and is finalized.
+
+Precondition:
+    `plant`'s applied-spatial-force input port is not already
+    connected.)""";
+          } AddToBuilder;
+          // Symbol: drake::multibody::meshcat::MeshcatMouseSpring::MeshcatMouseSpring
+          struct /* ctor */ {
+            // Source: drake/multibody/meshcat/meshcat_mouse_spring.h
+            const char* doc =
+R"""(Constructs a MeshcatMouseSpring for the given ``plant``.
+
+Parameter ``meshcat``:
+    The Meshcat instance the user will interact with. The pointer is
+    aliased and must outlive this system.
+
+Parameter ``plant``:
+    The MultibodyPlant whose bodies can be dragged. The pointer is
+    aliased and must outlive this system; the plant must already be
+    finalized.
+
+Parameter ``stiffness``:
+    The mass-normalized spring stiffness, in 1/s²; see the class
+    overview for the force it produces.
+
+Precondition:
+    plant->is_finalized() is true.
+
+Precondition:
+    stiffness >= 0.)""";
+          } ctor;
+          // Symbol: drake::multibody::meshcat::MeshcatMouseSpring::get_body_poses_input_port
+          struct /* get_body_poses_input_port */ {
+            // Source: drake/multibody/meshcat/meshcat_mouse_spring.h
+            const char* doc =
+R"""(Returns the input port for the bodies' poses (a
+``std∷vector<math∷RigidTransform<double>>``).)""";
+          } get_body_poses_input_port;
+          // Symbol: drake::multibody::meshcat::MeshcatMouseSpring::get_body_spatial_velocities_input_port
+          struct /* get_body_spatial_velocities_input_port */ {
+            // Source: drake/multibody/meshcat/meshcat_mouse_spring.h
+            const char* doc =
+R"""(Returns the input port for the bodies' spatial velocities (a
+``std∷vector<SpatialVelocity<double>>``).)""";
+          } get_body_spatial_velocities_input_port;
+          // Symbol: drake::multibody::meshcat::MeshcatMouseSpring::get_spatial_forces_output_port
+          struct /* get_spatial_forces_output_port */ {
+            // Source: drake/multibody/meshcat/meshcat_mouse_spring.h
+            const char* doc =
+R"""(Returns the output port for the applied spatial forces (a
+``std∷vector<ExternallyAppliedSpatialForce<double>>``).)""";
+          } get_spatial_forces_output_port;
+        } MeshcatMouseSpring;
       } meshcat;
     } multibody;
     // Symbol: drake::systems

geometry/meshcat.cc

@@ -1901,6 +1901,13 @@ class Meshcat::Impl {
     return gamepad_;
   }
 
+  std::optional<Meshcat::ObjectDrag> GetObjectDrag() const {
+    DRAKE_DEMAND(IsThread(main_thread_id_));
+
+    std::lock_guard<std::mutex> lock(controls_mutex_);
+    return mouse_drag_;
+  }
+
   // This function is for use by the websocket thread. The Meshcat::StaticHtml()
   // and Meshcat::StaticZip() outer functions call into here using appropriate
   // deferred handling.
@@ -2447,6 +2454,21 @@ class Meshcat::Impl {
       gamepad_.axes = std::move(data.gamepad->axes);
       return;
     }
+    if (data.type == "mouse_drag") {
+      if (data.drag_anchor.size() == 3 && data.drag_target.size() == 3) {
+        Meshcat::ObjectDrag drag;
+        drag.path = std::move(data.name);
+        drag.anchor_in_world = Eigen::Vector3d(
+            data.drag_anchor[0], data.drag_anchor[1], data.drag_anchor[2]);
+        drag.target_in_world = Eigen::Vector3d(
+            data.drag_target[0], data.drag_target[1], data.drag_target[2]);
+        mouse_drag_ = std::move(drag);
+      } else {
+        // An empty payload signals the end of a drag (e.g., mouse release).
+        mouse_drag_ = std::nullopt;
+      }
+      return;
+    }
     if (data.type == "camera_pose" && data.camera_pose.size() == 16 &&
         data.is_perspective.has_value()) {
       if (camera_pose_source_ != nullptr && camera_pose_source_ != ws) {
@@ -2541,6 +2563,12 @@ class Meshcat::Impl {
   // The socket for the browser that is sending the camera pose.
   WebSocket* camera_pose_source_{};
   std::optional<math::RigidTransformd> camera_pose_;
+  // The most recently received object-drag state (nullopt when not dragging),
+  // guarded by controls_mutex_. Drag messages are currently accepted from any
+  // browser;
+  // TODO(vincekurtz) add per-socket source tracking and mid-drag disconnect
+  // cleanup alongside the browser-side drag implementation.
+  std::optional<Meshcat::ObjectDrag> mouse_drag_;
 
   // These variables should only be accessed in the main thread, where "main
   // thread" is the thread in which this class was constructed.
@@ -2968,6 +2996,10 @@ Meshcat::Gamepad Meshcat::GetGamepad() const {
   return impl().GetGamepad();
 }
 
+std::optional<Meshcat::ObjectDrag> Meshcat::GetObjectDrag() const {
+  return impl().GetObjectDrag();
+}
+
 std::string Meshcat::StaticHtml() const {
   return impl().StaticHtml();
 }

geometry/meshcat.h

@@ -903,6 +903,41 @@ class Meshcat {
   gamepad is working), see https://beej.us/blog/data/javascript-gamepad/. */
   Gamepad GetGamepad() const;
 
+  /** The state of an in-progress mouse drag of a scene object, as reported by a
+  Meshcat browser. See GetObjectDrag(). */
+  struct ObjectDrag {
+    /** The "/"-delimited Meshcat path of the object being dragged (e.g.,
+    "/drake/visualizer/my_model/my_body/my_geometry"). */
+    std::string path;
+
+    /** The current position of the drag's *attachment point* -- the point on
+    the object where the drag began. The browser keeps this point rigidly
+    attached to the object, so as the object moves (e.g., under simulated
+    physics) this value tracks the world-frame location of that material point.
+    Expressed in Drake's z-up world frame (p_WA). */
+    Eigen::Vector3d anchor_in_world;
+
+    /** The current position of the cursor's drag *target*. As the user moves
+    the mouse, the cursor is projected into the scene to form this point. A
+    virtual spring should pull `anchor_in_world` toward this point. Expressed in
+    Drake's z-up world frame (p_WT). */
+    Eigen::Vector3d target_in_world;
+  };
+
+  /** Returns the current mouse-drag state if a user is presently dragging an
+  object in a connected Meshcat browser, or std::nullopt otherwise.
+
+  A drag is initiated in the browser by holding the <kbd>Ctrl</kbd> key and
+  pressing the left mouse button on an object, then moving the mouse; releasing
+  the mouse button ends the drag. A downstream system (see
+  multibody::meshcat::MeshcatMouseSpring) can read this state and convert it
+  into a force applied to a simulated body, letting users drag objects with the
+  cursor.
+
+  If multiple browsers report drags concurrently, the returned value reflects
+  the most recently received message. */
+  std::optional<ObjectDrag> GetObjectDrag() const;
+
   //@}
 
   /** Returns an HTML string that can be saved to a file for a snapshot of the

geometry/meshcat_types_internal.h

@@ -628,14 +628,28 @@ struct Gamepad {
 //       - If the flattened array doesn't have a valid rotation matrix in its
 //         unflattened upper-left 3x3 sub-matrix, it throws.
 //       - See Drake's meshcat.html for the source of the message.
+//
+//   Object dragging
+//     • Fields
+//       - type = "mouse_drag"
+//       - name = the Meshcat path of the object being dragged.
+//       - drag_anchor = the world-frame attachment point (3 values)
+//       - drag_target = the world-frame cursor target (3 values)
+//     • Semantics
+//       - If drag_anchor and drag_target each have 3 values, the drag state is
+//         updated; an empty payload ends the drag.
+//       - See Meshcat::GetObjectDrag().
 struct UserInterfaceEvent {
   std::string type;
   std::string name;
   std::optional<double> value;
   std::optional<internal::Gamepad> gamepad;
   std::vector<double> camera_pose;
   std::optional<bool> is_perspective{};
-  MSGPACK_DEFINE_MAP(type, name, value, gamepad, camera_pose, is_perspective);
+  std::vector<double> drag_anchor;
+  std::vector<double> drag_target;
+  MSGPACK_DEFINE_MAP(type, name, value, gamepad, camera_pose, is_perspective,
+                     drag_anchor, drag_target);
 };
 
 }  // namespace internal

multibody/meshcat/BUILD.bazel

@@ -23,6 +23,7 @@ drake_cc_package_library(
         ":contact_visualizer_params",
         ":hydroelastic_contact_visualizer",
         ":joint_sliders",
+        ":meshcat_mouse_spring",
         ":point_contact_visualizer",
     ],
 )
@@ -111,6 +112,30 @@ drake_cc_googletest(
     ],
 )
 
+drake_cc_library(
+    name = "meshcat_mouse_spring",
+    srcs = ["meshcat_mouse_spring.cc"],
+    hdrs = ["meshcat_mouse_spring.h"],
+    deps = [
+        "//common:essential",
+        "//geometry:meshcat",
+        "//multibody/plant",
+        "//systems/framework:leaf_system",
+    ],
+)
+
+drake_cc_googletest(
+    name = "meshcat_mouse_spring_test",
+    deps = [
+        ":meshcat_mouse_spring",
+        "//common/test_utilities:eigen_matrix_compare",
+        "//common/test_utilities:expect_throws_message",
+        "//geometry/test_utilities:meshcat_environment",
+        "//multibody/plant",
+        "@msgpack_internal//:msgpack",
+    ],
+)
+
 drake_cc_library(
     name = "point_contact_visualizer",
     srcs = ["point_contact_visualizer.cc"],