Skip to content

Commit 7e254a0

Browse files
committed
Add a LeafSystem for dragging objects from meshcat
Author: Vince Kurtz <vjkurtz@gmail.com> Assisted-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent c6c7016 commit 7e254a0

7 files changed

Lines changed: 656 additions & 1 deletion

File tree

geometry/meshcat.cc

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1901,6 +1901,13 @@ class Meshcat::Impl {
19011901
return gamepad_;
19021902
}
19031903

1904+
std::optional<Meshcat::ObjectDrag> GetObjectDrag() const {
1905+
DRAKE_DEMAND(IsThread(main_thread_id_));
1906+
1907+
std::lock_guard<std::mutex> lock(controls_mutex_);
1908+
return mouse_drag_;
1909+
}
1910+
19041911
// This function is for use by the websocket thread. The Meshcat::StaticHtml()
19051912
// and Meshcat::StaticZip() outer functions call into here using appropriate
19061913
// deferred handling.
@@ -2447,6 +2454,21 @@ class Meshcat::Impl {
24472454
gamepad_.axes = std::move(data.gamepad->axes);
24482455
return;
24492456
}
2457+
if (data.type == "mouse_drag") {
2458+
if (data.drag_anchor.size() == 3 && data.drag_target.size() == 3) {
2459+
Meshcat::ObjectDrag drag;
2460+
drag.path = std::move(data.name);
2461+
drag.anchor_in_world = Eigen::Vector3d(
2462+
data.drag_anchor[0], data.drag_anchor[1], data.drag_anchor[2]);
2463+
drag.target_in_world = Eigen::Vector3d(
2464+
data.drag_target[0], data.drag_target[1], data.drag_target[2]);
2465+
mouse_drag_ = std::move(drag);
2466+
} else {
2467+
// An empty payload signals the end of a drag (e.g., mouse release).
2468+
mouse_drag_ = std::nullopt;
2469+
}
2470+
return;
2471+
}
24502472
if (data.type == "camera_pose" && data.camera_pose.size() == 16 &&
24512473
data.is_perspective.has_value()) {
24522474
if (camera_pose_source_ != nullptr && camera_pose_source_ != ws) {
@@ -2541,6 +2563,12 @@ class Meshcat::Impl {
25412563
// The socket for the browser that is sending the camera pose.
25422564
WebSocket* camera_pose_source_{};
25432565
std::optional<math::RigidTransformd> camera_pose_;
2566+
// The most recently received object-drag state (nullopt when not dragging),
2567+
// guarded by controls_mutex_. Drag messages are currently accepted from any
2568+
// browser;
2569+
// TODO(vincekurtz) add per-socket source tracking and mid-drag disconnect
2570+
// cleanup alongside the browser-side drag implementation.
2571+
std::optional<Meshcat::ObjectDrag> mouse_drag_;
25442572

25452573
// These variables should only be accessed in the main thread, where "main
25462574
// thread" is the thread in which this class was constructed.
@@ -2968,6 +2996,10 @@ Meshcat::Gamepad Meshcat::GetGamepad() const {
29682996
return impl().GetGamepad();
29692997
}
29702998

2999+
std::optional<Meshcat::ObjectDrag> Meshcat::GetObjectDrag() const {
3000+
return impl().GetObjectDrag();
3001+
}
3002+
29713003
std::string Meshcat::StaticHtml() const {
29723004
return impl().StaticHtml();
29733005
}

geometry/meshcat.h

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -903,6 +903,41 @@ class Meshcat {
903903
gamepad is working), see https://beej.us/blog/data/javascript-gamepad/. */
904904
Gamepad GetGamepad() const;
905905

906+
/** The state of an in-progress mouse drag of a scene object, as reported by a
907+
Meshcat browser. See GetObjectDrag(). */
908+
struct ObjectDrag {
909+
/** The "/"-delimited Meshcat path of the object being dragged (e.g.,
910+
"/drake/visualizer/my_model/my_body/my_geometry"). */
911+
std::string path;
912+
913+
/** The current position of the drag's *attachment point* -- the point on
914+
the object where the drag began. The browser keeps this point rigidly
915+
attached to the object, so as the object moves (e.g., under simulated
916+
physics) this value tracks the world-frame location of that material point.
917+
Expressed in Drake's z-up world frame (p_WA). */
918+
Eigen::Vector3d anchor_in_world;
919+
920+
/** The current position of the cursor's drag *target*. As the user moves
921+
the mouse, the cursor is projected into the scene to form this point. A
922+
virtual spring should pull `anchor_in_world` toward this point. Expressed in
923+
Drake's z-up world frame (p_WT). */
924+
Eigen::Vector3d target_in_world;
925+
};
926+
927+
/** Returns the current mouse-drag state if a user is presently dragging an
928+
object in a connected Meshcat browser, or std::nullopt otherwise.
929+
930+
A drag is initiated in the browser by holding the <kbd>Ctrl</kbd> key and
931+
pressing the left mouse button on an object, then moving the mouse; releasing
932+
the mouse button ends the drag. A downstream system (see
933+
multibody::meshcat::MeshcatMouseSpring) can read this state and convert it
934+
into a force applied to a simulated body, letting users drag objects with the
935+
cursor.
936+
937+
If multiple browsers report drags concurrently, the returned value reflects
938+
the most recently received message. */
939+
std::optional<ObjectDrag> GetObjectDrag() const;
940+
906941
//@}
907942

908943
/** Returns an HTML string that can be saved to a file for a snapshot of the

geometry/meshcat_types_internal.h

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -628,14 +628,28 @@ struct Gamepad {
628628
// - If the flattened array doesn't have a valid rotation matrix in its
629629
// unflattened upper-left 3x3 sub-matrix, it throws.
630630
// - See Drake's meshcat.html for the source of the message.
631+
//
632+
// Object dragging
633+
// • Fields
634+
// - type = "mouse_drag"
635+
// - name = the Meshcat path of the object being dragged.
636+
// - drag_anchor = the world-frame attachment point (3 values)
637+
// - drag_target = the world-frame cursor target (3 values)
638+
// • Semantics
639+
// - If drag_anchor and drag_target each have 3 values, the drag state is
640+
// updated; an empty payload ends the drag.
641+
// - See Meshcat::GetObjectDrag().
631642
struct UserInterfaceEvent {
632643
std::string type;
633644
std::string name;
634645
std::optional<double> value;
635646
std::optional<internal::Gamepad> gamepad;
636647
std::vector<double> camera_pose;
637648
std::optional<bool> is_perspective{};
638-
MSGPACK_DEFINE_MAP(type, name, value, gamepad, camera_pose, is_perspective);
649+
std::vector<double> drag_anchor;
650+
std::vector<double> drag_target;
651+
MSGPACK_DEFINE_MAP(type, name, value, gamepad, camera_pose, is_perspective,
652+
drag_anchor, drag_target);
639653
};
640654

641655
} // namespace internal

multibody/meshcat/BUILD.bazel

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ drake_cc_package_library(
2323
":contact_visualizer_params",
2424
":hydroelastic_contact_visualizer",
2525
":joint_sliders",
26+
":meshcat_mouse_spring",
2627
":point_contact_visualizer",
2728
],
2829
)
@@ -111,6 +112,30 @@ drake_cc_googletest(
111112
],
112113
)
113114

115+
drake_cc_library(
116+
name = "meshcat_mouse_spring",
117+
srcs = ["meshcat_mouse_spring.cc"],
118+
hdrs = ["meshcat_mouse_spring.h"],
119+
deps = [
120+
"//common:essential",
121+
"//geometry:meshcat",
122+
"//multibody/plant",
123+
"//systems/framework:leaf_system",
124+
],
125+
)
126+
127+
drake_cc_googletest(
128+
name = "meshcat_mouse_spring_test",
129+
deps = [
130+
":meshcat_mouse_spring",
131+
"//common/test_utilities:eigen_matrix_compare",
132+
"//common/test_utilities:expect_throws_message",
133+
"//geometry/test_utilities:meshcat_environment",
134+
"//multibody/plant",
135+
"@msgpack_internal//:msgpack",
136+
],
137+
)
138+
114139
drake_cc_library(
115140
name = "point_contact_visualizer",
116141
srcs = ["point_contact_visualizer.cc"],
Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
#include "drake/multibody/meshcat/meshcat_mouse_spring.h"
2+
3+
#include <cmath>
4+
#include <optional>
5+
#include <utility>
6+
#include <vector>
7+
8+
#include "drake/common/drake_throw.h"
9+
#include "drake/multibody/math/spatial_algebra.h"
10+
#include "drake/multibody/plant/externally_applied_spatial_force.h"
11+
12+
namespace drake {
13+
namespace multibody {
14+
namespace meshcat {
15+
16+
using Eigen::Vector3d;
17+
using geometry::Meshcat;
18+
using math::RigidTransform;
19+
using systems::Context;
20+
using systems::DiagramBuilder;
21+
22+
namespace {
23+
24+
// Replicates MultibodyPlant's body-frame naming (see GetScopedName in
25+
// multibody_plant.cc) and converts the "::" model-instance separators into "/"
26+
// the same way MeshcatVisualizer does, so the result matches the body's path in
27+
// the Meshcat scene tree.
28+
std::string BodyFramePathSegment(const MultibodyPlant<double>& plant,
29+
const RigidBody<double>& body) {
30+
std::string name;
31+
const ModelInstanceIndex model_instance = body.model_instance();
32+
if (model_instance != world_model_instance() &&
33+
model_instance != default_model_instance()) {
34+
name = plant.GetModelInstanceName(model_instance) + "::" + body.name();
35+
} else {
36+
name = body.name();
37+
}
38+
// MultibodyPlant declares frames with SceneGraph using "::";
39+
// MeshcatVisualizer replaces those with "/" to expose the full tree.
40+
for (size_t pos = 0; (pos = name.find("::", pos)) != std::string::npos;) {
41+
name.replace(pos, 2, "/");
42+
pos += 1;
43+
}
44+
return name;
45+
}
46+
47+
} // namespace
48+
49+
MeshcatMouseSpring::MeshcatMouseSpring(std::shared_ptr<Meshcat> meshcat,
50+
const MultibodyPlant<double>* plant,
51+
double stiffness)
52+
: systems::LeafSystem<double>(),
53+
meshcat_(std::move(meshcat)),
54+
plant_(plant),
55+
stiffness_(stiffness) {
56+
DRAKE_THROW_UNLESS(meshcat_ != nullptr);
57+
DRAKE_THROW_UNLESS(plant_ != nullptr);
58+
DRAKE_THROW_UNLESS(plant_->is_finalized());
59+
DRAKE_THROW_UNLESS(stiffness_ >= 0.0);
60+
61+
BuildPathToBodyMap(*plant_);
62+
63+
body_poses_input_port_ =
64+
this->DeclareAbstractInputPort(
65+
"body_poses", Value<std::vector<RigidTransform<double>>>())
66+
.get_index();
67+
body_spatial_velocities_input_port_ =
68+
this->DeclareAbstractInputPort(
69+
"body_spatial_velocities",
70+
Value<std::vector<SpatialVelocity<double>>>())
71+
.get_index();
72+
spatial_forces_output_port_ =
73+
this->DeclareAbstractOutputPort(
74+
"spatial_forces",
75+
std::vector<ExternallyAppliedSpatialForce<double>>{},
76+
&MeshcatMouseSpring::CalcSpatialForces)
77+
.get_index();
78+
}
79+
80+
MeshcatMouseSpring::~MeshcatMouseSpring() = default;
81+
82+
void MeshcatMouseSpring::BuildPathToBodyMap(
83+
const MultibodyPlant<double>& plant) {
84+
for (BodyIndex index(0); index < plant.num_bodies(); ++index) {
85+
if (index == plant.world_body().index()) continue;
86+
const RigidBody<double>& body = plant.get_body(index);
87+
// MeshcatVisualizer publishes each body's geometry under a node named by
88+
// the body's scoped frame name (with "::" replaced by "/"), e.g.
89+
// "/drake/<prefix>/my_model/my_body/<geometry>". We key on just the scoped
90+
// name ("my_model/my_body") and match it within the dragged path below.
91+
path_to_body_[BodyFramePathSegment(plant, body)] = index;
92+
}
93+
}
94+
95+
void MeshcatMouseSpring::CalcSpatialForces(
96+
const Context<double>& context,
97+
std::vector<ExternallyAppliedSpatialForce<double>>* forces) const {
98+
forces->clear();
99+
const std::optional<Meshcat::ObjectDrag> drag = meshcat_->GetObjectDrag();
100+
if (!drag.has_value()) {
101+
return;
102+
}
103+
104+
// Find the body whose scoped frame name appears as a run of path segments
105+
// in the dragged object's Meshcat path. The path looks like
106+
// "/drake/<vis_prefix>/.../<model>/<body>/<geometry>...", so we look for
107+
// the scoped name ("<model>/<body>") bounded by '/' on both sides. Matching
108+
// this way is independent of which visualization layer (illustration,
109+
// proximity, inertia, ...) was clicked. Among matches we keep the longest
110+
// (most specific) scoped name.
111+
BodyIndex body_index;
112+
size_t best_len = 0;
113+
const std::string& drag_path = drag->path;
114+
for (const auto& [segment, index] : path_to_body_) {
115+
if (segment.size() <= best_len) continue;
116+
const std::string needle = "/" + segment;
117+
for (size_t pos = drag_path.find(needle); pos != std::string::npos;
118+
pos = drag_path.find(needle, pos + 1)) {
119+
const size_t after = pos + needle.size();
120+
if (after == drag_path.size() || drag_path[after] == '/') {
121+
best_len = segment.size();
122+
body_index = index;
123+
break;
124+
}
125+
}
126+
}
127+
if (!body_index.is_valid()) {
128+
// The dragged object doesn't belong to a movable body of this plant.
129+
return;
130+
}
131+
132+
const auto& X_WB_all =
133+
get_body_poses_input_port().Eval<std::vector<RigidTransform<double>>>(
134+
context);
135+
const auto& V_WB_all =
136+
get_body_spatial_velocities_input_port()
137+
.Eval<std::vector<SpatialVelocity<double>>>(context);
138+
const RigidTransform<double>& X_WB = X_WB_all[body_index];
139+
const SpatialVelocity<double>& V_WB = V_WB_all[body_index];
140+
141+
// The attachment point A (anchor) and the cursor target T, in world.
142+
const Vector3d& p_WA = drag->anchor_in_world;
143+
const Vector3d& p_WT = drag->target_in_world;
144+
145+
// The anchor expressed in the body frame, where the force is applied.
146+
const Vector3d p_BoBq_B = X_WB.inverse() * p_WA;
147+
148+
// The world velocity of the attachment point, for damping.
149+
const Vector3d p_BoA_W = p_WA - X_WB.translation();
150+
const Vector3d v_WA = V_WB.Shift(p_BoA_W).translational();
151+
152+
// Mass-scaled spring + damper force: scaling by the body's mass makes the
153+
// translational response frequency (sqrt(stiffness)) and damping ratio
154+
// independent of mass.
155+
// TODO(vincekurtz): consider using composite mass instead of body mass.
156+
const double mass = plant_->get_body(body_index).default_mass();
157+
const Vector3d f_W =
158+
mass * stiffness_ * (p_WT - p_WA) - mass * std::sqrt(stiffness_) * v_WA;
159+
160+
ExternallyAppliedSpatialForce<double> force;
161+
force.body_index = body_index;
162+
force.p_BoBq_B = p_BoBq_B;
163+
force.F_Bq_W = SpatialForce<double>(Vector3d::Zero(), f_W);
164+
forces->push_back(force);
165+
}
166+
167+
MeshcatMouseSpring& MeshcatMouseSpring::AddToBuilder(
168+
DiagramBuilder<double>* builder, const MultibodyPlant<double>* plant,
169+
std::shared_ptr<Meshcat> meshcat, double stiffness) {
170+
DRAKE_THROW_UNLESS(builder != nullptr);
171+
DRAKE_THROW_UNLESS(plant != nullptr);
172+
auto& spring = *builder->AddSystem<MeshcatMouseSpring>(std::move(meshcat),
173+
plant, stiffness);
174+
spring.set_name("meshcat_mouse_spring");
175+
builder->Connect(plant->get_body_poses_output_port(),
176+
spring.get_body_poses_input_port());
177+
builder->Connect(plant->get_body_spatial_velocities_output_port(),
178+
spring.get_body_spatial_velocities_input_port());
179+
builder->Connect(spring.get_spatial_forces_output_port(),
180+
plant->get_applied_spatial_force_input_port());
181+
return spring;
182+
}
183+
184+
} // namespace meshcat
185+
} // namespace multibody
186+
} // namespace drake

0 commit comments

Comments
 (0)