Sorted it out. The problem is the onmouseup event when you decide to stop dragging out the shape can fire on the underlying/attached shape, or on the shape you're dragging. It's random, depending on the cursor position, but favors the drawn shape if you don't have an offset or delay in your drag. (Moveable.js and Mover.js in dojox/gfx pointed me in the right direction.)
I changed my box to a path in the course of trying to make it work, and this seems to perform better, but isn't necessary.
The key was to make a general 'onMouseUp' function, then call that from both the originator-shape's onmouseup as well as the dragged shape's onmouseup. My example is sloppy, but I hope it gets the point across.
jsFiddle: http://jsfiddle.net/n3KGY/1/
Key code:
// General method to clear out a selector if
// one was being drawn.
var selectorMouseUp = function(e) {
reporter.innerHTML = "onmouseup";
isMouseDown = false;
whiteRect.groupSelector_ = null;
if(groupSelector) {
if(selectorUp) {
groupSelector.disconnect(selectorUp);
}
surface.remove(groupSelector);
groupSelector = null;
}
e.stopImmediatePropagation();
e.stopPropagation();
e.preventDefault();
};
// Mouseup event for the background/workspace
whiteRect.connect("onmouseup",function(e){
selectorMouseUp(e);
});
// Make a selector as a path on the surface
// and attach a mouseup to it
var makeSelector = function(x,y,w,h) {
groupSelector = surface.createPath()
.moveTo(x,y)
.hLineTo(x+w).vLineTo(y+h).hLineTo(x).vLineTo(y)
.setStroke({color: "blue", width: 3})
.closePath();
// Attach the same mouseup method as the workspace/background
selectorUp = groupSelector.connect("onmouseup",function(e){
reporter.innerHTML = "onmouseup (selector)";
selectorMouseUp(e);
});
};
bigRect.connect("onmousemove",function(e){
if(isMouseDown) {
if(bigRect.groupSelector_) {
var ex = e.clientX;
var ey = e.clientY;
reporter.innerHTML = "dragging at " + ex+","+ey;
var downX = bigRect.groupSelector_.x;
var downY = bigRect.groupSelector_.y;
var leadingX = (ex - grn.offsetLeft < downX ? ex - grn.offsetLeft : downX);
var leadingY = (ey - grn.offsetTop < downY ? ey - grn.offsetTop : downY);
var selWidth = Math.abs(ex - grn.offsetLeft - downX);
var selHeight = Math.abs(ey - grn.offsetTop - downY);
if(groupSelector) {
// If there's already a selector being drawn, get rid of it.
groupSelector.disconnect(selectorUp);
surface.remove(groupSelector);
}
// Draw the current selector
makeSelector(leadingX,leadingY,selWidth,selHeight);
e.stopImmediatePropagation();
e.stopPropagation();
e.preventDefault();
}
}
});