diff --git a/src/ui/GameWorldView.cpp b/src/ui/GameWorldView.cpp
index d32d282..2359b40 100644
--- a/src/ui/GameWorldView.cpp
+++ b/src/ui/GameWorldView.cpp
@@ -1074,17 +1074,21 @@ void GameWorldView::mousePressEvent(QMouseEvent* event)
 
         const bool ctrl = (event->modifiers() & Qt::ControlModifier) != 0;
 
-        selectAtPoint(tile, coordinates.widgetToWorld(event->pos()), ctrl);
-
-        // A click on empty space also starts a box drag. selectAtPoint has already
-        // cleared the selection unless Ctrl is preserving it for an additive drag.
-        m_boxSelecting    = true;
-        m_boxStartTile    = tile;
-        m_boxCurrentTile  = tile;
+        // Only a click that hit nothing starts a box drag. Starting one on a hit
+        // would re-resolve the same object as a 1x1 box on release and undo the
+        // click: a Ctrl+click would toggle the building off, then straight back on.
+        if (!selectAtPoint(tile, coordinates.widgetToWorld(event->pos()), ctrl))
+        {
+            // selectAtPoint has already cleared the selection unless Ctrl is
+            // preserving it for an additive drag.
+            m_boxSelecting    = true;
+            m_boxStartTile    = tile;
+            m_boxCurrentTile  = tile;
+        }
     }
 }
 
-void GameWorldView::selectAtPoint(QPoint tile, QVector2D worldPos, bool additive)
+bool GameWorldView::selectAtPoint(QPoint tile, QVector2D worldPos, bool additive)
 {
     // Point hit-test precedence: buildings win over actors, which win over debris
     // (REQ-UI-SELECTION-CATEGORIES). What each hit then does to the existing
@@ -1096,25 +1100,26 @@ void GameWorldView::selectAtPoint(QPoint tile, QVector2D worldPos, bool additive
     if (buildingHit.has_value())
     {
         m_selection.selectBuildings({*buildingHit}, mode);
-        return;
+        return true;
     }
 
     const entt::entity actorHit = entityAtWorldPos(m_sim->getAdmin(), worldPos);
     if (actorHit != entt::null)
     {
         m_selection.selectFieldObjects({actorHit}, {}, mode);
-        return;
+        return true;
     }
 
     const entt::entity debrisHit = debrisAtWorldPos(m_sim->getAdmin(), worldPos);
     if (debrisHit != entt::null)
     {
         m_selection.selectFieldObjects({}, {debrisHit}, mode);
-        return;
+        return true;
     }
 
     // Empty space: a plain click clears the whole selection, Ctrl preserves it.
     if (!additive) { m_selection.clearAll(); }
+    return false;
 }
 
 void GameWorldView::selectInBox(bool additive)
diff --git a/src/ui/GameWorldView.h b/src/ui/GameWorldView.h
index 1df2380..9f0ec35 100644
--- a/src/ui/GameWorldView.h
+++ b/src/ui/GameWorldView.h
@@ -193,8 +193,9 @@ private:
     void                     pruneDespawnedActors();
     // Resolves a click into the selection it should produce, applying the category
     // precedence of REQ-UI-SELECTION-CATEGORIES; the controller owns what that then
-    // does to the existing selection.
-    void                     selectAtPoint(QPoint tile, QVector2D worldPos, bool additive);
+    // does to the existing selection. Returns false when the click hit nothing,
+    // which is the only case that goes on to start a box drag.
+    bool                     selectAtPoint(QPoint tile, QVector2D worldPos, bool additive);
     void                     selectInBox(bool additive);
     void                     stepSpeed(int delta);
     void                     placeAtTile(QPoint tile);