diff --git a/packages/editor/src/lib/editor/Editor.ts b/packages/editor/src/lib/editor/Editor.ts
index 435692885..8a0ae7235 100644
--- a/packages/editor/src/lib/editor/Editor.ts
+++ b/packages/editor/src/lib/editor/Editor.ts
@@ -4269,6 +4269,8 @@ export class Editor extends EventEmitter<TLEventMap> {
 			renderingOnly?: boolean
 			margin?: number
 			hitInside?: boolean
+			// TODO: we probably need to rename this, we don't quite _always_
+			// respect this esp. in the part below that does "Check labels first"
 			hitLabels?: boolean
 			hitFrameInside?: boolean
 			filter?: (shape: TLShape) => boolean
diff --git a/packages/tldraw/src/lib/tools/SelectTool/childStates/EditingShape.ts b/packages/tldraw/src/lib/tools/SelectTool/childStates/EditingShape.ts
index 759cd480a..a1d799c6a 100644
--- a/packages/tldraw/src/lib/tools/SelectTool/childStates/EditingShape.ts
+++ b/packages/tldraw/src/lib/tools/SelectTool/childStates/EditingShape.ts
@@ -55,7 +55,7 @@ export class EditingShape extends StateNode {
 
 		switch (info.target) {
 			case 'canvas': {
-				const hitShape = getHitShapeOnCanvasPointerDown(this.editor)
+				const hitShape = getHitShapeOnCanvasPointerDown(this.editor, true /* hitLabels */)
 				if (hitShape) {
 					this.onPointerDown({
 						...info,
@@ -144,13 +144,8 @@ export class EditingShape extends StateNode {
 
 		this.editor.select(hitShape.id)
 
-		if (this.editor.getInstanceState().isCoarsePointer) {
-			this.editor.setEditingShape(null)
-			this.editor.setCurrentTool('select.idle')
-		} else {
-			this.editor.setEditingShape(hitShape.id)
-			updateHoveredId(this.editor)
-		}
+		this.editor.setEditingShape(hitShape.id)
+		updateHoveredId(this.editor)
 	}
 
 	override onComplete: TLEventHandlers['onComplete'] = (info) => {
diff --git a/packages/tldraw/src/lib/tools/selection-logic/getHitShapeOnCanvasPointerDown.ts b/packages/tldraw/src/lib/tools/selection-logic/getHitShapeOnCanvasPointerDown.ts
index 7d1c8344e..6e47d8322 100644
--- a/packages/tldraw/src/lib/tools/selection-logic/getHitShapeOnCanvasPointerDown.ts
+++ b/packages/tldraw/src/lib/tools/selection-logic/getHitShapeOnCanvasPointerDown.ts
@@ -1,6 +1,9 @@
 import { Editor, HIT_TEST_MARGIN, TLShape } from '@tldraw/editor'
 
-export function getHitShapeOnCanvasPointerDown(editor: Editor): TLShape | undefined {
+export function getHitShapeOnCanvasPointerDown(
+	editor: Editor,
+	hitLabels = false
+): TLShape | undefined {
 	const zoomLevel = editor.getZoomLevel()
 	const {
 		inputs: { currentPagePoint },
@@ -10,7 +13,7 @@ export function getHitShapeOnCanvasPointerDown(editor: Editor): TLShape | undefi
 		// hovered shape at point
 		editor.getShapeAtPoint(currentPagePoint, {
 			hitInside: false,
-			hitLabels: false,
+			hitLabels,
 			margin: HIT_TEST_MARGIN / zoomLevel,
 			renderingOnly: true,
 		}) ??