diff --git a/pearl.c b/pearl.c
index e53b4da..27b4860 100644
--- a/pearl.c
+++ b/pearl.c
@@ -2135,7 +2135,7 @@ static char *interpret_move(const game_state *state, game_ui *ui,
 	}
     }
 
-    if (button == 27 || button == '\b') {
+    if ((button == 27 || button == '\b') && ui->ndragcoords >= 0) {
         ui->ndragcoords = -1;
         return UI_UPDATE;
     }