diff --git a/CHANGELOG.md b/CHANGELOG.md
index 81409c0..5792098 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,9 @@
 ### Bugfixes
 - Fixed a bug where tokens wouldn't be moved to the corect end position on gridless maps
 - Ruler now appears immediately when the token is being dragged
+- On gridless maps the ruler will always start measuring at the center of the token
+  - This change has no impact on the distance that is being measured
+  - In addition to the cosmetical aspect this also fixes a bug that allowed players to glitch through walls
 
 ## v1.1.0
 ### New features
diff --git a/src/foundry_imports.js b/src/foundry_imports.js
index 4914389..fc0c6f3 100644
--- a/src/foundry_imports.js
+++ b/src/foundry_imports.js
@@ -81,7 +81,8 @@ export function onMouseMove(event) {
 
 	// Extract event data
 	const mt = event._measureTime || 0;
-	const { destination, originalEvent } = event.data;
+	const originalEvent = event.data.originalEvent;
+	const destination = {x: event.data.destination.x + this.rulerOffset.x, y: event.data.destination.y + this.rulerOffset.y}
 
 	// Hide any existing Token HUD
 	canvas.hud.token.clear();
diff --git a/src/main.js b/src/main.js
index 625c49b..e42c2f1 100644
--- a/src/main.js
+++ b/src/main.js
@@ -92,8 +92,12 @@ function hookRulerFunctions() {
 }
 
 function onTokenLeftDragStart(event) {
-	canvas.controls.ruler._onDragStart(event)
 	canvas.controls.ruler.draggedToken = this
+	const tokenCenter = {x: this.x + canvas.grid.size / 2, y: this.y + canvas.grid.size / 2}
+	canvas.controls.ruler.clear();
+	canvas.controls.ruler._state = Ruler.STATES.STARTING;
+	canvas.controls.ruler.rulerOffset = {x: tokenCenter.x - event.data.origin.x, y: tokenCenter.y - event.data.origin.y}
+	canvas.controls.ruler._addWaypoint(tokenCenter);
 }
 
 function onTokenLeftDragMove(event) {