diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2820ea0..912178a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,6 @@
 ## In development
 ### New features
-- The drag ruler will now be colored for other players than the dragging player as well
+- The drag ruler will now be colored for other players than the dragging player as well (only if they have at least observer permissions for that token)
 - The drag ruler won't be shown to other players if they cannot see the dragged token
 
 ## v1.0.1
diff --git a/src/main.js b/src/main.js
index e35ea59..f5775f0 100644
--- a/src/main.js
+++ b/src/main.js
@@ -172,6 +172,10 @@ function nativeSpeedProvider(token, playercolor) {
 function getColorForDistance(startDistance, subDistance) {
 	if (!this.isDragRuler)
 		return this.color
+	// Don't apply colors if the current user doesn't have at least observer permissions
+	if (this.draggedToken.actor.permission < 2) {
+		return this.color
+	}
 	const distance = startDistance + subDistance
 	const ranges = currentSpeedProvider(this.draggedToken, this.color)
 	if (ranges.length === 0)