diff --git a/static/canvas-renderer.js b/static/canvas-renderer.js
index 2523e78..e28d269 100644
--- a/static/canvas-renderer.js
+++ b/static/canvas-renderer.js
@@ -394,12 +394,19 @@ class CanvasRenderer extends HTMLElement {
     async #panningBehaviour() {
         while (true) {
             let mouseDown = await listen([this, "mousedown"]);
+
+            let startingPanX = this.viewport.panX;
+            let startingPanY = this.viewport.panY;
+
             if (mouseDown.button == 1 || mouseDown.button == 2) {
                 mouseDown.preventDefault();
                 while (true) {
                     let event = await listen([window, "mousemove"], [window, "mouseup"]);
                     if (event.type == "mousemove") {
-                        this.viewport.panAround(event.movementX, event.movementY);
+                        let deltaX = mouseDown.clientX - event.clientX;
+                        let deltaY = mouseDown.clientY - event.clientY;
+                        this.viewport.panX = startingPanX + deltaX / this.viewport.zoom;
+                        this.viewport.panY = startingPanY + deltaY / this.viewport.zoom;
                         this.sendViewportUpdate();
                     } else if (event.type == "mouseup" && event.button == mouseDown.button) {
                         this.dispatchEvent(new Event(".viewportUpdateEnd"));
diff --git a/static/viewport.js b/static/viewport.js
index 253b7c9..e9916ae 100644
--- a/static/viewport.js
+++ b/static/viewport.js
@@ -9,11 +9,6 @@ export class Viewport {
         return Math.pow(2, this.zoomLevel * 0.25);
     }
 
-    panAround(x, y) {
-        this.panX -= x / this.zoom;
-        this.panY -= y / this.zoom;
-    }
-
     zoomIn(delta) {
         this.zoomLevel += delta;
         this.zoomLevel = Math.max(-16, Math.min(20, this.zoomLevel));