diff --git a/readme.md b/readme.md
index b765c50..12f45b5 100644
--- a/readme.md
+++ b/readme.md
@@ -77,7 +77,7 @@ You can override the default options by passing an object in to the `.panZoom({o
 | wheelZoom    | true             | Enable mouse wheel zoom                                                                                        |
 | panButton    | 0                | Which mouse button to use for pan ([info](https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent/button)) |
 | oneFingerPan | false            | Enables the ability to pan with only one finger instead of two for touchdevices                                |
-| margins      | false            | An object {top, left, right, bottom} to restrict the pan area so that at least x px are still visible          |
+| margins      | false            | An object {top, left, right, bottom} to restrict the pan area towards this side so that at least x user units of the opposite side are still visible |
 | zoomFactor   | 2                | How quickly to zoom when using `wheelZoom`                                                                   |
 | zoomMin      | Number.MIN_VALUE | The minimum zoom level                                                                                         |
 | zoomMax      | Number.MAX_VALUE | The maximum zoom level                                                                                         |
diff --git a/src/svg.panzoom.js b/src/svg.panzoom.js
index 97f03a7..cb4d5dc 100644
--- a/src/svg.panzoom.js
+++ b/src/svg.panzoom.js
@@ -27,20 +27,103 @@ extend(Svg, {
     let lastTouches
     let zoomInProgress = false
 
+    const viewbox = this.viewbox()
+
     const restrictToMargins = box => {
       if (!margins) return box
       const { top, left, bottom, right } = margins
-      const zoom = this.width() / box.width
 
       const { width, height } = this.attr(['width', 'height'])
+      const preserveAspectRatio = this.node.preserveAspectRatio.baseVal
+
+      // The current viewport (exactly what is shown on the screen, what we ultimately want to restrict)
+      // is not always exactly the same as current viewbox. They are different when the viewbox aspectRatio and the svg aspectRatio
+      // are different and preserveAspectRatio is not "none". These offsets represent the difference in user coordinates
+      // between the side of the viewbox and the side of the viewport.
+      let viewportLeftOffset = 0
+      let viewportRightOffset = 0
+      let viewportTopOffset = 0
+      let viewportBottomOffset = 0
+
+      // preserveAspectRatio none has no offsets
+      if (preserveAspectRatio.align !== preserveAspectRatio.SVG_PRESERVEASPECTRATIO_NONE) {
+        const svgAspectRatio = width / height
+        const viewboxAspectRatio = viewbox.width / viewbox.height
+        // when aspectRatios are the same, there are no offsets
+        if (viewboxAspectRatio !== svgAspectRatio) {
+          // aspectRatio unknown is like meet because that's the default
+          const isMeet = preserveAspectRatio.meetOrSlice !== preserveAspectRatio.SVG_MEETORSLICE_SLICE
+          const changedAxis = svgAspectRatio > viewboxAspectRatio ? 'width' : 'height'
+          const isWidth = changedAxis === 'width'
+          const changeHorizontal = (isMeet && isWidth) || (!isMeet && !isWidth)
+          const ratio = changeHorizontal
+            ? svgAspectRatio / viewboxAspectRatio
+            : viewboxAspectRatio / svgAspectRatio
+
+          const offset = box[changedAxis] - box[changedAxis] * ratio
+          if (changeHorizontal) {
+            if (
+              preserveAspectRatio.align === preserveAspectRatio.SVG_PRESERVEASPECTRATIO_XMIDYMIN ||
+              preserveAspectRatio.align === preserveAspectRatio.SVG_PRESERVEASPECTRATIO_XMIDYMID ||
+              preserveAspectRatio.align === preserveAspectRatio.SVG_PRESERVEASPECTRATIO_XMIDYMAX) {
+              viewportLeftOffset = offset / 2
+              viewportRightOffset = -offset / 2
+            } else if (
+              preserveAspectRatio.align === preserveAspectRatio.SVG_PRESERVEASPECTRATIO_XMINYMIN ||
+              preserveAspectRatio.align === preserveAspectRatio.SVG_PRESERVEASPECTRATIO_XMINYMID ||
+              preserveAspectRatio.align === preserveAspectRatio.SVG_PRESERVEASPECTRATIO_XMINYMAX) {
+              viewportRightOffset = -offset
+            } else if (
+              preserveAspectRatio.align === preserveAspectRatio.SVG_PRESERVEASPECTRATIO_XMAXYMIN ||
+              preserveAspectRatio.align === preserveAspectRatio.SVG_PRESERVEASPECTRATIO_XMAXYMID ||
+              preserveAspectRatio.align === preserveAspectRatio.SVG_PRESERVEASPECTRATIO_XMAXYMAX) {
+              viewportLeftOffset = offset
+            }
+          } else {
+            if (
+              preserveAspectRatio.align === preserveAspectRatio.SVG_PRESERVEASPECTRATIO_XMINYMID ||
+              preserveAspectRatio.align === preserveAspectRatio.SVG_PRESERVEASPECTRATIO_XMIDYMID ||
+              preserveAspectRatio.align === preserveAspectRatio.SVG_PRESERVEASPECTRATIO_XMAXYMID) {
+              viewportTopOffset = offset / 2
+              viewportBottomOffset = -offset / 2
+            } else if (
+              preserveAspectRatio.align === preserveAspectRatio.SVG_PRESERVEASPECTRATIO_XMINYMIN ||
+              preserveAspectRatio.align === preserveAspectRatio.SVG_PRESERVEASPECTRATIO_XMIDYMIN ||
+              preserveAspectRatio.align === preserveAspectRatio.SVG_PRESERVEASPECTRATIO_XMAXYMIN) {
+              viewportBottomOffset = -offset
+            } else if (
+              preserveAspectRatio.align === preserveAspectRatio.SVG_PRESERVEASPECTRATIO_XMINYMAX ||
+              preserveAspectRatio.align === preserveAspectRatio.SVG_PRESERVEASPECTRATIO_XMIDYMAX ||
+              preserveAspectRatio.align === preserveAspectRatio.SVG_PRESERVEASPECTRATIO_XMAXYMAX) {
+              viewportTopOffset = offset
+            }
+          }
 
-      const leftLimit = width - left / zoom
-      const rightLimit = (right - width) / zoom
-      const topLimit = height - top / zoom
-      const bottomLimit = (bottom - height) / zoom
+        }
+      }
 
-      box.x = Math.min(leftLimit, Math.max(rightLimit, box.x))
-      box.y = Math.min(topLimit, Math.max(bottomLimit, box.y))
+      // when box.x == leftLimit, the image is panned to the left,
+      // i.e the current box is to the right of the initial viewbox,
+      // and only the right part of the initial image is visible, i.e.
+      // the right side of the initial viewbox minus left margin (viewbox.x+viewbox.width-left)
+      // is aligned with the left side of the viewport (box.x + viewportLeftOffset):
+      // viewbox.width + viewbox.x - left = box.x + viewportLeftOffset
+      // viewbox.width + viewbox.x - left - viewportLeftOffset = box.x (= leftLimit)
+      const leftLimit = viewbox.width + viewbox.x - left - viewportLeftOffset
+      // when box.x == rightLimit, the image is panned to the right,
+      // i.e the current box is to the left of the initial viewbox
+      // and only the left part of the initial image is visible, i.e
+      // the left side of the initial viewbox plus right margin (viewbox.x + right)
+      // is aligned with the right side of the viewport (box.x + box.width + viewportRightOffset)
+      // viewbox.x + right = box.x + box.width + viewportRightOffset
+      // viewbox.x + right - box.width - viewportRightOffset = box.x (= rightLimit)
+      const rightLimit = viewbox.x + right - box.width - viewportRightOffset
+      // same with top and bottom
+      const topLimit = viewbox.height + viewbox.y - top - viewportTopOffset
+      const bottomLimit = viewbox.y + bottom - box.height - viewportBottomOffset
+
+      box.x = Math.min(leftLimit, Math.max(rightLimit, box.x)) // enforce rightLimit <= box.x <= leftLimit
+      box.y = Math.min(topLimit, Math.max(bottomLimit, box.y)) // enforce bottomLimit <= box.y <= topLimit
       return box
     }