A first attempt at level checking for classes

Check that instantiated root variable of a method inside a class
is nested in the instance variable of the class.

For the moment I was not able to construct a counter example where this
makes a difference.

compiler/src/dotty/tools/dotc/cc/CheckCaptures.scala

@@ -319,7 +319,7 @@ class CheckCaptures extends Recheck, SymTransformer:
           includeCallCaptures(tree.symbol, tree.srcPos)
       else
         markFree(tree.symbol, tree.srcPos)
-      instantiateLocalRoots(tree.symbol, NoPrefix, pt):
+      instantiateLocalRoots(tree.symbol, NoPrefix, pt, tree.srcPos):
         super.recheckIdent(tree, pt)
 
     /** A specialized implementation of the selection rule.
@@ -349,7 +349,7 @@ class CheckCaptures extends Recheck, SymTransformer:
 
       val selType = recheckSelection(tree, qualType, name, disambiguate)
       val selCs = selType.widen.captureSet
-      instantiateLocalRoots(tree.symbol, qualType, pt):
+      instantiateLocalRoots(tree.symbol, qualType, pt, tree.srcPos):
         if selCs.isAlwaysEmpty || selType.widen.isBoxedCapturing || qualType.isBoxedCapturing then
           selType
         else
@@ -370,7 +370,7 @@ class CheckCaptures extends Recheck, SymTransformer:
      *   - `tp` is the type of a function that gets applied, either as a method
      *     or as a function value that gets applied.
      */
-    def instantiateLocalRoots(sym: Symbol, pre: Type, pt: Type)(tp: Type)(using Context): Type =
+    def instantiateLocalRoots(sym: Symbol, pre: Type, pt: Type, pos: SrcPos)(tp: Type)(using Context): Type =
       def canInstantiate =
         sym.is(Method, butNot = Accessor)
         || sym.isTerm && defn.isFunctionType(sym.info) && pt == AnySelectionProto
@@ -379,6 +379,16 @@ class CheckCaptures extends Recheck, SymTransformer:
         var tp1 = tpw
         val rootVar = CaptureRoot.Var(ctx.owner, sym)
         if sym.isLevelOwner then
+          val outerOwner = sym.skipConstructor.owner.levelOwner
+          if outerOwner.isClass then
+            val outerRoot = outerOwner.localRoot.termRef
+            outerRoot.asSeenFrom(pre, sym.owner) match
+              case outerLimit: CaptureRoot if outerLimit ne outerRoot =>
+                capt.println(i"constraining $rootVar of $sym by $outerLimit")
+                if !outerLimit.encloses(rootVar) then
+                // Should this be an assertion failure instead?
+                  report.error(em"outer instance $outerLimit does not enclose local root $rootVar", pos)
+              case _ =>
           tp1 = mapRoots(sym.localRoot.termRef, rootVar)(tp1)
           if tp1 ne tpw then
             ccSetup.println(i"INST local $sym: $tp, ${sym.localRoot} = $tp1")

tests/neg-custom-args/captures/nesting-inversion.scala

@@ -0,0 +1,43 @@
+def f(x: (() => Unit)): (() => Unit) => (() => Unit) =
+  def g(y: (() => Unit)): (() => Unit) = x
+  g
+
+def test1(x: (() => Unit)): Unit =
+  def test2(y: (() => Unit)) =
+    val a: (() => Unit) => (() => Unit) = f(y)
+    a(x)            // OK, but should be error
+  test2(() => ())
+
+def test2(x1: (() => Unit), x2: (() => Unit) => Unit) =
+  class C1(x1: (() => Unit), xx2: (() => Unit) => Unit):
+    def c2(y1: (() => Unit), y2: (() => Unit) => Unit): C2^{cap} = C2(y1, y2)
+    class C2(y1: (() => Unit), y2: (() => Unit) => Unit):
+      val a: (() => Unit) => (() => Unit) = f(y1)
+      a(x1)            //OK, but should be error
+    C2(() => (), x => ())
+
+  def test3(y1: (() => Unit), y2: (() => Unit) => Unit) =
+    val cc1 = C1(y1, y2)
+    val cc2 = cc1.c2(x1, x2)
+    val cc3: cc1.C2^{cap[test2]} = cc2 // error
+
+def test4(x1: () => Unit) =
+  class C1:
+    this: C1^ =>
+    class C2(z: () => Unit):
+      this: C2^ =>
+      val foo: () => Unit = ???
+
+  def test5(x2: () => Unit) =
+    val xx1: C1^{cap[test5]} = C1()
+    val y1 =
+      val xx2 = xx1.C2(x1)
+      val xx3: xx1.C2^{cap[test4]} = xx2 // ok, but dubious
+                                       // actual capture set is in test4
+                                       // but level constraints would determine that the root should be in test5
+                                       // only, there is no root in the set to be mapped
+      xx2
+    val f1 = y1.foo
+    val xx4 = xx1.C2(x2)
+    val xx5: xx1.C2^{cap[test4]} = xx4  // error
+

tests/pos-custom-args/captures/nested-classes-2.scala

@@ -20,6 +20,5 @@ def test2(x1: (() => Unit), x2: (() => Unit) => Unit) =
   def test3(y1: (() => Unit), y2: (() => Unit) => Unit) =
     val cc1/*: C1^{cap[test3]}*/ = C1(y1, y2)  // error (but should be OK)
     val cc2 = cc1.c2(x1, x2)         // error (but should be OK)
-    ()
     //val cc3: cc1.C2^{cap[test2]} = cc2