add flexible types to deal with Java-defined signatures under -Yexpli…

…cit-nulls (#18112)

This is a continuation of #17369.

When dealing with reference types from Java, it's essential to address
the implicit nullability of these types. The most accurate way to
represent them in Scala is to use nullable types, though working with
lots of nullable types
directly can be annoying. To streamline interactions with Java
libraries, we introduce the concept of flexible types.

The flexible type, denoted by `T?`, functions as an abstract type with
unique bounds: `T | Null ... T`, ensuring that `T | Null <: T? <: T`.
The subtyping rule treats a reference type coming from Java as either
nullable or non-nullable depending on the context. This concept draws
inspiration from Kotlin's [platform
types](https://kotlinlang.org/docs/java-interop.html#null-safety-and-platform-types).
By relaxing null checks for such types, Scala aligns its safety
guarantees with those of Java. Notably, flexible types are
non-denotable, meaning users cannot explicitly write them in the code;
only the compiler can construct or infer these types.

Consequently, a value with a flexible type can serve as both a nullable
and non-nullable value. Additionally, both nullable and non-nullable
values can be passed as parameters with flexible types during function
calls. Invoking the member functions of a flexible type is allowed, but
it can trigger a `NullPointerException` if the value is indeed `null`
during runtime.

```scala
// Considering class J is from Java
class J {
  // Translates to def f(s: String?): Unit
  public void f(String s) {
  }

  // Translates to def g(): String?
  public String g() {
    return "";
  }
}

// Use J in Scala
def useJ(j: J) =
  val x1: String = ""
  val x2: String | Null = null
  j.f(x1) // Passing String to String?
  j.f(x2) // Passing String | Null to String?
  j.f(null) // Passing Null to String?

  // Assign String? to String
  val y1: String = j.g()
  // Assign String? to String | Null
  val y2: String | Null = j.g()

  // Calling member functions on flexible types
  j.g().trim().length()
```

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -417,6 +417,7 @@ private sealed trait YSettings:
   // Experimental language features
   val YnoKindPolymorphism: Setting[Boolean] = BooleanSetting(ForkSetting, "Yno-kind-polymorphism", "Disable kind polymorphism.")
   val YexplicitNulls: Setting[Boolean] = BooleanSetting(ForkSetting, "Yexplicit-nulls", "Make reference types non-nullable. Nullable types can be expressed with unions: e.g. String|Null.")
+  val YnoFlexibleTypes: Setting[Boolean] = BooleanSetting(ForkSetting, "Yno-flexible-types", "Disable turning nullable Java return types and parameter types into flexible types, which behave like abstract types with a nullable lower bound and non-nullable upper bound.")
   val YcheckInit: Setting[Boolean] = BooleanSetting(ForkSetting, "Ysafe-init", "Ensure safe initialization of objects.")
   val YcheckInitGlobal: Setting[Boolean] = BooleanSetting(ForkSetting, "Ysafe-init-global", "Check safe initialization of global objects.")
   val YrequireTargetName: Setting[Boolean] = BooleanSetting(ForkSetting, "Yrequire-targetName", "Warn if an operator is defined without a @targetName annotation.")

compiler/src/dotty/tools/dotc/core/ConstraintHandling.scala

@@ -696,9 +696,11 @@ trait ConstraintHandling {
       tp.rebind(tp.parent.hardenUnions)
     case tp: HKTypeLambda =>
       tp.derivedLambdaType(resType = tp.resType.hardenUnions)
+    case tp: FlexibleType =>
+      tp.derivedFlexibleType(tp.hi.hardenUnions)
     case tp: OrType =>
-      val tp1 = tp.stripNull
-      if tp1 ne tp then tp.derivedOrType(tp1.hardenUnions, defn.NullType)
+      val tp1 = tp.stripNull(stripFlexibleTypes = false)
+      if tp1 ne tp then tp.derivedOrType(tp1.hardenUnions, defn.NullType, soft = false)
       else tp.derivedOrType(tp.tp1.hardenUnions, tp.tp2.hardenUnions, soft = false)
     case _ =>
       tp

compiler/src/dotty/tools/dotc/core/Contexts.scala

@@ -472,6 +472,9 @@ object Contexts {
     /** Is the explicit nulls option set? */
     def explicitNulls: Boolean = base.settings.YexplicitNulls.value
 
+    /** Is the flexible types option set? */
+    def flexibleTypes: Boolean = base.settings.YexplicitNulls.value && !base.settings.YnoFlexibleTypes.value
+
     /** A fresh clone of this context embedded in this context. */
     def fresh: FreshContext = freshOver(this)
 

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -648,7 +648,7 @@ class Definitions {
   @tu lazy val StringModule: Symbol = StringClass.linkedClass
     @tu lazy val String_+ : TermSymbol = enterMethod(StringClass, nme.raw.PLUS, methOfAny(StringType), Final)
     @tu lazy val String_valueOf_Object: Symbol = StringModule.info.member(nme.valueOf).suchThat(_.info.firstParamTypes match {
-      case List(pt) => pt.isAny || pt.stripNull.isAnyRef
+      case List(pt) => pt.isAny || pt.stripNull().isAnyRef
       case _ => false
     }).symbol
 
@@ -660,13 +660,13 @@ class Definitions {
   @tu lazy val ClassCastExceptionClass: ClassSymbol   = requiredClass("java.lang.ClassCastException")
     @tu lazy val ClassCastExceptionClass_stringConstructor: TermSymbol  = ClassCastExceptionClass.info.member(nme.CONSTRUCTOR).suchThat(_.info.firstParamTypes match {
       case List(pt) =>
-        pt.stripNull.isRef(StringClass)
+        pt.stripNull().isRef(StringClass)
       case _ => false
     }).symbol.asTerm
   @tu lazy val ArithmeticExceptionClass: ClassSymbol  = requiredClass("java.lang.ArithmeticException")
     @tu lazy val ArithmeticExceptionClass_stringConstructor: TermSymbol  = ArithmeticExceptionClass.info.member(nme.CONSTRUCTOR).suchThat(_.info.firstParamTypes match {
       case List(pt) =>
-        pt.stripNull.isRef(StringClass)
+        pt.stripNull().isRef(StringClass)
       case _ => false
     }).symbol.asTerm
 

compiler/src/dotty/tools/dotc/core/JavaNullInterop.scala

@@ -78,11 +78,11 @@ object JavaNullInterop {
    *  but the result type is not nullable.
    */
   private def nullifyExceptReturnType(tp: Type)(using Context): Type =
-    new JavaNullMap(true)(tp)
+    new JavaNullMap(outermostLevelAlreadyNullable = true)(tp)
 
   /** Nullifies a Java type by adding `| Null` in the relevant places. */
   private def nullifyType(tp: Type)(using Context): Type =
-    new JavaNullMap(false)(tp)
+    new JavaNullMap(outermostLevelAlreadyNullable = false)(tp)
 
   /** A type map that implements the nullification function on types. Given a Java-sourced type, this adds `| Null`
    *  in the right places to make the nulls explicit in Scala.
@@ -96,25 +96,29 @@ object JavaNullInterop {
    *                                       to `(A & B) | Null`, instead of `(A | Null & B | Null) | Null`.
    */
   private class JavaNullMap(var outermostLevelAlreadyNullable: Boolean)(using Context) extends TypeMap {
+    def nullify(tp: Type): Type = if ctx.flexibleTypes then FlexibleType(tp) else OrNull(tp)
+
     /** Should we nullify `tp` at the outermost level? */
     def needsNull(tp: Type): Boolean =
-      !outermostLevelAlreadyNullable && (tp match {
-        case tp: TypeRef =>
+      if outermostLevelAlreadyNullable then false
+      else tp match
+        case tp: TypeRef if
           // We don't modify value types because they're non-nullable even in Java.
-          !tp.symbol.isValueClass &&
+          tp.symbol.isValueClass
+          // We don't modify unit types.
+          || tp.isRef(defn.UnitClass)
           // We don't modify `Any` because it's already nullable.
-          !tp.isRef(defn.AnyClass) &&
+          || tp.isRef(defn.AnyClass)
           // We don't nullify Java varargs at the top level.
           // Example: if `setNames` is a Java method with signature `void setNames(String... names)`,
           // then its Scala signature will be `def setNames(names: (String|Null)*): Unit`.
           // This is because `setNames(null)` passes as argument a single-element array containing the value `null`,
           // and not a `null` array.
-          !tp.isRef(defn.RepeatedParamClass)
+          || !ctx.flexibleTypes && tp.isRef(defn.RepeatedParamClass) => false
         case _ => true
-      })
 
     override def apply(tp: Type): Type = tp match {
-      case tp: TypeRef if needsNull(tp) => OrNull(tp)
+      case tp: TypeRef if needsNull(tp) => nullify(tp)
       case appTp @ AppliedType(tycon, targs) =>
         val oldOutermostNullable = outermostLevelAlreadyNullable
         // We don't make the outmost levels of type arguments nullable if tycon is Java-defined.
@@ -124,7 +128,7 @@ object JavaNullInterop {
         val targs2 = targs map this
         outermostLevelAlreadyNullable = oldOutermostNullable
         val appTp2 = derivedAppliedType(appTp, tycon, targs2)
-        if needsNull(tycon) then OrNull(appTp2) else appTp2
+        if needsNull(tycon) then nullify(appTp2) else appTp2
       case ptp: PolyType =>
         derivedLambdaType(ptp)(ptp.paramInfos, this(ptp.resType))
       case mtp: MethodType =>
@@ -138,12 +142,12 @@ object JavaNullInterop {
         // nullify(A & B) = (nullify(A) & nullify(B)) | Null, but take care not to add
         // duplicate `Null`s at the outermost level inside `A` and `B`.
         outermostLevelAlreadyNullable = true
-        OrNull(derivedAndType(tp, this(tp.tp1), this(tp.tp2)))
-      case tp: TypeParamRef if needsNull(tp) => OrNull(tp)
+        nullify(derivedAndType(tp, this(tp.tp1), this(tp.tp2)))
+      case tp: TypeParamRef if needsNull(tp) => nullify(tp)
       // In all other cases, return the type unchanged.
       // In particular, if the type is a ConstantType, then we don't nullify it because it is the
       // type of a final non-nullable field.
       case _ => tp
     }
   }
-}
+}

compiler/src/dotty/tools/dotc/core/NullOpsDecorator.scala

@@ -14,7 +14,7 @@ object NullOpsDecorator:
      *  If this type isn't (syntactically) nullable, then returns the type unchanged.
      *  The type will not be changed if explicit-nulls is not enabled.
      */
-    def stripNull(using Context): Type = {
+    def stripNull(stripFlexibleTypes: Boolean = true)(using Context): Type = {
       def strip(tp: Type): Type =
         val tpWiden = tp.widenDealias
         val tpStripped = tpWiden match {
@@ -33,6 +33,9 @@ object NullOpsDecorator:
             if (tp1s ne tp1) && (tp2s ne tp2) then
               tp.derivedAndType(tp1s, tp2s)
             else tp
+          case tp: FlexibleType =>
+            val hi1 = strip(tp.hi)
+            if stripFlexibleTypes then hi1 else tp.derivedFlexibleType(hi1)
           case tp @ TypeBounds(lo, hi) =>
             tp.derivedTypeBounds(strip(lo), strip(hi))
           case tp => tp
@@ -44,7 +47,7 @@ object NullOpsDecorator:
 
     /** Is self (after widening and dealiasing) a type of the form `T | Null`? */
     def isNullableUnion(using Context): Boolean = {
-      val stripped = self.stripNull
+      val stripped = self.stripNull()
       stripped ne self
     }
   end extension

compiler/src/dotty/tools/dotc/core/OrderingConstraint.scala

@@ -562,11 +562,11 @@ class OrderingConstraint(private val boundsMap: ParamBounds,
         val underlying1 = recur(tp.underlying)
         if underlying1 ne tp.underlying then underlying1 else tp
       case CapturingType(parent, refs) =>
-        val parent1 = recur(parent)
-        if parent1 ne parent then tp.derivedCapturingType(parent1, refs) else tp
+        tp.derivedCapturingType(recur(parent), refs)
+      case tp: FlexibleType =>
+        tp.derivedFlexibleType(recur(tp.hi))
       case tp: AnnotatedType =>
-        val parent1 = recur(tp.parent)
-        if parent1 ne tp.parent then tp.derivedAnnotatedType(parent1, tp.annot) else tp
+        tp.derivedAnnotatedType(recur(tp.parent), tp.annot)
       case _ =>
         val tp1 = tp.dealiasKeepAnnots
         if tp1 ne tp then

compiler/src/dotty/tools/dotc/core/PatternTypeConstrainer.scala

@@ -163,7 +163,7 @@ trait PatternTypeConstrainer { self: TypeComparer =>
       }
     }
 
-    def dealiasDropNonmoduleRefs(tp: Type) = tp.dealias match {
+    def dealiasDropNonmoduleRefs(tp: Type): Type = tp.dealias match {
       case tp: TermRef =>
         // we drop TermRefs that don't have a class symbol, as they can't
         // meaningfully participate in GADT reasoning and just get in the way.
@@ -172,6 +172,7 @@ trait PatternTypeConstrainer { self: TypeComparer =>
         // additional trait - argument-less enum cases desugar to vals.
         // See run/enum-Tree.scala.
         if tp.classSymbol.exists then tp else tp.info
+      case tp: FlexibleType => dealiasDropNonmoduleRefs(tp.underlying)
       case tp => tp
     }
 

compiler/src/dotty/tools/dotc/core/TypeApplications.scala

@@ -541,6 +541,7 @@ class TypeApplications(val self: Type) extends AnyVal {
    */
   final def argInfos(using Context): List[Type] = self.stripped match
     case AppliedType(tycon, args) => args
+    case tp: FlexibleType => tp.underlying.argInfos
     case _ => Nil
 
   /** If this is an encoding of a function type, return its arguments, otherwise return Nil.

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -864,6 +864,8 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
             false
         }
         compareClassInfo
+      case tp2: FlexibleType =>
+        recur(tp1, tp2.lo)
       case _ =>
         fourthTry
     }
@@ -1059,6 +1061,8 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
       case tp1: ExprType if ctx.phaseId > gettersPhase.id =>
         // getters might have converted T to => T, need to compensate.
         recur(tp1.widenExpr, tp2)
+      case tp1: FlexibleType =>
+        recur(tp1.hi, tp2)
       case _ =>
         false
     }
@@ -3437,6 +3441,8 @@ class MatchReducer(initctx: Context) extends TypeComparer(initctx) {
             isConcrete(tp1.underlying)
           case tp1: AndOrType =>
             isConcrete(tp1.tp1) && isConcrete(tp1.tp2)
+          case tp1: FlexibleType =>
+            isConcrete(tp1.hi)
           case _ =>
             val tp2 = tp1.stripped.stripLazyRef
             (tp2 ne tp) && isConcrete(tp2)