From 14681626e8fc772248b10b8dd942f9de9e9aef81 Mon Sep 17 00:00:00 2001 From: Arthur McGibbon Date: Sun, 24 Apr 2022 18:04:58 +0100 Subject: [PATCH] Add filesystem support for jars --- .../java.nio.file.spi.FileSystemProvider | 1 + .../SyntheticsDecorationProvider.scala | 16 +- .../ImplementationProvider.scala | 7 +- .../internal/metals/BuildTargetInfo.scala | 5 +- .../meta/internal/metals/BuildTargets.scala | 85 ++-- .../meta/internal/metals/ClientCommands.scala | 10 + .../internal/metals/ClientConfiguration.scala | 3 + .../internal/metals/DefinitionProvider.scala | 64 ++- .../internal/metals/FileDecoderProvider.scala | 273 +++++++------ .../scala/meta/internal/metals/Indexer.scala | 44 +- .../metals/InitializationOptions.scala | 4 + .../metals/InteractiveSemanticdbs.scala | 35 +- .../meta/internal/metals/JarTopLevels.scala | 13 +- .../metals/JavaInteractiveSemanticdb.scala | 210 +++++----- .../metals/LSPFileSystemProvider.scala | 105 +++++ .../internal/metals/MetalsEnrichments.scala | 32 +- .../metals/MetalsLanguageServer.scala | 34 +- .../internal/metals/PackageProvider.scala | 3 +- .../meta/internal/metals/ServerCommands.scala | 36 ++ .../metals/StandaloneSymbolSearch.scala | 18 +- .../meta/internal/metals/TargetData.scala | 2 + .../debug/ClientConfigurationAdapter.scala | 3 +- .../metals/debug/SourcePathAdapter.scala | 41 +- .../filesystem/MetalsDirectoryStream.scala | 18 + .../filesystem/MetalsFileAttributes.scala | 31 ++ .../metals/filesystem/MetalsFileSystem.scala | 367 +++++++++++++++++ .../filesystem/MetalsFileSystemProvider.scala | 263 ++++++++++++ .../metals/filesystem/MetalsPath.scala | 377 ++++++++++++++++++ .../findfiles/FindTextInDependencyJars.scala | 107 ++--- .../MetalsURLConnection.scala | 16 + .../MetalsURLStreamHandler.scala | 19 + .../MetalsURLStreamHandlerFactory.scala | 26 ++ .../meta/internal/tvp/ClasspathSymbols.scala | 12 +- .../internal/tvp/MetalsTreeViewProvider.scala | 3 +- .../tvp/TreeViewSymbolInformation.scala | 8 +- .../meta/internal/mtags/OpenClassLoader.scala | 6 +- .../internal/mtags/SymbolIndexBucket.scala | 18 +- 37 files changed, 1853 insertions(+), 462 deletions(-) create mode 100644 metals/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider create mode 100644 metals/src/main/scala/scala/meta/internal/metals/LSPFileSystemProvider.scala create mode 100644 metals/src/main/scala/scala/meta/internal/metals/filesystem/MetalsDirectoryStream.scala create mode 100644 metals/src/main/scala/scala/meta/internal/metals/filesystem/MetalsFileAttributes.scala create mode 100644 metals/src/main/scala/scala/meta/internal/metals/filesystem/MetalsFileSystem.scala create mode 100644 metals/src/main/scala/scala/meta/internal/metals/filesystem/MetalsFileSystemProvider.scala create mode 100644 metals/src/main/scala/scala/meta/internal/metals/filesystem/MetalsPath.scala create mode 100644 metals/src/main/scala/scala/meta/internal/metals/urlstreamhandler/MetalsURLConnection.scala create mode 100644 metals/src/main/scala/scala/meta/internal/metals/urlstreamhandler/MetalsURLStreamHandler.scala create mode 100644 metals/src/main/scala/scala/meta/internal/metals/urlstreamhandler/MetalsURLStreamHandlerFactory.scala diff --git a/metals/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider b/metals/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider new file mode 100644 index 00000000000..4e7c79f4597 --- /dev/null +++ b/metals/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider @@ -0,0 +1 @@ +scala.meta.internal.metals.filesystem.MetalsFileSystemProvider \ No newline at end of file diff --git a/metals/src/main/scala/scala/meta/internal/decorations/SyntheticsDecorationProvider.scala b/metals/src/main/scala/scala/meta/internal/decorations/SyntheticsDecorationProvider.scala index 01a1608ab2e..8f98e9f48b0 100644 --- a/metals/src/main/scala/scala/meta/internal/decorations/SyntheticsDecorationProvider.scala +++ b/metals/src/main/scala/scala/meta/internal/decorations/SyntheticsDecorationProvider.scala @@ -282,12 +282,16 @@ final class SyntheticsDecorationProvider( textDocument: Option[s.TextDocument], path: AbsolutePath ): Option[TextDocument] = { - for { - doc <- textDocument - source <- fingerprints.loadLastValid(path, doc.md5, charset) - docWithText = doc.withText(source) - _ = Document.set(docWithText) - } yield docWithText + // TODO what does `enrichWithText` do and should it be run on readOnly files? fingerprints causes slurping ;-( + if (path.isReadOnly) + textDocument + else + for { + doc <- textDocument + source <- fingerprints.loadLastValid(path, doc.md5, charset) + docWithText = doc.withText(source) + _ = Document.set(docWithText) + } yield docWithText } private def localSymbolName(symbol: String, textDoc: TextDocument): String = { diff --git a/metals/src/main/scala/scala/meta/internal/implementation/ImplementationProvider.scala b/metals/src/main/scala/scala/meta/internal/implementation/ImplementationProvider.scala index 5528060976f..ed00b9419e5 100644 --- a/metals/src/main/scala/scala/meta/internal/implementation/ImplementationProvider.scala +++ b/metals/src/main/scala/scala/meta/internal/implementation/ImplementationProvider.scala @@ -395,11 +395,16 @@ final class ImplementationProvider( def isDefinitionOccurrence(occ: SymbolOccurrence) = occ.role.isDefinition && occ.symbol == symbol + val input = + if (source.isReadOnly) + source.toInputFromBuffers(buffer) + else + source.toInput semanticDb.occurrences .find(isDefinitionOccurrence) .orElse( Mtags - .allToplevels(source.toInput, scalaVersionSelector.getDialect(source)) + .allToplevels(input, scalaVersionSelector.getDialect(source)) .occurrences .find(isDefinitionOccurrence) ) diff --git a/metals/src/main/scala/scala/meta/internal/metals/BuildTargetInfo.scala b/metals/src/main/scala/scala/meta/internal/metals/BuildTargetInfo.scala index cf45826ee98..2d1d2883baf 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/BuildTargetInfo.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/BuildTargetInfo.scala @@ -1,5 +1,6 @@ package scala.meta.internal.metals +import java.nio.file.Files import java.nio.file.Path import scala.collection.mutable.ListBuffer @@ -154,7 +155,7 @@ class BuildTargetInfo(buildTargets: BuildTargets) { val sourceJarName = jarName.replace(".jar", "-sources.jar") buildTargets .sourceJarFile(sourceJarName) - .exists(_.toFile.exists()) + .exists(path => path.exists) } private def getSingleClassPathInfo( @@ -164,7 +165,7 @@ class BuildTargetInfo(buildTargets: BuildTargets) { ): String = { val filename = shortPath.toString() val padding = " " * (maxFilenameSize - filename.size) - val status = if (path.toFile.exists) { + val status = if (Files.exists(path)) { val blankWarning = " " * 9 if (path.toFile().isDirectory() || jarHasSource(filename)) blankWarning diff --git a/metals/src/main/scala/scala/meta/internal/metals/BuildTargets.scala b/metals/src/main/scala/scala/meta/internal/metals/BuildTargets.scala index ea2c5717473..8b9e9a233a9 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/BuildTargets.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/BuildTargets.scala @@ -13,6 +13,7 @@ import scala.util.control.NonFatal import scala.meta.internal.io.PathIO import scala.meta.internal.metals.MetalsEnrichments._ +import scala.meta.internal.metals.filesystem.MetalsPath import scala.meta.internal.mtags.Symbol import scala.meta.io.AbsolutePath @@ -93,6 +94,9 @@ final class BuildTargets() { def allJava: Iterator[JavaTarget] = data.fromIterators(_.allJava) + def allJDKs: Iterator[String] = + data.fromIterators(_.allJDKs).distinct + def info(id: BuildTargetIdentifier): Option[BuildTarget] = data.fromOptions(_.info(id)) @@ -267,7 +271,8 @@ final class BuildTargets() { } yield imports /** - * Tries to guess what build target this readonly file belongs to from the symbols it defines. + * Tries to guess what build target this metalsfs or readonly file belongs + * to from the symbols it defines. * * By default, we rely on carefully recording what build target produced what * files in the `.metals/readonly/` directory. This approach has the problem @@ -276,51 +281,53 @@ final class BuildTargets() { * - a new metals feature forgot to record the build target * - a user removes `.metals/metals.h2.db` * - * When encountering an unknown `readonly/` file we do the following steps to + * When encountering an unknown `readonly/` or metalfs file we do the following steps to * infer what build target it belongs to: * * - check if file is in `.metals/readonly/dependencies/${source-jar-name}` + * - check if the file is a jdk file * - find the build targets that have a sourceDependency with that name - * - * Otherwise if it's a jar file we find a build target it belongs to. + * - find the build targets that have a workspace jar with that name as class files can be viewed * * This approach is not glamorous but it seems to work reasonably well. */ def inferBuildTarget( source: AbsolutePath ): Option[BuildTargetIdentifier] = { - if (source.isJarFileSystem) { - for { - jarName <- source.jarPath.map(_.filename) - sourceJarFile <- sourceJarFile(jarName) - buildTargetId <- inverseDependencySource(sourceJarFile).headOption - } yield buildTargetId - } else { - val readonly = workspace.resolve(Directories.readonly) - source.toRelativeInside(readonly) match { - case Some(rel) => - val names = rel.toNIO.iterator().asScala.toList.map(_.filename) - names match { - case Directories.dependenciesName :: jarName :: _ => - // match build target by source jar name - sourceJarFile(jarName) - .flatMap(inverseDependencySource(_).headOption) - case _ => None - } - case None => - // else it can be a source file inside a jar - val fromJar = jarPath(source) - .flatMap { jar => - allBuildTargetIdsInternal.find { case (_, id) => - targetJarClasspath(id).exists(_.contains(jar)) - } - } - fromJar.map { case (data0, id) => - data0.addSourceItem(source, id) - id - } - } + // source could be on metalFS or in readonly area + val metalsPath = source.toNIO match { + case metalsPath: MetalsPath => Some(metalsPath) + case _ => + val readonly = workspace.resolve(Directories.readonly) + source.toRelativeInside(readonly) match { + case Some(rel) => Some(MetalsPath.fromReadOnly(rel)) + case None => None + } } + metalsPath.flatMap(metalsPath => { + // check JDK - any build target is OK + if (metalsPath.isJDK) + allBuildTargetIdsInternal.headOption.map(_._2) + else { + // check source jars + for { + jarName <- metalsPath.jarName + jarPath <- sourceJarFile(jarName) + buildTargetId <- inverseDependencySource(jarPath).headOption + } yield buildTargetId + }.orElse({ + // check workspace jars + // TODO speed this up - we're iterating over all build targets and all classpath entries - create inverseDependencySource equivalent + val targetIds = for { + targetId <- allBuildTargetIds + classpathEntries <- targetJarClasspath(targetId).toList + classpathEntry <- classpathEntries + jarName <- metalsPath.jarName + if classpathEntry.filename == jarName + } yield targetId + targetIds.headOption + }) + }) } def findByDisplayName(name: String): Option[BuildTarget] = { @@ -329,14 +336,6 @@ final class BuildTargets() { .find(_.getDisplayName() == name) } - private def jarPath(source: AbsolutePath): Option[AbsolutePath] = { - source.jarPath.map { sourceJarPath => - sourceJarPath.parent.resolve( - source.filename.replace("-sources.jar", ".jar") - ) - } - } - /** * Returns meta build target for `*.sbt` or `*.scala` files. * It selects build target by directory of its connection diff --git a/metals/src/main/scala/scala/meta/internal/metals/ClientCommands.scala b/metals/src/main/scala/scala/meta/internal/metals/ClientCommands.scala index 33b94595e8f..b9e22105066 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/ClientCommands.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/ClientCommands.scala @@ -241,6 +241,15 @@ object ClientCommands { |""".stripMargin ) + val CreateLibraryFileSystem = new ParametrizedCommand[String]( + "metals-create-library-filesystem", + "Create Library FS", + """|Notifies the client that it should create an empty + |filesystem to navigate jar dependencies + |""".stripMargin, + arguments = """`string`, the URI root of the filesystem.""".stripMargin + ) + val RefreshModel = new Command( "metals-model-refresh", "Refresh model", @@ -325,6 +334,7 @@ object ClientCommands { FocusDiagnostics, GotoLocation, EchoCommand, + CreateLibraryFileSystem, RefreshModel, ShowStacktrace, CopyWorksheetOutput, diff --git a/metals/src/main/scala/scala/meta/internal/metals/ClientConfiguration.scala b/metals/src/main/scala/scala/meta/internal/metals/ClientConfiguration.scala index 8d6bd2f66e9..8071bc758a1 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/ClientConfiguration.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/ClientConfiguration.scala @@ -57,6 +57,9 @@ case class ClientConfiguration(initialConfig: MetalsServerConfig) { def isVirtualDocumentSupported(): Boolean = initializationOptions.isVirtualDocumentSupported.getOrElse(false) + def isLibraryFileSystemSupported(): Boolean = true + // initializationOptions.isLibraryFileSystemSupported.getOrElse(false) + def icons(): Icons = initializationOptions.icons .map(Icons.fromString) diff --git a/metals/src/main/scala/scala/meta/internal/metals/DefinitionProvider.scala b/metals/src/main/scala/scala/meta/internal/metals/DefinitionProvider.scala index 49c56b0691e..c3f19adb74d 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/DefinitionProvider.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/DefinitionProvider.scala @@ -1,5 +1,6 @@ package scala.meta.internal.metals +import java.nio.file.Paths import java.util.Collections import java.{util => ju} @@ -8,6 +9,7 @@ import scala.concurrent.Future import scala.meta.inputs.Input import scala.meta.internal.metals.MetalsEnrichments._ +import scala.meta.internal.metals.filesystem.MetalsFileSystem import scala.meta.internal.mtags.GlobalSymbolIndex import scala.meta.internal.mtags.Mtags import scala.meta.internal.mtags.Semanticdbs @@ -16,9 +18,11 @@ import scala.meta.internal.mtags.SymbolDefinition import scala.meta.internal.parsing.TokenEditDistance import scala.meta.internal.parsing.Trees import scala.meta.internal.remotels.RemoteLanguageServer +import scala.meta.internal.semanticdb.Language import scala.meta.internal.semanticdb.Scala._ import scala.meta.internal.semanticdb.SymbolOccurrence import scala.meta.internal.semanticdb.TextDocument +import scala.meta.internal.tvp.ClasspathSymbols import scala.meta.io.AbsolutePath import scala.meta.pc.CancelToken @@ -265,6 +269,7 @@ final class DefinitionProvider( } private def fromMtags(source: AbsolutePath, dirtyPos: Position) = { + // TODO why does a writable file use `source.toInput` instead of `source.toInputFromBuffers(buffers)` Mtags .allToplevels(source.toInput, scalaVersionSelector.getDialect(source)) .occurrences @@ -316,12 +321,28 @@ class DestinationProvider( private def bestTextDocument( symbolDefinition: SymbolDefinition ): TextDocument = { - val defnRevisedInput = symbolDefinition.path.toInput + val decodedClassfile = + if (symbolDefinition.path.isClassfile) + MetalsFileSystem.metalsFS + .decodeCFRFromClassFile(symbolDefinition.path.toNIO) + .map(text => + Input.VirtualFile(symbolDefinition.path.toURI.toString(), text) + ) + else None + val defnRevisedInput = decodedClassfile.getOrElse( + if (symbolDefinition.path.isReadOnly) + symbolDefinition.path.toInputFromBuffers(buffers) + else + symbolDefinition.path.toInput + ) // Read text file from disk instead of editor buffers because the file // on disk is more likely to parse. + val language = + if (symbolDefinition.path.isClassfile) Language.JAVA + else symbolDefinition.path.toLanguage lazy val parsed = mtags.index( - symbolDefinition.path.toLanguage, + language, defnRevisedInput, symbolDefinition.dialect ) @@ -346,15 +367,46 @@ class DestinationProvider( definition(symbol, targets) } + private val classpathSymbols = new ClasspathSymbols( + isStatisticsEnabled = false + ) + def definition( symbol: String, allowedBuildTargets: Set[BuildTargetIdentifier] ): Option[SymbolDefinition] = { - val definitions = index.definitions(Symbol(symbol)).filter(_.path.exists) + val querySymbol: Symbol = Symbol(symbol) + val definitions = index.definitions(querySymbol).filter(_.path.exists) + val extraDefinitions = if (definitions.isEmpty) { + // search in all classpath, i.e. jars with no source + buildTargets + .inferBuildTarget(List(querySymbol.toplevel)) + .flatMap(buildTarget => { + val metalsFSJar = + MetalsFileSystem.metalsFS + .getOrElseUpdateWorkspaceJar(buildTarget.jar) + classpathSymbols + .symbols(metalsFSJar, symbol) + .filter(_.symbol == symbol) + .flatMap(_.uri) + .headOption + .map(symbolFullURI => { + val fullJarPath = AbsolutePath(Paths.get(symbolFullURI)) + SymbolDefinition( + querySymbol, + querySymbol, + fullJarPath, + scala.meta.dialects.Scala212Source3, + None + ) + }) + }) + .toList + } else definitions if (allowedBuildTargets.isEmpty) - definitions.headOption + extraDefinitions.headOption else { - val matched = definitions.find { defn => + val matched = extraDefinitions.find { defn => sourceBuildTargets(defn.path).exists(id => allowedBuildTargets.contains(id) ) @@ -362,7 +414,7 @@ class DestinationProvider( // Fallback to any definition - it's needed for worksheets // They might have dynamic `import $dep` and these sources jars // aren't registered in buildTargets - matched.orElse(definitions.headOption) + matched.orElse(extraDefinitions.headOption) } } diff --git a/metals/src/main/scala/scala/meta/internal/metals/FileDecoderProvider.scala b/metals/src/main/scala/scala/meta/internal/metals/FileDecoderProvider.scala index 95b0aab6ce2..77ea446af4d 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/FileDecoderProvider.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/FileDecoderProvider.scala @@ -5,7 +5,7 @@ import java.io.File import java.io.PrintStream import java.net.URI import java.net.URLEncoder -import java.nio.charset.StandardCharsets +import java.nio.file.Files import javax.annotation.Nullable import scala.annotation.tailrec @@ -19,11 +19,11 @@ import scala.util.control.NonFatal import scala.meta.cli.Reporter import scala.meta.internal.builds.ShellRunner -import scala.meta.internal.io.FileIO import scala.meta.internal.metals.MetalsEnrichments._ import scala.meta.internal.metals.clients.language.MetalsLanguageClient import scala.meta.internal.metals.clients.language.MetalsQuickPickItem import scala.meta.internal.metals.clients.language.MetalsQuickPickParams +import scala.meta.internal.metals.filesystem.MetalsFileSystemProvider import scala.meta.internal.metap.DocumentPrinter import scala.meta.internal.metap.Main import scala.meta.internal.mtags.SemanticdbClasspath @@ -129,23 +129,24 @@ final class FileDecoderProvider( * semanticdb: * metalsDecode:file:///somePath/someFile.java.semanticdb-compact * metalsDecode:file:///somePath/someFile.java.semanticdb-detailed + * metalsDecode:file:///somePath/someFile.java.semanticdb-proto * * metalsDecode:file:///somePath/someFile.scala.semanticdb-compact * metalsDecode:file:///somePath/someFile.scala.semanticdb-detailed + * metalsDecode:file:///somePath/someFile.scala.semanticdb-proto * * metalsDecode:file:///somePath/someFile.java.semanticdb.semanticdb-compact * metalsDecode:file:///somePath/someFile.java.semanticdb.semanticdb-detailed + * metalsDecode:file:///somePath/someFile.java.semanticdb.semanticdb-proto * * metalsDecode:file:///somePath/someFile.scala.semanticdb.semanticdb-compact * metalsDecode:file:///somePath/someFile.scala.semanticdb.semanticdb-detailed + * metalsDecode:file:///somePath/someFile.scala.semanticdb.semanticdb-proto * * tasty: * metalsDecode:file:///somePath/someFile.scala.tasty-decoded * metalsDecode:file:///somePath/someFile.tasty.tasty-decoded * - * jar: - * metalsDecode:jar:file:///somePath/someFile-sources.jar!/somePackage/someFile.java - * * build target: * metalsDecode:file:///workspacePath/buildTargetName.metals-buildtarget */ @@ -153,13 +154,8 @@ final class FileDecoderProvider( Try(URI.create(URIEncoderDecoder.encode(uriAsStr))) match { case Success(uri) => uri.getScheme() match { - case "jar" => - val jarURI = convertJarStrToURI(uriAsStr) - if (semanticdbExtensions.exists(uriAsStr.endsWith)) - decodeMetalsFile(jarURI) - else - Future { decodeJar(jarURI) } - case "file" => decodeMetalsFile(uri) + case "file" | MetalsFileSystemProvider.scheme => + decodeMetalsFile(uri) case "metalsDecode" => decodedFileContents(uri.getSchemeSpecificPart()) case _ => @@ -177,25 +173,6 @@ final class FileDecoderProvider( } } - private def convertJarStrToURI(uriAsStr: String): URI = { - - /** - * URI.create will decode the string, which means ZipFileSystemProvider will not work - * Related stack question: https://stackoverflow.com/questions/9873845/java-7-zip-file-system-provider-doesnt-seem-to-accept-spaces-in-uri - */ - new URI("jar", uriAsStr.stripPrefix("jar:"), null) - } - - private def decodeJar(uri: URI): DecoderResponse = { - Try { - val path = uri.toAbsolutePath - FileIO.slurp(path, StandardCharsets.UTF_8) - } match { - case Failure(exception) => DecoderResponse.failed(uri, exception) - case Success(value) => DecoderResponse.success(uri, value) - } - } - private val semanticdbExtensions = Set( "semanticdb-compact", "semanticdb-detailed", @@ -304,7 +281,7 @@ final class FileDecoderProvider( path: AbsolutePath, format: Format ): DecoderResponse = { - if (path.isScalaOrJava) + if (path.isScalaOrJava || path.isClassfile) interactiveSemanticdbs .textDocument(path) .documentIncludingStale @@ -510,6 +487,13 @@ final class FileDecoderProvider( private def decodeJavapFromClassFile( verbose: Boolean )(path: AbsolutePath): Future[DecoderResponse] = { + // javap doesn't understand the metals filesystem so save to disk + val (workDir, pathOnDisk) = if (path.isMetalsFileSystem) { + val workDir = AbsolutePath(Files.createTempDirectory("metals-javap")) + val classFile = workDir.resolve(path.filename) + Files.copy(path.toNIO, classFile.toNIO) + (Some(workDir), classFile) + } else (None, path) try { val defaultArgs = List("-private") val args = if (verbose) "-verbose" :: defaultArgs else defaultArgs @@ -519,9 +503,9 @@ final class FileDecoderProvider( .run( "Decode using javap", JavaBinary(userConfig().javaHome, "javap") :: args ::: List( - path.filename + pathOnDisk.filename ), - path.parent, + pathOnDisk.parent, redirectErrorOutput = false, Map.empty, s => { @@ -536,6 +520,7 @@ final class FileDecoderProvider( logInfo = false ) .map(_ => { + workDir.foreach(_.deleteRecursively()) if (sbErr.nonEmpty) DecoderResponse.failed(path.toURI, sbErr.toString) else @@ -543,6 +528,7 @@ final class FileDecoderProvider( }) } catch { case NonFatal(e) => + workDir.foreach(_.deleteRecursively()) scribe.error(e.toString()) Future.successful(DecoderResponse.failed(path.toURI, e)) } @@ -550,104 +536,8 @@ final class FileDecoderProvider( private def decodeCFRFromClassFile( path: AbsolutePath - ): Future[DecoderResponse] = { - val cfrDependency = Dependency.of("org.benf", "cfr", "0.151") - val cfrMain = "org.benf.cfr.reader.Main" - - val buildTarget = buildTargets - .inferBuildTarget(path) - .orElse( - buildTargets.allScala - .find(buildTarget => - path.isInside(buildTarget.classDirectory.toAbsolutePath) - ) - .map(_.id) - ) - .orElse( - buildTargets.allJava - .find(buildTarget => - path.isInside(buildTarget.classDirectory.toAbsolutePath) - ) - .map(_.id) - ) - val classpaths = buildTarget - .flatMap(id => - buildTargets - .targetClasspath(id) - .map(path => path.toAbsoluteClasspath.toList) - ) - .getOrElse(Nil) - val classesDirs = buildTarget - .map(id => - buildTargets.targetClassDirectories(id).toAbsoluteClasspath.toList - ) - .getOrElse(Nil) - val extraClassPaths = classesDirs ::: classpaths - val extraClassPath = - if (extraClassPaths.nonEmpty) - List("--extraclasspath", extraClassPaths.mkString(File.pathSeparator)) - else Nil - - // if the class file is in the classes dir then use that classes dir to allow CFR to use the other classes - val (parent, className) = classesDirs - .find(classesPath => path.isInside(classesPath)) - .map(classesPath => { - val classPath = path.toRelative(classesPath) - val className = classPath.toString - (classesPath, className) - }) - .getOrElse({ - val parent = path.parent - val className = path.filename - (parent, className) - }) - - val args = extraClassPath ::: - List( - // elideScala - must be lowercase - hide @@ScalaSignature and serialVersionUID for aesthetic reasons - "--elidescala", - "true", - // analyseAs - must be lowercase - "--analyseas", - "CLASS", - s"$className" - ) - - val sbOut = new StringBuilder() - val sbErr = new StringBuilder() - try { - shellRunner - .runJava( - cfrDependency, - cfrMain, - parent, - args, - redirectErrorOutput = false, - s => { - sbOut.append(s) - sbOut.append(Properties.lineSeparator) - }, - s => { - sbErr.append(s) - sbErr.append(Properties.lineSeparator) - }, - propagateError = true - ) - .map(_ => { - if (sbErr.nonEmpty) - DecoderResponse.failed( - path.toURI, - s"$cfrDependency\n$cfrMain\n$parent\n$args\n${sbErr.toString}" - ) - else - DecoderResponse.success(path.toURI, sbOut.toString) - }) - } catch { - case NonFatal(e) => - scribe.error(e.toString()) - Future.successful(DecoderResponse.failed(path.toURI, e)) - } - } + ): Future[DecoderResponse] = + FileDecoderProvider.decodeCFRFromClassFile(shellRunner, buildTargets, path) private def decodeFromSemanticDB( path: AbsolutePath, @@ -690,7 +580,6 @@ final class FileDecoderProvider( printer.print() } ) - private def decodeFromSemanticDBFile( path: AbsolutePath, format: Format @@ -755,4 +644,122 @@ object FileDecoderProvider { URI.create( s"metalsDecode:${URLEncoder.encode(s"file:///${workspace.filename}/${buildTargetName}.metals-buildtarget")}" ) + + def decodeCFRFromClassFile( + shellRunner: ShellRunner, + buildTargets: BuildTargets, + path: AbsolutePath + )(implicit ec: ExecutionContext): Future[DecoderResponse] = { + + val cfrDependency = Dependency.of("org.benf", "cfr", "0.151") + val cfrMain = "org.benf.cfr.reader.Main" + + val buildTarget = buildTargets + .inferBuildTarget(path) + .orElse( + buildTargets.allScala + .find(buildTarget => + path.isInside(buildTarget.classDirectory.toAbsolutePath) + ) + .map(_.id) + ) + .orElse( + buildTargets.allJava + .find(buildTarget => + path.isInside(buildTarget.classDirectory.toAbsolutePath) + ) + .map(_.id) + ) + val classpaths = buildTarget + .flatMap(id => + buildTargets + .targetClasspath(id) + .map(path => path.toAbsoluteClasspath.toList) + ) + .getOrElse(Nil) + val classesDirs = buildTarget + .map(id => + buildTargets.targetClassDirectories(id).toAbsoluteClasspath.toList + ) + .getOrElse(Nil) + val extraClassPaths = classesDirs ::: classpaths + val extraClassPath = + if (extraClassPaths.nonEmpty) + List("--extraclasspath", extraClassPaths.mkString(File.pathSeparator)) + else Nil + + // CFR doesn't understand the metals filesystem so save to disk + val (workDir, parent, className) = if (path.isMetalsFileSystem) { + val workDir = AbsolutePath(Files.createTempDirectory("metals-cfr")) + val classFile = workDir.resolve(path.filename) + Files.copy(path.toNIO, classFile.toNIO) + val parent = classFile.parent + val className = classFile.filename + (Some(workDir), parent, className) + } else + // if the class file is in the classes dir then use that classes dir to allow CFR to use the other classes + classesDirs + .find(classesPath => path.isInside(classesPath)) + .map(classesPath => { + val classPath = path.toRelative(classesPath) + val className = classPath.toString + (None, classesPath, className) + }) + .getOrElse({ + val parent = path.parent + val className = path.filename + (None, parent, className) + }) + + val args = extraClassPath ::: + List( + // elideScala - must be lowercase - hide @@ScalaSignature and serialVersionUID for aesthetic reasons + "--elidescala", + "true", + // useNameTable - must be lowercase - produced `void` variables which meant the output can't be recompiled in interactivesemanticdb + "--usenametable", + "false", + // analyseAs - must be lowercase + "--analyseas", + "CLASS", + s"$className" + ) + val sbOut = new StringBuilder() + val sbErr = new StringBuilder() + + try { + shellRunner + .runJava( + cfrDependency, + cfrMain, + parent, + args, + redirectErrorOutput = false, + s => { + sbOut.append(s) + sbOut.append(Properties.lineSeparator) + }, + s => { + sbErr.append(s) + sbErr.append(Properties.lineSeparator) + }, + propagateError = true + ) + .map(_ => { + workDir.foreach(_.deleteRecursively()) + if (sbErr.nonEmpty) + DecoderResponse.failed( + path.toURI, + s"$cfrDependency\n$cfrMain\n$parent\n$args\n${sbErr.toString}" + ) + else + DecoderResponse.success(path.toURI, sbOut.toString) + }) + } catch { + case NonFatal(e) => + workDir.foreach(_.deleteRecursively()) + scribe.error(e.toString()) + Future.successful(DecoderResponse.failed(path.toURI, e)) + } + } } diff --git a/metals/src/main/scala/scala/meta/internal/metals/Indexer.scala b/metals/src/main/scala/scala/meta/internal/metals/Indexer.scala index 0562e820c34..a2908bade6a 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/Indexer.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/Indexer.scala @@ -27,6 +27,8 @@ import scala.meta.internal.metals.clients.language.DelegatingLanguageClient import scala.meta.internal.metals.clients.language.ForwardingMetalsBuildClient import scala.meta.internal.metals.debug.BuildTargetClasses import scala.meta.internal.metals.doctor.Doctor +import scala.meta.internal.metals.filesystem.MetalsFileSystem +import scala.meta.internal.metals.urlstreamhandler.MetalsURLStreamHandlerFactory import scala.meta.internal.metals.watcher.FileWatcher import scala.meta.internal.mtags.OnDemandSymbolIndex import scala.meta.internal.semanticdb.Scala._ @@ -43,6 +45,7 @@ import ch.epfl.scala.{bsp4j => b} final case class Indexer( workspaceReload: () => WorkspaceReload, doctor: () => Doctor, + lspFileSystemProvider: () => LSPFileSystemProvider, languageClient: DelegatingLanguageClient, bspSession: () => Option[BspSession], executionContext: ExecutionContextExecutorService, @@ -252,6 +255,8 @@ final case class Indexer( ) { indexWorkspaceSources(buildTool.data) } + // register metalsfs url handler before indexing jars/jdks + MetalsURLStreamHandlerFactory.register var usedJars = Set.empty[AbsolutePath] for (buildTool <- allBuildTargetsData) timerProvider.timedThunk( @@ -267,6 +272,11 @@ final case class Indexer( buildTool.importedBuild.dependencySources ) } + buildTargets.allWorkspaceJars.foreach( + MetalsFileSystem.metalsFS.getOrElseUpdateWorkspaceJar(_) + ) + // create library FS after jars have been indexed + lspFileSystemProvider().createLibraryFileSystem() // Schedule removal of unused toplevel symbols from cache if (usedJars.nonEmpty) sh.schedule( @@ -346,16 +356,20 @@ final case class Indexer( for { item <- dependencySources.getItems.asScala sourceUri <- Option(item.getSources).toList.flatMap(_.asScala) - path = sourceUri.toAbsolutePath - _ = data.addDependencySource(path, item.getTarget) + directPath = sourceUri.toAbsolutePath + mappedPath = + if (directPath.isJar) + MetalsFileSystem.metalsFS.getOrElseUpdateSourceJar(directPath) + else directPath + _ = data.addDependencySource(mappedPath, item.getTarget) if !isVisited.contains(sourceUri) } { isVisited.add(sourceUri) try { - if (path.isJar) { - usedJars += path - addSourceJarSymbols(path) - } else if (path.isDirectory) { + if (mappedPath.isJar) { + usedJars += mappedPath + addSourceJarSymbols(mappedPath) + } else if (mappedPath.isDirectory) { val dialect = buildTargets .scalaTarget(item.getTarget) .map(scalaTarget => @@ -365,9 +379,9 @@ final case class Indexer( ) ) .getOrElse(Scala213) - definitionIndex.addSourceDirectory(path, dialect) + definitionIndex.addSourceDirectory(mappedPath, dialect) } else { - scribe.warn(s"unexpected dependency: $path") + scribe.warn(s"unexpected dependency: $mappedPath") } } catch { case NonFatal(e) => @@ -388,20 +402,24 @@ final case class Indexer( val jdkSources = JdkSources(userConfig().javaHome) jdkSources match { case Right(zip) => - usedJars += zip - addSourceJarSymbols(zip) + val metalsFSZip = MetalsFileSystem.metalsFS.getOrElseUpdateJDK(zip) + usedJars += metalsFSZip + addSourceJarSymbols(metalsFSZip) case Left(notFound) => val candidates = notFound.candidates.mkString(", ") scribe.warn( s"Could not find java sources in $candidates. Java symbols will not be available." ) } + for { item <- dependencySources.getItems.asScala } { - jdkSources.foreach(source => - data.addDependencySource(source, item.getTarget) - ) + jdkSources.foreach(source => { + val metalsFSZip = + MetalsFileSystem.metalsFS.getOrElseUpdateJDK(source) + data.addDependencySource(metalsFSZip, item.getTarget) + }) } usedJars.toSet } diff --git a/metals/src/main/scala/scala/meta/internal/metals/InitializationOptions.scala b/metals/src/main/scala/scala/meta/internal/metals/InitializationOptions.scala index fcc4e54bd77..c2558bbecd1 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/InitializationOptions.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/InitializationOptions.scala @@ -65,6 +65,7 @@ final case class InitializationOptions( isHttpEnabled: Option[Boolean], commandInHtmlFormat: Option[CommandHTMLFormat], isVirtualDocumentSupported: Option[Boolean], + isLibraryFileSystemSupported: Option[Boolean], openFilesOnRenameProvider: Option[Boolean], quickPickProvider: Option[Boolean], renameFileThreshold: Option[Int], @@ -112,6 +113,7 @@ object InitializationOptions { None, None, None, + None, None ) @@ -155,6 +157,8 @@ object InitializationOptions { .flatMap(CommandHTMLFormat.fromString), isVirtualDocumentSupported = jsonObj.getBooleanOption("isVirtualDocumentSupported"), + isLibraryFileSystemSupported = + jsonObj.getBooleanOption("isLibraryFileSystemSupported"), openFilesOnRenameProvider = jsonObj.getBooleanOption("openFilesOnRenameProvider"), quickPickProvider = jsonObj.getBooleanOption("quickPickProvider"), diff --git a/metals/src/main/scala/scala/meta/internal/metals/InteractiveSemanticdbs.scala b/metals/src/main/scala/scala/meta/internal/metals/InteractiveSemanticdbs.scala index c6c045a423f..356e08eb85e 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/InteractiveSemanticdbs.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/InteractiveSemanticdbs.scala @@ -75,11 +75,12 @@ final class InteractiveSemanticdbs( source.isSbt || // sbt files source.isWorksheet || // worksheets doesNotBelongToBuildTarget // standalone files - ) || source.isJarFileSystem // dependencies + ) || source.isMetalsFileSystem } - // anything aside from `*.scala`, `*.sbt`, `*.sc`, `*.java` file - def isExcludedFile = !source.isScalaFilename && !source.isJavaFilename + // anything aside from `*.scala`, `*.sbt`, `*.sc`, `*.java`, `*.class` file + def isExcludedFile = + !source.isScalaFilename && !source.isJavaFilename && !source.isClassfile if (isExcludedFile || !shouldTryCalculateInteractiveSemanticdb) { TextDocumentLookup.NotFound(source) @@ -87,18 +88,22 @@ final class InteractiveSemanticdbs( val result = textDocumentCache.compute( source, (path, existingDoc) => { - val text = unsavedContents.getOrElse(FileIO.slurp(source, charset)) - val sha = MD5.compute(text) - if (existingDoc == null || existingDoc.md5 != sha) { - Try(compile(path, text)) match { - case Success(doc) if doc != null => - if (!source.isDependencySource(workspace)) - semanticdbIndexer().onChange(source, doc) - doc - case _ => null - } - } else + if (existingDoc != null && source.isReadOnly) existingDoc + else { + val text = unsavedContents.getOrElse(FileIO.slurp(source, charset)) + val sha = MD5.compute(text) + if (existingDoc == null || existingDoc.md5 != sha) { + Try(compile(path, text)) match { + case Success(doc) if doc != null => + if (!source.isDependencySource(workspace)) + semanticdbIndexer().onChange(source, doc) + doc + case _ => null + } + } else + existingDoc + } } ) TextDocumentLookup.fromOption(source, Option(result)) @@ -161,7 +166,7 @@ final class InteractiveSemanticdbs( } private def compile(source: AbsolutePath, text: String): s.TextDocument = { - if (source.isJavaFilename) + if (source.isJavaFilename || source.isClassfile) javaInteractiveSemanticdb.fold(s.TextDocument())( _.textDocument(source, text) ) diff --git a/metals/src/main/scala/scala/meta/internal/metals/JarTopLevels.scala b/metals/src/main/scala/scala/meta/internal/metals/JarTopLevels.scala index 609caf9e500..ff7ae8460eb 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/JarTopLevels.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/JarTopLevels.scala @@ -1,15 +1,14 @@ package scala.meta.internal.metals import java.nio.file.Files +import java.nio.file.Paths import java.nio.file.attribute.BasicFileAttributeView import java.sql.Connection import java.sql.PreparedStatement import java.sql.Statement import java.util.zip.ZipError -import scala.meta.internal.io.PlatformFileIO import scala.meta.internal.metals.JdbcEnrichments._ -import scala.meta.internal.metals.MetalsEnrichments._ import scala.meta.internal.mtags.MD5 import scala.meta.io.AbsolutePath @@ -30,14 +29,6 @@ final class JarTopLevels(conn: () => Connection) { path: AbsolutePath ): Option[List[(String, AbsolutePath)]] = try { - val fs = path.jarPath - .map(jarPath => - PlatformFileIO.newFileSystem( - jarPath.toURI, - new java.util.HashMap[String, String]() - ) - ) - .getOrElse(PlatformFileIO.newJarFileSystem(path, create = false)) val toplevels = List.newBuilder[(String, AbsolutePath)] conn() .query( @@ -49,7 +40,7 @@ final class JarTopLevels(conn: () => Connection) { ) { _.setString(1, getMD5Digest(path)) } { rs => if (rs.getString(1) != null && rs.getString(2) != null) { val symbol = rs.getString(1) - val path = AbsolutePath(fs.getPath(rs.getString(2))) + val path = AbsolutePath(Paths.get(rs.getString(2))) toplevels += (symbol -> path) } } diff --git a/metals/src/main/scala/scala/meta/internal/metals/JavaInteractiveSemanticdb.scala b/metals/src/main/scala/scala/meta/internal/metals/JavaInteractiveSemanticdb.scala index 27044f95eac..6c9cb0085a3 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/JavaInteractiveSemanticdb.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/JavaInteractiveSemanticdb.scala @@ -13,6 +13,7 @@ import scala.util.control.NonFatal import scala.meta.internal.io.FileIO import scala.meta.internal.metals.MetalsEnrichments._ +import scala.meta.internal.metals.filesystem.MetalsPath import scala.meta.internal.mtags.MD5 import scala.meta.internal.process.SystemProcess import scala.meta.internal.{semanticdb => s} @@ -32,92 +33,105 @@ class JavaInteractiveSemanticdb( private val readonly = workspace.resolve(Directories.readonly) - def textDocument(source: AbsolutePath, text: String): s.TextDocument = { - val workDir = AbsolutePath( - Files.createTempDirectory("metals-javac-semanticdb") - ) - val targetRoot = workDir.resolve("target") - Files.createDirectory(targetRoot.toNIO) + def textDocument(source: AbsolutePath, text: String): s.TextDocument = + if (source.filename == "module-info.java") { + // can't run semanticdb on module-info.java files + s.TextDocument( + uri = source.toURI.toString(), + text = text, + md5 = MD5.compute(text) + ) + } else { + val workDir = AbsolutePath( + Files.createTempDirectory("metals-javac-semanticdb") + ) + val targetRoot = workDir.resolve("target") + Files.createDirectory(targetRoot.toNIO) + + val localSource = + if (source.isLocalFileSystem(workspace) && source.isJavaFilename) + source + else { + val sourceRoot = workDir.resolve("source") + Files.createDirectory(sourceRoot.toNIO) + // cope with this file being a de-compiled class file with ext .class + val localSource = + sourceRoot.resolve( + s"${source.filename.stripSuffix(".class").stripSuffix(".java")}.java" + ) + Files.write(localSource.toNIO, text.getBytes) + localSource + } + + val sourceRoot = localSource.parent + val targetClasspath = + buildTargets + .inferBuildTarget(source) + .flatMap(buildTargets.targetJarClasspath) + .getOrElse(Nil) + .map(_.toString) + + val jigsawOptions = + addExportsFlags ++ patchModuleFlags(localSource, sourceRoot, source) + val mainOptions = + List( + javac.toString, + "-cp", + (pluginJars ++ targetClasspath).mkString(File.pathSeparator), + "-d", + targetRoot.toString + ) + val pluginOption = + s"-Xplugin:semanticdb -sourceroot:${sourceRoot} -targetroot:${targetRoot}" + val cmd = + mainOptions ::: jigsawOptions ::: pluginOption :: localSource.toString :: Nil + + val stdout = List.newBuilder[String] + val ps = SystemProcess.run( + cmd, + workDir, + false, + Map.empty, + Some(outLine => stdout += outLine), + Some(errLine => stdout += errLine) + ) - val localSource = - if (source.isLocalFileSystem(workspace)) { - source + val future = ps.complete.recover { case NonFatal(e) => + scribe.error(s"Running javac-semanticdb failed for $localSource", e) + 1 + } + + val exitCode = Await.result(future, 10.seconds) + val semanticdbFile = + targetRoot + .resolve("META-INF") + .resolve("semanticdb") + .resolve(s"${localSource.filename}.semanticdb") + + val doc = if (semanticdbFile.exists) { + FileIO + .readAllDocuments(semanticdbFile) + .headOption + .getOrElse(s.TextDocument()) } else { - val sourceRoot = workDir.resolve("source") - Files.createDirectory(sourceRoot.toNIO) - val localSource = sourceRoot.resolve(source.filename) - Files.write(localSource.toNIO, text.getBytes) - localSource + val log = stdout.result() + if (exitCode != 0 || log.nonEmpty) + scribe.warn( + s"Running javac-semanticdb failed for ${source}.\n$cmd\nOutput:\n${log.mkString("\n")}" + ) + s.TextDocument() } - val sourceRoot = localSource.parent - val targetClasspath = buildTargets - .inferBuildTarget(source) - .flatMap(buildTargets.targetJarClasspath) - .getOrElse(Nil) - .map(_.toString) - - val jigsawOptions = - addExportsFlags ++ patchModuleFlags(localSource, sourceRoot, source) - val mainOptions = - List( - javac.toString, - "-cp", - (pluginJars ++ targetClasspath).mkString(File.pathSeparator), - "-d", - targetRoot.toString + val out = doc.copy( + uri = source.toURI.toString(), + text = text, + md5 = MD5.compute(text) ) - val pluginOption = - s"-Xplugin:semanticdb -sourceroot:${sourceRoot} -targetroot:${targetRoot}" - val cmd = - mainOptions ::: jigsawOptions ::: pluginOption :: localSource.toString :: Nil - - val stdout = List.newBuilder[String] - val ps = SystemProcess.run( - cmd, - workDir, - false, - Map.empty, - Some(outLine => stdout += outLine), - Some(errLine => stdout += errLine) - ) - - val future = ps.complete.recover { case NonFatal(e) => - scribe.error(s"Running javac-semanticdb failed for $localSource", e) - 1 - } - val exitCode = Await.result(future, 10.seconds) - val semanticdbFile = - targetRoot - .resolve("META-INF") - .resolve("semanticdb") - .resolve(s"${localSource.filename}.semanticdb") - - val doc = if (semanticdbFile.exists) { - FileIO - .readAllDocuments(semanticdbFile) - .headOption - .getOrElse(s.TextDocument()) - } else { - val log = stdout.result() - if (exitCode != 0 || log.nonEmpty) - scribe.warn( - s"Running javac-semanticdb failed for ${source.toURI}. Output:\n${log.mkString("\n")}" - ) - s.TextDocument() + workDir.deleteRecursively() + out } - val out = doc.copy( - uri = source.toURI.toString(), - text = text, - md5 = MD5.compute(text) - ) - - workDir.deleteRecursively() - out - } - private def patchModuleFlags( source: AbsolutePath, sourceRoot: AbsolutePath, @@ -127,37 +141,25 @@ class JavaInteractiveSemanticdb( // that is declared in some existing module. // It fails with: `error: package exists in another module: $packageName` // but it might be fixed by passing `--patch-module $moduleName=$sourceRoot` option. - // - // Currently there is no infrastucture to detect if package belong to jigsaw module or not - // so this case is covered only for JDK sources. - if (jdkVersion.hasJigsaw) { - source.toRelativeInside(readonly) match { - case Some(rel) => - val names = rel.toNIO.iterator().asScala.toList.map(_.filename) - names match { - case Directories.dependenciesName :: JdkSources.zipFileName :: moduleName :: _ => - List("--patch-module", s"$moduleName=$sourceRoot") - case _ => - Nil - } - case None => - if ( - originalSource.jarPath.exists(_.filename == JdkSources.zipFileName) - ) { - originalSource.toNIO - .iterator() - .asScala - .headOption - .map(_.filename) + source.toRelativeInside(readonly) match { + case Some(rel) => + MetalsPath + .fromReadOnly(rel) + .moduleName + .map(moduleName => List("--patch-module", s"$moduleName=$sourceRoot")) + .getOrElse(Nil) + case None => + originalSource.toNIO match { + case metalsPath: MetalsPath => + metalsPath.moduleName .map(moduleName => List("--patch-module", s"$moduleName=$sourceRoot") ) .getOrElse(Nil) - } else { + case _ => Nil - } - } - } else Nil + } + } } private def addExportsFlags: List[String] = { diff --git a/metals/src/main/scala/scala/meta/internal/metals/LSPFileSystemProvider.scala b/metals/src/main/scala/scala/meta/internal/metals/LSPFileSystemProvider.scala new file mode 100644 index 00000000000..9ca1128ac17 --- /dev/null +++ b/metals/src/main/scala/scala/meta/internal/metals/LSPFileSystemProvider.scala @@ -0,0 +1,105 @@ +package scala.meta.internal.metals + +import java.net.URI +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.util.stream.Collectors + +import scala.concurrent.ExecutionContext +import scala.concurrent.Future +import scala.util.control.NonFatal + +import scala.meta.internal.metals.MetalsEnrichments._ +import scala.meta.internal.metals.clients.language.MetalsLanguageClient +import scala.meta.internal.metals.filesystem.MetalsFileSystem +import scala.meta.internal.metals.filesystem.MetalsFileSystemProvider + +final case class FSReadDirectoryResponse( + name: String, + isFile: Boolean +) +final case class FSReadDirectoriesResponse( + name: String, + directories: Array[FSReadDirectoryResponse], + error: String +) + +final case class FSReadFileResponse( + name: String, + value: String, + error: String +) + +final case class FSStatResponse( + name: String, + isFile: Boolean, + error: String +) + +final class LSPFileSystemProvider( + languageClient: MetalsLanguageClient, + clientConfig: ClientConfiguration +)(implicit ec: ExecutionContext) { + + private def getPath(uriAsStr: String): Path = + Paths.get(URI.create(uriAsStr)) + + def createLibraryFileSystem(): Unit = + if (clientConfig.isLibraryFileSystemSupported()) { + val params = + ClientCommands.CreateLibraryFileSystem.toExecuteCommandParams( + MetalsFileSystemProvider.rootURI.toString() + ) + languageClient.metalsExecuteClientCommand(params) + } + + def readDirectory( + uriAsStr: String + ): Future[FSReadDirectoriesResponse] = Future { + try { + val path = getPath(uriAsStr) + val subPaths = Files + .list(path) + .map(subPath => + FSReadDirectoryResponse( + subPath.getFileName().toString(), + Files.isRegularFile(subPath) + ) + ) + .collect(Collectors.toList()) + .asScala + .toArray + FSReadDirectoriesResponse(uriAsStr, subPaths, null) + } catch { + case NonFatal(e) => FSReadDirectoriesResponse(uriAsStr, null, e.toString) + } + } + + def readFile(uriAsStr: String): Future[FSReadFileResponse] = Future { + try { + val path = getPath(uriAsStr) + val contents = if (path.isClassfile) { + MetalsFileSystem.metalsFS + .decodeCFRFromClassFile(path) + .getOrElse(s"Unable to decode class file ${path.toUri}") + } else + new String(Files.readAllBytes(path), StandardCharsets.UTF_8) + FSReadFileResponse(uriAsStr, contents, null) + } catch { + case NonFatal(e) => FSReadFileResponse(uriAsStr, null, e.toString) + } + } + + def getSystemStat(uriAsStr: String): Future[FSStatResponse] = Future { + try { + val path = getPath(uriAsStr) + val filename = path.getFileName.toString + val isFile = Files.isRegularFile(path) + FSStatResponse(filename, isFile, null) + } catch { + case NonFatal(e) => FSStatResponse(uriAsStr, false, e.toString()) + } + } +} diff --git a/metals/src/main/scala/scala/meta/internal/metals/MetalsEnrichments.scala b/metals/src/main/scala/scala/meta/internal/metals/MetalsEnrichments.scala index f5108952782..c4be1e10046 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/MetalsEnrichments.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/MetalsEnrichments.scala @@ -35,6 +35,7 @@ import scala.meta.Term import scala.meta.Tree import scala.meta.inputs.Input import scala.meta.internal.io.FileIO +import scala.meta.internal.metals.filesystem.MetalsFileSystem import scala.meta.internal.mtags.MtagsEnrichments import scala.meta.internal.parsing.EmptyResult import scala.meta.internal.semanticdb.Scala.Descriptor @@ -309,7 +310,9 @@ object MetalsEnrichments def isDependencySource(workspace: AbsolutePath): Boolean = { (isLocalFileSystem(workspace) && - isInReadonlyDirectory(workspace)) || isJarFileSystem + isInReadonlyDirectory( + workspace + )) || isJarFileSystem || isMetalsFileSystem } def isWorkspaceSource(workspace: AbsolutePath): Boolean = @@ -323,11 +326,16 @@ object MetalsEnrichments def isJarFileSystem: Boolean = path.toNIO.getFileSystem().provider().getScheme().equals("jar") + def isMetalsFileSystem: Boolean = + MetalsFileSystem.metalsFS.isMetalsFileSystem(path.toNIO) + def isInReadonlyDirectory(workspace: AbsolutePath): Boolean = path.toNIO.startsWith( workspace.resolve(Directories.readonly).toNIO ) + def isReadOnly: Boolean = !Files.isWritable(path.toNIO) + def toRelativeInside(prefix: AbsolutePath): Option[RelativePath] = { // windows throws an exception on toRelative when on different drives if (path.toNIO.getRoot() != prefix.toNIO.getRoot()) @@ -408,7 +416,12 @@ object MetalsEnrichments out.toFile.setWritable(true) } try { - Files.copy(path.toNIO, out, StandardCopyOption.REPLACE_EXISTING) + if (path.isClassfile) { + val decompiledText = + MetalsFileSystem.metalsFS.decodeCFRFromClassFile(path.toNIO) + decompiledText.foreach(text => Files.write(out, text.getBytes())) + } else + Files.copy(path.toNIO, out, StandardCopyOption.REPLACE_EXISTING) // Don't use readOnly files on Windows, makes it impossible to walk // the entire directory later on. if (!Properties.isWin) { @@ -426,16 +439,18 @@ object MetalsEnrichments throw new Exception(s"Unable to save $path in workspace") } else if (path.toNIO.getFileSystem == workspace.toNIO.getFileSystem) { path - } else { - path.jarPath match { - case Some(jar) => + } else + path.toNIO match { + case metalsPath: m.internal.metals.filesystem.MetalsPath => val jarDir = - workspace.resolve(Directories.dependencies).resolve(jar.filename) + workspace + .resolve(Directories.dependencies) + .resolve(metalsPath.filename) val out = jarDir.resolveZipPath(path.toNIO) val jarMetaFile = jarDir.resolve(".jar.meta") lazy val currentJarMeta = readJarMeta(jarMetaFile) - lazy val jarMeta = toJarMeta(jar) + lazy val jarMeta = toJarMeta(AbsolutePath(metalsPath)) val updateMeta = !jarDir.exists || !currentJarMeta.contains(jarMeta) if (!out.exists || updateMeta) { @@ -451,12 +466,11 @@ object MetalsEnrichments }(retry) } else out - case None => + case _ => val out = workspace.resolve(Directories.readonly).resolveZipPath(path.toNIO) copyFile(path, out) } - } } def toTextDocumentIdentifier: TextDocumentIdentifier = { diff --git a/metals/src/main/scala/scala/meta/internal/metals/MetalsLanguageServer.scala b/metals/src/main/scala/scala/meta/internal/metals/MetalsLanguageServer.scala index 094802e56e0..07e89348366 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/MetalsLanguageServer.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/MetalsLanguageServer.scala @@ -61,6 +61,7 @@ import scala.meta.internal.metals.debug.BuildTargetClasses import scala.meta.internal.metals.debug.DebugProvider import scala.meta.internal.metals.doctor.Doctor import scala.meta.internal.metals.doctor.DoctorVisibilityDidChangeParams +import scala.meta.internal.metals.filesystem.MetalsFileSystem import scala.meta.internal.metals.findfiles._ import scala.meta.internal.metals.formatting.OnTypeFormattingProvider import scala.meta.internal.metals.formatting.RangeFormattingProvider @@ -274,6 +275,7 @@ class MetalsLanguageServer( private var compilers: Compilers = _ private var scalafixProvider: ScalafixProvider = _ private var fileDecoderProvider: FileDecoderProvider = _ + private var lspFileSystemProvider: LSPFileSystemProvider = _ private var testProvider: TestSuitesProvider = _ private var workspaceReload: WorkspaceReload = _ private var buildToolSelector: BuildToolSelector = _ @@ -443,6 +445,11 @@ class MetalsLanguageServer( shellRunner = register( new ShellRunner(languageClient, () => userConfig, time, statusBar) ) + MetalsFileSystem.metalsFS.setShellRunner( + shellRunner, + buildTargets, + executionContext + ) bloopInstall = new BloopInstall( workspace, languageClient, @@ -737,6 +744,10 @@ class MetalsLanguageServer( clientConfig, classFinder ) + lspFileSystemProvider = new LSPFileSystemProvider( + languageClient, + clientConfig + ) popupChoiceReset = new PopupChoiceReset( workspace, tables, @@ -1045,7 +1056,8 @@ class MetalsLanguageServer( recentlyOpenedFiles.add(path) // Update md5 fingerprint from file contents on disk - fingerprints.add(path, FileIO.slurp(path, charset)) + if (!path.isReadOnly) + fingerprints.add(path, FileIO.slurp(path, charset)) // Update in-memory buffer contents from LSP client buffers.put(path, params.getTextDocument.getText) @@ -1060,7 +1072,14 @@ class MetalsLanguageServer( * that we don't try to generate it for project files */ val interactive = buildServerPromise.future.map { _ => - interactiveSemanticdbs.textDocument(path) + // reduce loading of readonly files by passing the contents directly in + if (path.isReadOnly) + interactiveSemanticdbs.textDocument( + path, + Some(params.getTextDocument.getText) + ) + else + interactiveSemanticdbs.textDocument(path) } // We need both parser and semanticdb for synthetic decorations val publishSynthetics = for { @@ -1737,6 +1756,12 @@ class MetalsLanguageServer( params.kind == "class" ) .asJavaObject + case ServerCommands.FileSystemStat(uriAsStr) => + lspFileSystemProvider.getSystemStat(uriAsStr).asJavaObject + case ServerCommands.FileSystemReadFile(uriAsStr) => + lspFileSystemProvider.readFile(uriAsStr).asJavaObject + case ServerCommands.FileSystemReadDirectory(uriAsStr) => + lspFileSystemProvider.readDirectory(uriAsStr).asJavaObject case ServerCommands.RunDoctor() => Future { doctor.onVisibilityDidChange(true) @@ -2345,6 +2370,7 @@ class MetalsLanguageServer( private val indexer = Indexer( () => workspaceReload, () => doctor, + () => lspFileSystemProvider, languageClient, () => bspSession, executionContext, @@ -2452,7 +2478,7 @@ class MetalsLanguageServer( definitionOnly: Boolean = false ): Future[DefinitionResult] = { val source = positionParams.getTextDocument.getUri.toAbsolutePath - if (source.isScalaFilename || source.isJavaFilename) { + if (source.isScalaFilename || source.isJavaFilename || source.isClassfile) { val semanticDBDoc = semanticdbs.textDocument(source).documentIncludingStale (for { @@ -2514,7 +2540,7 @@ class MetalsLanguageServer( token: CancelToken = EmptyCancelToken ): Future[DefinitionResult] = { val source = position.getTextDocument.getUri.toAbsolutePath - if (source.isScalaFilename || source.isJavaFilename) { + if (source.isScalaFilename || source.isJavaFilename || source.isClassfile) { val result = timerProvider.timedThunk( "definition", diff --git a/metals/src/main/scala/scala/meta/internal/metals/PackageProvider.scala b/metals/src/main/scala/scala/meta/internal/metals/PackageProvider.scala index 088b343db45..6e89aae6640 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/PackageProvider.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/PackageProvider.scala @@ -45,7 +45,8 @@ class PackageProvider(private val buildTargets: BuildTargets) { } if ( - path.isScalaOrJava && !path.isJarFileSystem && path.toFile.length() == 0 + path.isScalaOrJava && !path.isMetalsFileSystem && path.toFile + .length() == 0 ) { buildTargets .inverseSourceItem(path) diff --git a/metals/src/main/scala/scala/meta/internal/metals/ServerCommands.scala b/metals/src/main/scala/scala/meta/internal/metals/ServerCommands.scala index 5d3e8ca38c6..fba13be46eb 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/ServerCommands.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/ServerCommands.scala @@ -149,6 +149,42 @@ object ServerCommands { |""".stripMargin ) + val FileSystemStat = new ParametrizedCommand[String]( + "filesystem-stat", + "Read Directory", + """|Return information about the uri. E.g. uri is a directory + | + |Virtual File Systems allow client to display filesystem data + |in the format that Metals dictates. + |e.g. jar library dependencies + |""".stripMargin, + "[uri], uri of the file or directory." + ) + + val FileSystemReadDirectory = new ParametrizedCommand[String]( + "filesystem-read-directory", + "Read Directory", + """|Return the contents of a virtual filesystem directory. + | + |Virtual File Systems allow client to display filesystem data + |in the format that Metals dictates. + |e.g. jar library dependencies + |""".stripMargin, + "[uri], uri of the directory." + ) + + val FileSystemReadFile = new ParametrizedCommand[String]( + "filesystem-read-file", + "Read File", + """|Return the contents of a virtual filesystem file. + | + |Virtual File Systems allow client to display filesystem data + |in the format that Metals dictates. + |e.g. jar library dependencies + |""".stripMargin, + "[uri], uri of the file." + ) + val RunDoctor = new Command( "doctor-run", "Run doctor", diff --git a/metals/src/main/scala/scala/meta/internal/metals/StandaloneSymbolSearch.scala b/metals/src/main/scala/scala/meta/internal/metals/StandaloneSymbolSearch.scala index bd1ab55e49c..e1753a93d52 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/StandaloneSymbolSearch.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/StandaloneSymbolSearch.scala @@ -7,6 +7,7 @@ import java.{util => ju} import scala.collection.concurrent.TrieMap import scala.meta.internal.metals.MetalsEnrichments._ +import scala.meta.internal.metals.filesystem.MetalsFileSystem import scala.meta.internal.mtags.Mtags import scala.meta.internal.mtags.OnDemandSymbolIndex import scala.meta.internal.parsing.Trees @@ -185,16 +186,25 @@ object StandaloneSymbolSearch { (missingScala, JdkSources(javaHome)) match { case (true, Left(_)) => val (scalaSources, scalaClasspath) = getScala(scalaVersion) - (sources ++ scalaSources, classpath ++ scalaClasspath) + val mappedSources = scalaSources.map(source => + MetalsFileSystem.metalsFS.getOrElseUpdateSourceJar(source) + ) + (sources ++ mappedSources, classpath ++ scalaClasspath) case (true, Right(absPath)) => val (scalaSources, scalaClasspath) = getScala(scalaVersion) + val mappedSources = scalaSources.map(source => + MetalsFileSystem.metalsFS.getOrElseUpdateSourceJar(source) + ) + val jdkPath = MetalsFileSystem.metalsFS.getOrElseUpdateJDK(absPath) ( - (scalaSources :+ absPath) ++ sources, + (mappedSources :+ jdkPath) ++ sources, classpath ++ scalaClasspath ) - case (false, Left(_)) => (sources, classpath) + case (false, Left(_)) => + (sources, classpath) case (false, Right(absPath)) => - (sources :+ absPath, classpath) + val jdkPath = MetalsFileSystem.metalsFS.getOrElseUpdateJDK(absPath) + (sources :+ jdkPath, classpath) } } diff --git a/metals/src/main/scala/scala/meta/internal/metals/TargetData.scala b/metals/src/main/scala/scala/meta/internal/metals/TargetData.scala index ea9845034d4..be42772d3f5 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/TargetData.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/TargetData.scala @@ -76,6 +76,8 @@ final class TargetData { val allTargetRoots = scalaTargetRoots.toSet ++ javaTargetRoots.toSet allTargetRoots.iterator } + def allJDKs: Iterator[String] = scalaTargetInfo.flatMap(_._2.jvmHome).iterator + def all: Iterator[BuildTarget] = buildTargetInfo.values.toIterator diff --git a/metals/src/main/scala/scala/meta/internal/metals/debug/ClientConfigurationAdapter.scala b/metals/src/main/scala/scala/meta/internal/metals/debug/ClientConfigurationAdapter.scala index fc52b02f92d..135fdc79015 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/debug/ClientConfigurationAdapter.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/debug/ClientConfigurationAdapter.scala @@ -53,7 +53,8 @@ private[debug] final case class ClientConfigurationAdapter( def adaptPathForClient(path: AbsolutePath): String = { pathFormat match { case InitializeRequestArgumentsPathFormat.PATH => - if (path.isJarFileSystem) path.toURI.toString else path.toString + if (path.isJarFileSystem || path.isMetalsFileSystem) path.toURI.toString + else path.toString case InitializeRequestArgumentsPathFormat.URI => path.toURI.toString } } diff --git a/metals/src/main/scala/scala/meta/internal/metals/debug/SourcePathAdapter.scala b/metals/src/main/scala/scala/meta/internal/metals/debug/SourcePathAdapter.scala index 6c90dc95c66..e750922150e 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/debug/SourcePathAdapter.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/debug/SourcePathAdapter.scala @@ -10,6 +10,8 @@ import scala.meta.internal.io.FileIO import scala.meta.internal.metals.BuildTargets import scala.meta.internal.metals.Directories import scala.meta.internal.metals.MetalsEnrichments._ +import scala.meta.internal.metals.filesystem.MetalsFileSystem +import scala.meta.internal.metals.filesystem.MetalsPath import scala.meta.internal.mtags.URIEncoderDecoder import scala.meta.io.AbsolutePath @@ -34,20 +36,26 @@ private[debug] final class SourcePathAdapter( .stripPrefix("jar:") val decodedPath = URIEncoderDecoder.decode(sourceFileJarPath).toAbsolutePath - val parts = decodedPath.toNIO.iterator().asScala.map(_.toString).toVector - val jarPartIndex = parts.indexWhere(path => path.endsWith(".jar!")) - val jarPath = AbsolutePath( - parts - .take(jarPartIndex + 1) - .mkString(File.separator, File.separator, "") - .stripSuffix("!") - ) - val relative = parts.drop(jarPartIndex + 1).mkString(File.separator) + decodedPath.toNIO match { + case metalsPath: MetalsPath => metalsPath.originalJarURI + case _ => + val parts = + decodedPath.toNIO.iterator().asScala.map(_.toString).toVector + val jarPartIndex = parts.indexWhere(path => path.endsWith(".jar!")) + val jarPath = AbsolutePath( + parts + .take(jarPartIndex + 1) + .mkString(File.separator, File.separator, "") + .stripSuffix("!") + ) + val relative = parts.drop(jarPartIndex + 1).mkString(File.separator) - val sourceURI = FileIO.withJarFileSystem(jarPath, create = false)(root => - root.resolve(relative).toURI - ) - Some(sourceURI) + val sourceURI = + FileIO.withJarFileSystem(jarPath, create = false)(root => + root.resolve(relative).toURI + ) + Some(sourceURI) + } } else { // if sourcePath is a dependency source file // we retrieve the original source jar and we build the uri innside the source jar filesystem @@ -70,8 +78,11 @@ private[debug] final class SourcePathAdapter( Try(URI.create(sourcePath)).getOrElse(Paths.get(sourcePath).toUri()) sourceUri.getScheme match { case "jar" => - val path = sourceUri.toAbsolutePath - Some(if (saveJarFileToDisk) path.toFileOnDisk(workspace) else path) + MetalsFileSystem.metalsFS + .getMetalsJarPath(sourceUri) + .map(path => + if (saveJarFileToDisk) path.toFileOnDisk(workspace) else path + ) case "file" => Some(AbsolutePath(Paths.get(sourceUri))) case _ => None } diff --git a/metals/src/main/scala/scala/meta/internal/metals/filesystem/MetalsDirectoryStream.scala b/metals/src/main/scala/scala/meta/internal/metals/filesystem/MetalsDirectoryStream.scala new file mode 100644 index 00000000000..46a862c1728 --- /dev/null +++ b/metals/src/main/scala/scala/meta/internal/metals/filesystem/MetalsDirectoryStream.scala @@ -0,0 +1,18 @@ +package scala.meta.internal.metals.filesystem + +import java.nio.file.DirectoryStream +import java.nio.file.Path +import java.{util => ju} + +import scala.collection.JavaConverters._ + +final case class MetalsDirectoryStream( + paths: Iterable[Path], + filter: DirectoryStream.Filter[_ >: Path] +) extends DirectoryStream[Path] { + + override def close(): Unit = {} + + override def iterator(): ju.Iterator[Path] = + paths.filter(filter.accept).toIterator.asJava +} diff --git a/metals/src/main/scala/scala/meta/internal/metals/filesystem/MetalsFileAttributes.scala b/metals/src/main/scala/scala/meta/internal/metals/filesystem/MetalsFileAttributes.scala new file mode 100644 index 00000000000..821d10d5daf --- /dev/null +++ b/metals/src/main/scala/scala/meta/internal/metals/filesystem/MetalsFileAttributes.scala @@ -0,0 +1,31 @@ +package scala.meta.internal.metals.filesystem + +import java.nio.file.attribute.BasicFileAttributes +import java.nio.file.attribute.FileTime +import java.time.ZonedDateTime + +final case class MetalsFileAttributes(path: MetalsPath, isFile: Boolean) + extends BasicFileAttributes { + + override def lastModifiedTime(): FileTime = MetalsFileAttributes.zeroTime + + override def lastAccessTime(): FileTime = MetalsFileAttributes.zeroTime + + override def creationTime(): FileTime = MetalsFileAttributes.zeroTime + + override def isRegularFile(): Boolean = isFile + + override def isDirectory(): Boolean = !isFile + + override def isSymbolicLink(): Boolean = false + + override def isOther(): Boolean = false + + override def size(): Long = 0L + + override def fileKey(): Object = path +} + +object MetalsFileAttributes { + val zeroTime: FileTime = FileTime.from(ZonedDateTime.now().toInstant()) +} diff --git a/metals/src/main/scala/scala/meta/internal/metals/filesystem/MetalsFileSystem.scala b/metals/src/main/scala/scala/meta/internal/metals/filesystem/MetalsFileSystem.scala new file mode 100644 index 00000000000..8939a9a8733 --- /dev/null +++ b/metals/src/main/scala/scala/meta/internal/metals/filesystem/MetalsFileSystem.scala @@ -0,0 +1,367 @@ +package scala.meta.internal.metals.filesystem + +import java.net.URI +import java.net.URL +import java.nio.file.FileStore +import java.nio.file.FileSystem +import java.nio.file.FileSystemNotFoundException +import java.nio.file.FileSystems +import java.nio.file.Path +import java.nio.file.PathMatcher +import java.nio.file.Paths +import java.nio.file.ProviderNotFoundException +import java.nio.file.WatchService +import java.nio.file.attribute.UserPrincipalLookupService +import java.nio.file.spi.FileSystemProvider +import java.{lang => jl} +import java.{util => ju} + +import scala.annotation.tailrec +import scala.collection.concurrent.TrieMap +import scala.concurrent.Await +import scala.concurrent.ExecutionContext +import scala.concurrent.duration._ + +import scala.meta.internal.builds.ShellRunner +import scala.meta.internal.metals.BuildTargets +import scala.meta.internal.metals.FileDecoderProvider +import scala.meta.internal.metals.MetalsEnrichments._ +import scala.meta.io.AbsolutePath + +class MetalsFileSystem(fileSystemProvider: MetalsFileSystemProvider) + extends FileSystem { + + // TODO work out when to reset this - on a new build import - or just built up as build targets gets new jars? + private val workspaceJars = TrieMap.empty[String, JarInfo] + private val sourceJars = TrieMap.empty[String, JarInfo] + private val jdks = TrieMap.empty[String, JarInfo] + private val metalsPathToJarInfo = TrieMap.empty[String, JarInfo] + + private var shellRunner: ShellRunner = _ + private var buildTargets: BuildTargets = _ + private var executionContext: ExecutionContext = _ + + def setShellRunner( + shellRunner: ShellRunner, + buildTargets: BuildTargets, + executionContext: ExecutionContext + ): Unit = { + this.shellRunner = shellRunner + this.buildTargets = buildTargets + this.executionContext = executionContext + } + + val rootPath: MetalsPath = + MetalsPath(this, MetalsFileSystemProvider.rootNames) + + private val rootDirectories: jl.Iterable[Path] = + ju.Collections.singletonList(rootPath) + + override def provider(): FileSystemProvider = fileSystemProvider + + override def close(): Unit = {} + + override def isOpen(): Boolean = true + + override def isReadOnly(): Boolean = true + + override def getSeparator(): String = MetalsFileSystemProvider.separator + + override def getRootDirectories(): jl.Iterable[Path] = rootDirectories + + override def getFileStores(): jl.Iterable[FileStore] = null + + override def supportedFileAttributeViews(): ju.Set[String] = null + + override def getPath(first: String, more: String*): Path = { + def tidyPath(path: String): String = { + if (!path.startsWith("//")) + path + else + tidyPath(path.stripPrefix("/")) + } + val pathAsStr = List(List(first), more).flatten + .mkString(MetalsFileSystemProvider.separator) + // handle both absolute paths "/root/foo/bar" and relative "foo/bar" + // URLs will request a path of "///xxxx" so remove additional "/" + val tidiedPath = tidyPath(pathAsStr) + val items = if (tidiedPath.startsWith(MetalsFileSystemProvider.separator)) { + val split = tidiedPath.drop(1).split(MetalsFileSystemProvider.separator) + split(0) = s"${MetalsFileSystemProvider.separator}${split(0)}" + split + } else + tidiedPath.split(MetalsFileSystemProvider.separator) + val names = new Array[String](items.length) + items.zipWithIndex.foreach(f => names(f._2) = f._1) + MetalsPath(this, names) + } + + // TODO is default file system the right one to use? or re-implement + // https://docs.oracle.com/javase/8/docs/api/java/nio/file/FileSystem.html#getPathMatcher-java.lang.String- + override def getPathMatcher(syntaxAndPattern: String): PathMatcher = + FileSystems.getDefault().getPathMatcher(syntaxAndPattern) + + override def getUserPrincipalLookupService(): UserPrincipalLookupService = + throw new UnsupportedOperationException("getUserPrincipalLookupService") + + override def newWatchService(): WatchService = + throw new UnsupportedOperationException("newWatchService") + + private def createJarInfo( + filename: String, + subName: String, + path: AbsolutePath + ): JarInfo = { + val names = new Array[String](3) + names(0) = MetalsFileSystemProvider.rootName + names(1) = subName + names(2) = filename + val metalsPath = MetalsPath(this, names) + JarInfo( + names.mkString(getSeparator), + URI.create(s"jar:${path.toNIO.toUri.toString}"), + path.toNIO.toUri.toURL.getPath, + metalsPath, + new ju.HashMap[String, Any] + ) + } + + private def getOrElseUpdateJar( + jarMap: TrieMap[String, JarInfo], + filename: String, + subName: String, + jarPath: AbsolutePath + ): AbsolutePath = { + val jarInfo = jarMap + .getOrElseUpdate(filename, createJarInfo(filename, subName, jarPath)) + metalsPathToJarInfo.update(jarInfo.metalsJarPath, jarInfo) + jarInfo.metalsAbsolutePath + } + + def getOrElseUpdateWorkspaceJar(path: AbsolutePath): AbsolutePath = + getOrElseUpdateJar( + workspaceJars, + path.filename, + MetalsFileSystem.workspaceJarSubName, + path + ) + + def getOrElseUpdateSourceJar(path: AbsolutePath): AbsolutePath = + getOrElseUpdateJar( + sourceJars, + path.filename, + MetalsFileSystem.sourceJarSubName, + path + ) + + def getOrElseUpdateJDK(path: AbsolutePath): AbsolutePath = { + + def findDecentJDKName(path: AbsolutePath): AbsolutePath = + if (!path.hasParent || !List("lib", "src.zip").contains(path.filename)) + path + else + findDecentJDKName(path.parent) + + val filename = findDecentJDKName(path).filename + getOrElseUpdateJar(jdks, filename, MetalsFileSystem.jdkSubName, path) + } + + def getWorkspaceJars: Iterable[String] = workspaceJars.keySet + + def getSourceJars: Iterable[String] = sourceJars.keySet + + def getJdks: Iterable[String] = jdks.keySet + + def getWorkspaceJarPaths: Iterable[AbsolutePath] = + workspaceJars.values.map(_.metalsAbsolutePath) + + // metalsfs:/xxx/name.jar#runtime -> jar:/yyy/name.jar#runtime + // metalsfs:/xxx/name.jar/package/class -> jar:/yyy/name.jar!/package/class + def getOriginalJarURL(url: URL): Option[URL] = { + // convert to uri this way to remove fragments + val metalsURI = + new URI(url.getProtocol(), url.getHost(), url.getPath(), null) + Paths.get(metalsURI) match { + case metalsPath: MetalsPath => + getOriginalJarURI(metalsPath).map(uri => { + val uriWithFragments = + if (url.getRef != null && url.getRef.nonEmpty) { + val strippedPath = uri.getPath().stripSuffix("!/") + new URI(uri.getScheme(), uri.getHost(), strippedPath, url.getRef) + } else + uri + uriWithFragments.toURL() + }) + case _ => throw new ProviderNotFoundException(url.toString()) + } + } + + // metalsfs:/xxx/name.jar/package/class -> jar:/yyy/name.jar!/package/class + // metalsfs:/xxx/name.jar -> jar:/yyy/name.jar + def getOriginalJarURI(metalsPath: MetalsPath): Option[URI] = { + metalsPath.jarName.flatMap(jarName => { + workspaceJars + .get(jarName) + .orElse(sourceJars.get(jarName)) + .orElse(jdks.get(jarName)) + .map(jarInfo => { + val jarDirs = metalsPath.jarDirs + URI.create( + s"jar:file://${jarInfo.jarURLFile}!/${jarDirs.mkString(jarInfo.jarFS.getSeparator)}" + ) + }) + }) + } + + // jar:/yyy/name.jar!/package/class -> metalsfs:/xxx/name.jar/package/class + def getMetalsJarPath(uri: URI): Option[AbsolutePath] = { + @tailrec + def resolve(path: Path, dirs: Seq[String]): Path = + if (dirs.isEmpty) + path + else + resolve(path.resolve(dirs.head), dirs.tail) + val path = uri.toAbsolutePath + path.jarPath.flatMap(jarPath => { + val jarName = jarPath.filename + workspaceJars + .get(jarName) + .orElse(sourceJars.get(jarName)) + .orElse(jdks.get(jarName)) + .map(jarInfo => + jarInfo.metalsAbsolutePath.resolve(path.toString.stripPrefix("/")) + ) + }) + } + + def accessLibrary[A]( + path: MetalsPath, + convert: Path => A + ): Option[A] = { + + @tailrec + def resolve(path: Path, dirs: Seq[String]): Path = + if (dirs.isEmpty) + path + else + resolve(path.resolve(dirs.head), dirs.tail) + + if (!path.isAbsolute()) + accessLibrary(path.toAbsolutePath(), convert) + else + path.jarName.flatMap(jarName => { + val jarDirs = path.jarDirs + workspaceJars + .get(jarName) + .orElse(sourceJars.get(jarName)) + .orElse(jdks.get(jarName)) + .map(jarInfo => { + val inJarPath = resolve( + jarInfo.jarFS.getPath(jarInfo.jarFS.getSeparator), + jarDirs + ) + convert(inJarPath) + }) + }) + } + + // cache the last few decompiled class files as they're repeatedly accessed by metals once opened or referenced + private val cache = LRUCache[Path, Some[String]](10) + + def decodeCFRFromClassFile(path: Path): Option[String] = { + if (shellRunner == null) { + scribe.info("Wait for Metals to finish indexing") + None + } else { + val cacheResult = cache.get(path) + cacheResult.getOrElse({ + val result = + FileDecoderProvider + .decodeCFRFromClassFile( + shellRunner, + buildTargets, + AbsolutePath(path) + )(executionContext) + val response = Await.result(result, Duration("10min")) + val success = Option(response.value) + success.foreach(content => cache += path -> Some(content)) + success.orElse(Option(response.error)) + }) + } + } + + def isMetalsFileSystem(path: Path): Boolean = + path match { + case _: MetalsPath => true + case _ => false + } +} + +object MetalsFileSystem { + val jdkSubName: String = "jdk" + val workspaceJarSubName: String = "jar" + val sourceJarSubName: String = "source" + + private var _metalsFS: MetalsFileSystem = _ + + def metalsFS: MetalsFileSystem = { + if (_metalsFS == null) + synchronized { + if (_metalsFS == null) + _metalsFS = FileSystems.newFileSystem( + MetalsFileSystemProvider.rootURI, + ju.Collections.emptyMap[String, String] + ) match { + case mfs: MetalsFileSystem => mfs + case _ => + throw new ProviderNotFoundException( + MetalsFileSystemProvider.rootURI.toString() + ) + } + } + _metalsFS + } +} + +final case class JarInfo( + metalsAbsolutePath: AbsolutePath, + jarFS: FileSystem, + jarURLFile: String, + metalsJarPath: String +) + +object JarInfo { + def apply( + metalsJarPath: String, + jarURI: URI, + jarURLFile: String, + metalsPath: MetalsPath, + envMap: ju.Map[String, Any] + ): JarInfo = { + val jarFS = + try { + FileSystems.getFileSystem(jarURI) + } catch { + case _: FileSystemNotFoundException => + FileSystems.newFileSystem(jarURI, envMap) + } + JarInfo(AbsolutePath(metalsPath), jarFS, jarURLFile, metalsJarPath) + } +} + +import scala.collection.mutable +// TODO there must be one of these somewhere already in the Metals codebase +// TODO this is not thread safe +class LRUCache[K, V](maxEntries: Int) + extends java.util.LinkedHashMap[K, V](maxEntries, 1.0f, true) { + + override def removeEldestEntry(eldest: java.util.Map.Entry[K, V]): Boolean = + size > maxEntries +} + +object LRUCache { + def apply[K, V](maxEntries: Int): mutable.Map[K, V] = { + import scala.meta.internal.jdk.CollectionConverters._ + new LRUCache[K, V](maxEntries).asScala + } +} diff --git a/metals/src/main/scala/scala/meta/internal/metals/filesystem/MetalsFileSystemProvider.scala b/metals/src/main/scala/scala/meta/internal/metals/filesystem/MetalsFileSystemProvider.scala new file mode 100644 index 00000000000..49fe053da44 --- /dev/null +++ b/metals/src/main/scala/scala/meta/internal/metals/filesystem/MetalsFileSystemProvider.scala @@ -0,0 +1,263 @@ +package scala.meta.internal.metals.filesystem + +import java.io.IOException +import java.io.InputStream +import java.net.URI +import java.nio.channels.SeekableByteChannel +import java.nio.file.AccessMode +import java.nio.file.CopyOption +import java.nio.file.DirectoryStream +import java.nio.file.FileStore +import java.nio.file.FileSystemAlreadyExistsException +import java.nio.file.FileSystemNotFoundException +import java.nio.file.Files +import java.nio.file.LinkOption +import java.nio.file.OpenOption +import java.nio.file.Path +import java.nio.file.ProviderMismatchException +import java.nio.file.ReadOnlyFileSystemException +import java.nio.file.attribute.BasicFileAttributes +import java.nio.file.attribute.FileAttribute +import java.nio.file.attribute.FileAttributeView +import java.nio.file.spi.FileSystemProvider +import java.util.stream.Collectors +import java.{util => ju} + +import scala.collection.JavaConverters._ + +final case class MetalsFileSystemProvider() extends FileSystemProvider { + + private var fileSystem: MetalsFileSystem = _ + + override def getScheme: String = MetalsFileSystemProvider.scheme + + override def newFileSystem( + uri: URI, + env: ju.Map[String, _] + ): MetalsFileSystem = { + if (getScheme != uri.getScheme) + throw new ProviderMismatchException() + synchronized { + if (fileSystem != null) + throw new FileSystemAlreadyExistsException(uri.toString) + fileSystem = new MetalsFileSystem(this) + fileSystem + } + } + + override def getFileSystem(uri: URI) = { + if (getScheme != uri.getScheme) + throw new ProviderMismatchException() + synchronized { + if (fileSystem == null) + throw new FileSystemNotFoundException(uri.toString) + else + fileSystem + } + } + + override def getPath(uri: URI): Path = { + val pathStr = uri.getSchemeSpecificPart() + return getFileSystem(uri).getPath(pathStr) + } + + override def newDirectoryStream( + dir: Path, + filter: DirectoryStream.Filter[_ >: Path] + ): DirectoryStream[Path] = { + dir match { + case metalsPath: MetalsPath => + if (metalsPath == fileSystem.rootPath) { + MetalsDirectoryStream( + List( + MetalsFileSystem.workspaceJarSubName, + MetalsFileSystem.sourceJarSubName, + MetalsFileSystem.jdkSubName + ).map(name => metalsPath.resolve(name)), + filter + ) + } else if (!metalsPath.isInJar) { + MetalsDirectoryStream( + { + metalsPath.getFileName().toString match { + case MetalsFileSystem.workspaceJarSubName => + fileSystem.getWorkspaceJars + case MetalsFileSystem.sourceJarSubName => + fileSystem.getSourceJars + case MetalsFileSystem.jdkSubName => fileSystem.getJdks + } + }.map(name => metalsPath.resolve(name)), + filter + ) + } else + MetalsDirectoryStream( + fileSystem + .accessLibrary( + metalsPath, + path => + Files + .list(path) + .collect(Collectors.toList()) + .asScala + .map(subPath => + metalsPath.resolve(path.relativize(subPath)) + ) + .toList + ) + .getOrElse(List.empty), + filter + ) + case _ => throw new ProviderMismatchException() + } + } + + override def newInputStream(path: Path, options: OpenOption*): InputStream = + path match { + case metalsPath: MetalsPath => + fileSystem + .accessLibrary( + metalsPath, + path => Files.newInputStream(path, options: _*) + ) + .getOrElse(throw new IOException(s"Can't access $path")) + case _ => throw new ProviderMismatchException() + } + + override def newByteChannel( + path: Path, + options: ju.Set[_ <: OpenOption], + attrs: FileAttribute[_]* + ): SeekableByteChannel = { + path match { + case metalsPath: MetalsPath => + if (!metalsPath.isInJar) + throw new IOException(path.toUri.toString) + else + fileSystem + .accessLibrary( + metalsPath, + path => Files.newByteChannel(path, options, attrs: _*) + ) + .getOrElse(throw new IOException(s"$path")) + case _ => throw new ProviderMismatchException() + } + } + + override def createDirectory(dir: Path, attrs: FileAttribute[_]*): Unit = + throw new ReadOnlyFileSystemException() + + override def delete(path: Path): Unit = + throw new ReadOnlyFileSystemException() + + override def copy(source: Path, target: Path, options: CopyOption*): Unit = + throw new ReadOnlyFileSystemException() + + override def move(source: Path, target: Path, options: CopyOption*): Unit = + throw new ReadOnlyFileSystemException() + + override def isSameFile(path: Path, path2: Path): Boolean = + path.toAbsolutePath().equals(path2.toAbsolutePath()) + + override def isHidden(path: Path): Boolean = false + + override def getFileStore(path: Path): FileStore = null + + override def checkAccess(path: Path, modes: AccessMode*): Unit = { + if (modes.contains(AccessMode.WRITE)) + throw new IOException(s"Read only filesystem $path") + path match { + case metalsPath: MetalsPath => + if (metalsPath.isInJar) + fileSystem + .accessLibrary( + metalsPath, + path => path.getFileSystem().provider().checkAccess(path) + ) + case _ => throw new ProviderMismatchException() + } + } + + override def getFileAttributeView[V <: FileAttributeView]( + path: Path, + _type: Class[V], + options: LinkOption* + ): V = { + path match { + case metalsPath: MetalsPath => + if (!metalsPath.isInJar) + throw new IOException(path.toUri.toString) + else { + fileSystem + .accessLibrary( + metalsPath, + path => Files.getFileAttributeView(path, _type, options: _*) + ) + .getOrElse(null.asInstanceOf[V]) + } + case _ => throw new ProviderMismatchException() + } + } + + override def readAttributes[A <: BasicFileAttributes]( + path: Path, + _type: Class[A], + options: LinkOption* + ): A = { + path match { + case metalsPath: MetalsPath => + if (!metalsPath.isInJar) { + if (_type != classOf[BasicFileAttributes]) + throw new UnsupportedOperationException( + s"readAttributes ${path.toUri} ${_type}" + ) + new MetalsFileAttributes(metalsPath, false).asInstanceOf[A] + } else + fileSystem + .accessLibrary( + metalsPath, + path => Files.readAttributes(path, _type, options: _*) + ) + .getOrElse(throw new IOException(path.toUri.toString)) + case _ => throw new ProviderMismatchException() + } + } + + override def readAttributes( + path: Path, + attributes: String, + options: LinkOption* + ): ju.Map[String, Object] = { + path match { + case metalsPath: MetalsPath => + if (!metalsPath.isInJar) + throw new IOException(path.toUri.toString) + else + fileSystem + .accessLibrary( + metalsPath, + path => Files.readAttributes(path, attributes, options: _*) + ) + .getOrElse(throw new IOException(path.toUri.toString)) + case _ => throw new ProviderMismatchException() + } + } + + override def setAttribute( + path: Path, + attribute: String, + value: Object, + options: LinkOption* + ): Unit = + throw new ReadOnlyFileSystemException() +} + +object MetalsFileSystemProvider { + val scheme: String = "metalsfs" + val rootName: String = "/metalsLibraries" + val separator: String = "/" + val rootURI: URI = URI.create(s"${scheme}:$rootName") + val rootNames: Array[String] = Array(rootName) + val buildTargetsKey: String = "BuildTargets" + val shellRunnerKey: String = "ShellRunner" + val executionContextKey: String = "ExecutionContext" +} diff --git a/metals/src/main/scala/scala/meta/internal/metals/filesystem/MetalsPath.scala b/metals/src/main/scala/scala/meta/internal/metals/filesystem/MetalsPath.scala new file mode 100644 index 00000000000..1abe081eec6 --- /dev/null +++ b/metals/src/main/scala/scala/meta/internal/metals/filesystem/MetalsPath.scala @@ -0,0 +1,377 @@ +package scala.meta.internal.metals.filesystem + +import java.io.File +import java.net.URI +import java.nio.file.LinkOption +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.ProviderMismatchException +import java.nio.file.WatchEvent +import java.nio.file.WatchKey +import java.nio.file.WatchService +import java.{util => ju} + +import scala.collection.JavaConverters._ +import scala.annotation.tailrec +import scala.meta.io.RelativePath + +final case class MetalsPath( + metalsFileSystem: MetalsFileSystem, + names: Array[String] +) extends Path { + + override def getFileSystem: MetalsFileSystem = metalsFileSystem + + override def isAbsolute: Boolean = + names.nonEmpty && names(0) == MetalsFileSystemProvider.rootName + + override def getRoot: MetalsPath = + if (isAbsolute()) + metalsFileSystem.rootPath + else + null + + override def getFileName: MetalsPath = + if (names.isEmpty) + null + else if (names.length == 1) + this + else MetalsPath(metalsFileSystem, Array(names.last)) + + override def getParent: MetalsPath = + if (names.length < 2) + null + else if (isAbsolute() && names.length == 2) + getRoot + else + MetalsPath(metalsFileSystem, names.take(names.length - 1)) + + override def getNameCount: Int = names.size + + override def getName(index: Int): MetalsPath = + if (isAbsolute && names.length == 1 && index == 0) + getRoot + else if (index >= names.length) + throw new IllegalArgumentException() + else + MetalsPath(metalsFileSystem, Array(names(index))) + + override def subpath(beginIndex: Int, endIndex: Int): MetalsPath = + if (isAbsolute && names.length == 1 && beginIndex == 0 && endIndex == 1) + getRoot + else if (beginIndex >= endIndex) + throw new IllegalArgumentException() + else if (beginIndex < 0) + throw new IllegalArgumentException() + else if (endIndex > names.length) + throw new IllegalArgumentException() + else if (beginIndex == 0 && endIndex == names.length) + this + else + MetalsPath(metalsFileSystem, names.slice(beginIndex, endIndex)) + + private def checkPath[F](path: Path, apply: MetalsPath => F): F = + if (path == null) + throw new NullPointerException() + else + path match { + case metalsPath: MetalsPath => apply(metalsPath) + case _ => throw new ProviderMismatchException() + } + + override def startsWith(other: Path): Boolean = + checkPath( + other, + metalsPath => { + if (this == other) + true + else if ( + isAbsolute() != metalsPath.isAbsolute() || + names.length < metalsPath.names.length + ) + false + else { + var idx = metalsPath.names.length - 1 + var different = false + while (!different && idx >= 0) { + if (names(idx) != metalsPath.names(idx)) + different = true + idx = idx - 1 + } + !different + } + } + ) + + override def startsWith(other: String): Boolean = + startsWith(getFileSystem().getPath(other)) + + override def endsWith(other: Path): Boolean = + checkPath( + other, + metalsPath => { + if (this == other) + true + else if ( + metalsPath.isAbsolute() || + names.length < metalsPath.names.length + ) + false + else { + var idx = metalsPath.names.length - 1 + var different = false + while (!different && idx >= 0) { + if ( + names(idx + names.length - metalsPath.names.length) != metalsPath + .names(idx) + ) + different = true + idx = idx - 1 + } + !different + } + } + ) + + override def endsWith(other: String): Boolean = + return endsWith(getFileSystem().getPath(other)) + + override def normalize(): MetalsPath = { + val count = names.count(f => f == "." || f == "..") + if (count == 0) + this + else { + val newNames = new Array[String](names.length - count) + var oldIdx = names.length - 1 + var newIdx = newNames.length - 1 + var doubleDotCount = 0 + while (oldIdx >= 0) { + if (names(oldIdx) == ".") + oldIdx = oldIdx - 1 + else if (names(oldIdx) == "..") { + doubleDotCount = doubleDotCount + 1 + oldIdx = oldIdx - 1 + } else if (doubleDotCount > 0) { + doubleDotCount = doubleDotCount - 1 + oldIdx = oldIdx - 1 + } else { + newNames(newIdx) = names(oldIdx) + oldIdx = oldIdx - 1 + newIdx = newIdx - 1 + } + } + MetalsPath(metalsFileSystem, newNames) + } + } + + override def resolve(other: Path): MetalsPath = + if (other.isAbsolute()) { + other match { + case mp: MetalsPath => mp + case _ => throw new ProviderMismatchException() + } + } else { + val newNames = new Array[String](names.length + other.getNameCount) + var idx = 0 + while (idx < names.length) { + newNames(idx) = names(idx) + idx = idx + 1 + } + idx = 0 + while (idx < other.getNameCount) { + newNames(idx + names.length) = other.getName(idx).toString + idx = idx + 1 + } + MetalsPath(metalsFileSystem, newNames) + } + + override def resolve(other: String): MetalsPath = + resolve(getFileSystem().getPath(other)) + + override def resolveSibling(other: Path): MetalsPath = + checkPath( + other, + metalsPath => { + val parent = getParent(); + if (parent == null) metalsPath else parent.resolve(other) + } + ) + + override def resolveSibling(other: String): MetalsPath = + resolveSibling(getFileSystem().getPath(other)) + + override def relativize(other: Path): MetalsPath = { + if (isAbsolute() != other.isAbsolute()) + throw new IllegalArgumentException( + s"Cannot relativize ${this.toUri()} with ${other.toUri()}" + ) + if (getNameCount() > other.getNameCount()) + throw new IllegalArgumentException( + s"Cannot relativize ${this.toUri()} with ${other.toUri()}" + ) + // TODO both being !absolute is valid but what's it supposed to return? + if (!isAbsolute()) + throw new UnsupportedOperationException("relativize") + checkPath( + other, + metalsPath => { + var i = 0 + var matching = true + while (matching && i < names.length) { + if (names(i) != metalsPath.names(i)) + matching = false + i = i + 1 + } + if (!matching) + throw new IllegalArgumentException( + s"Cannot relativize ${this.toUri()} with ${other.toUri()}" + ) + val newNames = new Array[String](metalsPath.names.length - names.length) + var idx = names.length + while (idx < metalsPath.names.length) { + newNames(idx - names.length) = metalsPath.names(idx) + idx = idx + 1 + } + MetalsPath(metalsFileSystem, newNames) + } + ) + } + + override def toUri: URI = { + try { + if (isAbsolute()) { + if (names.lengthCompare(1) == 0) + MetalsFileSystemProvider.rootURI + else { + val uriAsStr = + s"${MetalsFileSystemProvider.scheme}:${names.mkString(MetalsFileSystemProvider.separator)}" + URI.create(uriAsStr) + } + } else + toAbsolutePath.toUri + } catch { + case e: Exception => + scribe.error("toURI error ", e) + throw e + } + } + + override def toAbsolutePath: MetalsPath = + if (isAbsolute()) + this + else + metalsFileSystem.rootPath.resolve(this) + + override def toRealPath(options: LinkOption*): MetalsPath = + toAbsolutePath.normalize + + override def toFile: File = { + throw new UnsupportedOperationException("toFile") + } + + override def register( + watcher: WatchService, + events: Array[WatchEvent.Kind[_]], + modifiers: WatchEvent.Modifier* + ) = throw new UnsupportedOperationException("register") + + override def register( + watcher: WatchService, + events: WatchEvent.Kind[_]* + ): WatchKey = throw new UnsupportedOperationException("register") + + override def iterator(): ju.Iterator[Path] = { + val paths = names + .map(name => MetalsPath(metalsFileSystem, Array(name)): Path) + .toIterator + if (isAbsolute()) + paths.drop(1).asJava + else + paths.asJava + } + + override def compareTo(other: Path): Int = + checkPath( + other, + metalsPath => { + var result = 0 + var idx = 0 + while ( + result == 0 && idx < names.length && idx < metalsPath.names.length + ) { + val nameCompare = names(idx).compareTo(metalsPath.names(idx)) + if (nameCompare != 0) + result = nameCompare + idx = idx + 1 + } + if (result == 0) + names.length.compareTo(metalsPath.names.length) + else + result + } + ) + + override def hashCode: Int = + names.toSeq.hashCode + + override def equals(other: Any): Boolean = + other match { + case otherPath: MetalsPath => + metalsFileSystem == otherPath.metalsFileSystem && + names.sameElements(otherPath.names) + case _ => false + } + + override def toString(): String = + names.mkString(MetalsFileSystemProvider.separator) + + def isJDK: Boolean = + names.lengthCompare(2) >= 0 && names(2) == MetalsFileSystem.jdkSubName + + def isInJar: Boolean = names.lengthCompare(3) >= 0 + + // metalsfs:/xxx/name.jar/package/class -> metalsfs:/xxx/name.jar + def jarPath: Option[MetalsPath] = { + @tailrec + def jarPath(path: MetalsPath): MetalsPath = + if (path.names.lengthCompare(3) == 0) + path + else jarPath(path.getParent) + if (names.lengthCompare(2) < 0) None else Some(jarPath(this)) + } + + // metalsfs:/xxx/name.jar/package/class -> jar:/yyy/name.jar!/package/class + def originalJarURI: Option[URI] = + metalsFileSystem.getOriginalJarURI(this) + + def jarName: Option[String] = + if (names.lengthCompare(3) < 0) None else Some(names(2)) + + def jarDirs: Seq[String] = + if (names.lengthCompare(3) < 0) Seq.empty else names.toSeq.drop(3) + + def moduleName: Option[String] = + if (names.lengthCompare(3) < 0) None + else { + // jdk uses module/module-info.java + val moduleLocationNames = new Array[String](5) + var i = 0 + while (i < 4) { + moduleLocationNames(i) = names(i) + i = i + 1 + } + moduleLocationNames(4) = "module-info.java" + val moduleInfoLocation = MetalsPath(metalsFileSystem, moduleLocationNames) + val tmp = if (Files.exists(moduleInfoLocation)) { + Some(moduleLocationNames(3)) + } else None + // TODO - how to find out module name from non-jdk jar??? We've only tested with src.zip + scribe.error(s"Module info for ${moduleInfoLocation.toUri()} is $tmp") + tmp + } + +} +object MetalsPath { + def fromReadOnly(relativePath: RelativePath): MetalsPath = + MetalsFileSystem.metalsFS.rootPath.resolve(relativePath.toNIO) +} diff --git a/metals/src/main/scala/scala/meta/internal/metals/findfiles/FindTextInDependencyJars.scala b/metals/src/main/scala/scala/meta/internal/metals/findfiles/FindTextInDependencyJars.scala index f441fa35448..005873a4d29 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/findfiles/FindTextInDependencyJars.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/findfiles/FindTextInDependencyJars.scala @@ -3,18 +3,19 @@ package scala.meta.internal.metals.findfiles import java.io.BufferedReader import java.io.InputStreamReader import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.PathMatcher +import java.util.stream.Collectors import scala.collection.mutable import scala.concurrent.ExecutionContext import scala.concurrent.Future -import scala.util.control.NonFatal -import scala.meta.internal.io.FileIO import scala.meta.internal.metals.BuildTargets import scala.meta.internal.metals.MetalsEnrichments._ -import scala.meta.internal.metals.PathMatcher.Nio import scala.meta.internal.metals.clients.language.MetalsInputBoxParams import scala.meta.internal.metals.clients.language.MetalsLanguageClient +import scala.meta.internal.metals.filesystem.MetalsFileSystem import scala.meta.io.AbsolutePath import org.eclipse.lsp4j.Location @@ -47,39 +48,33 @@ class FindTextInDependencyJars( .zip(maybePattern) .map { case (include, pattern) => val allLocations = mutable.ArrayBuffer.empty[Location] - val includeMatcher = Nio(s"glob:**$include") - val excludeMatcher = - req.options.flatMap(_.exclude).map(e => Nio(s"glob:**$e")) - - val allJars = - buildTargets.allWorkspaceJars.map((_, false)) ++ - buildTargets.allSourceJars.map((_, true)) - - allJars.foreach { case (classpathEntry, isSourceJar) => - try { - val locations: List[Location] = - if ( - classpathEntry.isFile && (classpathEntry.isJar || classpathEntry.isZip) - ) { - visitJar( - path = classpathEntry, - include = includeMatcher, - exclude = excludeMatcher, - pattern = pattern, - isSource = isSourceJar + val mfs = MetalsFileSystem.metalsFS + val includeMatcher = mfs.getPathMatcher(s"glob:**$include") + val excludeMatcher = req.options + .flatMap(_.exclude) + .map(e => mfs.getPathMatcher(s"glob:**$e")) + + val locations = Files + .walk(mfs.rootPath) + .filter(isSuitableFile(_, includeMatcher, excludeMatcher)) + .collect(Collectors.toList()) + .asScala + .flatMap { path => + val fileRanges: List[Range] = visitFile(path, pattern) + if (fileRanges.isEmpty) + Nil + else { + val locationPath = + if (saveJarFileToDisk) + AbsolutePath(path).toFileOnDisk(workspace()) + else AbsolutePath(path) + fileRanges + .map(range => + new Location(locationPath.toNIO.toUri.toString, range) ) - } else Nil - - allLocations ++= locations - } catch { - case NonFatal(e) => - scribe.error( - s"Failed to find text in dependency files for $classpathEntry", - e - ) + } } - } - + allLocations ++= locations allLocations.toList } .toList @@ -88,43 +83,17 @@ class FindTextInDependencyJars( } private def isSuitableFile( - path: AbsolutePath, - include: Nio, - exclude: Option[Nio] + path: Path, + includeMatcher: PathMatcher, + excludeMatcher: Option[PathMatcher] ): Boolean = { - path.isFile && - include.matches(path) && - exclude.forall(matcher => !matcher.matches(path)) - } - - private def visitJar( - path: AbsolutePath, - include: Nio, - exclude: Option[Nio], - pattern: String, - isSource: Boolean - ): List[Location] = { - FileIO - .withJarFileSystem(path, create = false, close = !isSource) { root => - FileIO - .listAllFilesRecursively(root) - .filter(isSuitableFile(_, include, exclude)) - .flatMap { absPath => - val fileRanges: List[Range] = visitFileInsideJar(absPath, pattern) - if (fileRanges.nonEmpty) { - val result = - if (saveJarFileToDisk) absPath.toFileOnDisk(workspace()) - else absPath - fileRanges - .map(range => new Location(result.toURI.toString, range)) - } else Nil - } - } - .toList + Files.isRegularFile(path) && + includeMatcher.matches(path) && + excludeMatcher.forall(matcher => !matcher.matches(path)) } - private def visitFileInsideJar( - path: AbsolutePath, + private def visitFile( + path: Path, pattern: String ): List[Range] = { var reader: BufferedReader = null @@ -134,7 +103,7 @@ class FindTextInDependencyJars( try { reader = new BufferedReader( - new InputStreamReader(Files.newInputStream(path.toNIO)) + new InputStreamReader(Files.newInputStream(path)) ) var lineNumber: Int = 0 var line: String = reader.readLine() diff --git a/metals/src/main/scala/scala/meta/internal/metals/urlstreamhandler/MetalsURLConnection.scala b/metals/src/main/scala/scala/meta/internal/metals/urlstreamhandler/MetalsURLConnection.scala new file mode 100644 index 00000000000..18aa25c4f2e --- /dev/null +++ b/metals/src/main/scala/scala/meta/internal/metals/urlstreamhandler/MetalsURLConnection.scala @@ -0,0 +1,16 @@ +package scala.meta.internal.metals.urlstreamhandler + +import java.io.InputStream +import java.net.URL +import java.net.URLConnection + +final class MetalsURLConnection(jarURL: URL, metalsURL: URL) + extends URLConnection(metalsURL) { + + private val jarURLConnection = jarURL.openConnection() + + override def connect(): Unit = jarURLConnection.connect() + + override def getInputStream(): InputStream = + jarURLConnection.getInputStream() +} diff --git a/metals/src/main/scala/scala/meta/internal/metals/urlstreamhandler/MetalsURLStreamHandler.scala b/metals/src/main/scala/scala/meta/internal/metals/urlstreamhandler/MetalsURLStreamHandler.scala new file mode 100644 index 00000000000..ad8e54ba735 --- /dev/null +++ b/metals/src/main/scala/scala/meta/internal/metals/urlstreamhandler/MetalsURLStreamHandler.scala @@ -0,0 +1,19 @@ +package scala.meta.internal.metals.urlstreamhandler + +import java.net.URL +import java.net.URLConnection +import java.net.URLStreamHandler + +import scala.meta.internal.metals.filesystem.MetalsFileSystem +import scala.meta.internal.metals.filesystem.MetalsFileSystemProvider + +final object MetalsURLStreamHandler extends URLStreamHandler { + + override protected def openConnection(url: URL): URLConnection = + if (MetalsFileSystemProvider.scheme == url.getProtocol) + MetalsFileSystem.metalsFS + .getOriginalJarURL(url) + .map(new MetalsURLConnection(_, url)) + .orNull + else null +} diff --git a/metals/src/main/scala/scala/meta/internal/metals/urlstreamhandler/MetalsURLStreamHandlerFactory.scala b/metals/src/main/scala/scala/meta/internal/metals/urlstreamhandler/MetalsURLStreamHandlerFactory.scala new file mode 100644 index 00000000000..08eefc6fc1f --- /dev/null +++ b/metals/src/main/scala/scala/meta/internal/metals/urlstreamhandler/MetalsURLStreamHandlerFactory.scala @@ -0,0 +1,26 @@ +package scala.meta.internal.metals.urlstreamhandler + +import java.net.URL +import java.net.URLStreamHandler +import java.net.URLStreamHandlerFactory + +import scala.meta.internal.metals.filesystem.MetalsFileSystemProvider + +final object MetalsURLStreamHandlerFactory extends URLStreamHandlerFactory { + + private var registered: Boolean = false + + def register: Unit = { + if (!registered) + synchronized { + if (!registered) { + URL.setURLStreamHandlerFactory(MetalsURLStreamHandlerFactory); + registered = true + } + } + } + + override def createURLStreamHandler(protocol: String): URLStreamHandler = + if (MetalsFileSystemProvider.scheme == protocol) MetalsURLStreamHandler + else null +} diff --git a/metals/src/main/scala/scala/meta/internal/tvp/ClasspathSymbols.scala b/metals/src/main/scala/scala/meta/internal/tvp/ClasspathSymbols.scala index 8b1eaaa93f1..98206aff80a 100644 --- a/metals/src/main/scala/scala/meta/internal/tvp/ClasspathSymbols.scala +++ b/metals/src/main/scala/scala/meta/internal/tvp/ClasspathSymbols.scala @@ -117,7 +117,8 @@ class ClasspathSymbols(isStatisticsEnabled: Boolean = false) { buf += TreeViewSymbolInformation( i.symbol, i.kind, - i.properties + i.properties, + Some(path.toURI) ) } } @@ -129,13 +130,8 @@ class ClasspathSymbols(isStatisticsEnabled: Boolean = false) { case _ => } - - } - if (in.extension == "jar") { - FileIO.withJarFileSystem(in, create = false, close = true)(list) - } else { - list(in) } + list(in) buf.result() } @@ -173,7 +169,7 @@ class ClasspathSymbols(isStatisticsEnabled: Boolean = false) { d.Package(path.filename.stripSuffix("/")) ) } - result += TreeViewSymbolInformation(dummySymbol, k.CLASS, 0) + result += TreeViewSymbolInformation(dummySymbol, k.CLASS, 0, None) } FileVisitResult.CONTINUE } diff --git a/metals/src/main/scala/scala/meta/internal/tvp/MetalsTreeViewProvider.scala b/metals/src/main/scala/scala/meta/internal/tvp/MetalsTreeViewProvider.scala index ff65021130b..e114aeaefef 100644 --- a/metals/src/main/scala/scala/meta/internal/tvp/MetalsTreeViewProvider.scala +++ b/metals/src/main/scala/scala/meta/internal/tvp/MetalsTreeViewProvider.scala @@ -11,6 +11,7 @@ import scala.meta.dialects import scala.meta.internal.metals.MetalsEnrichments._ import scala.meta.internal.metals._ import scala.meta.internal.metals.clients.language.MetalsLanguageClient +import scala.meta.internal.metals.filesystem.MetalsFileSystem import scala.meta.internal.mtags.GlobalSymbolIndex import scala.meta.internal.mtags.Mtags import scala.meta.internal.mtags.Symbol @@ -51,7 +52,7 @@ class MetalsTreeViewProvider( _.toAbsolutePath, _.filename, _.toString, - () => buildTargets.allWorkspaceJars, + () => MetalsFileSystem.metalsFS.getWorkspaceJarPaths.iterator, (path, symbol) => classpath.symbols(path, symbol) ) diff --git a/metals/src/main/scala/scala/meta/internal/tvp/TreeViewSymbolInformation.scala b/metals/src/main/scala/scala/meta/internal/tvp/TreeViewSymbolInformation.scala index d2fb63512eb..fe648568857 100644 --- a/metals/src/main/scala/scala/meta/internal/tvp/TreeViewSymbolInformation.scala +++ b/metals/src/main/scala/scala/meta/internal/tvp/TreeViewSymbolInformation.scala @@ -1,5 +1,7 @@ package scala.meta.internal.tvp +import java.net.URI + import scala.meta.internal.mtags.MtagsEnrichments._ import scala.meta.internal.semanticdb.Scala._ import scala.meta.internal.semanticdb.SymbolInformation @@ -8,7 +10,8 @@ import scala.meta.internal.semanticdb.SymbolInformation.{Kind => k} case class TreeViewSymbolInformation( symbol: String, kind: SymbolInformation.Kind, - properties: Int = 0 + properties: Int = 0, + uri: Option[URI] ) { def isVal: Boolean = kind.isMethod && properties.isVal def isVar: Boolean = kind.isMethod && properties.isVar @@ -23,7 +26,8 @@ case class TreeViewSymbolInformation( else if (s.isType) k.CLASS else if (s.isTypeParameter) k.TYPE_PARAMETER else k.UNKNOWN_KIND, - 0 + 0, + None ) info :: loop(s.owner) } diff --git a/mtags/src/main/scala/scala/meta/internal/mtags/OpenClassLoader.scala b/mtags/src/main/scala/scala/meta/internal/mtags/OpenClassLoader.scala index 2220f823dce..2b6038a4f4b 100644 --- a/mtags/src/main/scala/scala/meta/internal/mtags/OpenClassLoader.scala +++ b/mtags/src/main/scala/scala/meta/internal/mtags/OpenClassLoader.scala @@ -1,5 +1,6 @@ package scala.meta.internal.mtags +import java.net.URL import java.net.URLClassLoader import java.nio.file.Paths @@ -13,7 +14,10 @@ final class OpenClassLoader extends URLClassLoader(Array.empty) { private val isAdded = mutable.Set.empty[AbsolutePath] def addEntry(entry: AbsolutePath): Boolean = { if (!isAdded(entry)) { - super.addURL(entry.toNIO.toUri.toURL) + // url must end in "/" for the correct Loader to be selected for metalsfs:/somejar.jar + val url = entry.toNIO.toUri.toURL + val formattedURL = new URL(s"${url}/") + super.addURL(formattedURL) isAdded += entry true } else { diff --git a/mtags/src/main/scala/scala/meta/internal/mtags/SymbolIndexBucket.scala b/mtags/src/main/scala/scala/meta/internal/mtags/SymbolIndexBucket.scala index 0f685e25c59..0c65105ee3c 100644 --- a/mtags/src/main/scala/scala/meta/internal/mtags/SymbolIndexBucket.scala +++ b/mtags/src/main/scala/scala/meta/internal/mtags/SymbolIndexBucket.scala @@ -1,6 +1,5 @@ package scala.meta.internal.mtags -import java.io.UncheckedIOException import java.nio.CharBuffer import java.nio.charset.StandardCharsets @@ -58,18 +57,11 @@ class SymbolIndexBucket( def addSourceJar(jar: AbsolutePath): List[(String, AbsolutePath)] = { if (sourceJars.addEntry(jar)) { - FileIO.withJarFileSystem(jar, create = false) { root => - try { - root.listRecursive.toList.flatMap { - case source if source.isScala => - addSourceFile(source, None).map(sym => (sym, source)) - case _ => - List.empty - } - } catch { - // this happens in broken jars since file from FileWalker should exists - case _: UncheckedIOException => Nil - } + jar.listRecursive.toList.flatMap { + case source if source.isScala => + addSourceFile(source, Some(jar)).map(sym => (sym, source)) + case _ => + List.empty } } else List.empty