From cca5308915b28b69fc0abf26ae8be80b774bf121 Mon Sep 17 00:00:00 2001 From: rochala Date: Fri, 29 Mar 2024 15:58:11 +0100 Subject: [PATCH] Remove source code sanitizer, switch to tapir, use sttp as client, add throttling, deduplicate sent reports --- build.sbt | 9 +- .../internal/builds/NewProjectProvider.scala | 18 +- .../ImplementationProvider.scala | 4 +- .../meta/internal/metals/Compilers.scala | 21 +- .../metals/InlayHintResolveProvider.scala | 6 +- .../internal/metals/MetalsLspService.scala | 28 ++- .../internal/metals/ReferenceProvider.scala | 4 +- .../internal/metals/UserConfiguration.scala | 12 +- .../meta/internal/rename/RenameProvider.scala | 2 +- .../internal/telemetry/TelemetryClient.scala | 88 ++----- .../telemetry/TelemetryReportContext.scala | 72 ++---- .../internal/tvp/MetalsTreeViewProvider.scala | 1 - .../internal/metals/SourceCodeSanitizer.scala | 112 --------- .../metals/SourceCodeTransformer.scala | 222 ------------------ .../internal/mtags/MtagsEnrichments.scala | 2 +- .../internal/pc/PcInlayHintsProvider.scala | 2 +- .../pc/ScalaPresentationCompiler.scala | 1 - .../pc/completions/MatchCaseCompletions.scala | 2 +- .../ScalametaSourceCodeTransformer.scala | 82 ------- .../meta/internal/telemetry/Reports.scala | 2 +- .../telemetry/TelemetryEndpoints.scala | 14 ++ .../internal/telemetry/TelemetryService.scala | 24 -- .../scala/tests/telemetry/SampleReports.scala | 2 +- .../telemetry/SourceCodeSanitizerSuite.scala | 162 ------------- .../telemetry/TelemetryReporterSuite.scala | 209 ++--------------- 25 files changed, 155 insertions(+), 946 deletions(-) delete mode 100644 mtags-shared/src/main/scala/scala/meta/internal/metals/SourceCodeSanitizer.scala delete mode 100644 mtags-shared/src/main/scala/scala/meta/internal/metals/SourceCodeTransformer.scala delete mode 100644 mtags/src/main/scala/scala/meta/internal/metals/ScalametaSourceCodeTransformer.scala create mode 100644 telemetry-interfaces/src/main/scala/scala/meta/internal/telemetry/TelemetryEndpoints.scala delete mode 100644 telemetry-interfaces/src/main/scala/scala/meta/internal/telemetry/TelemetryService.scala delete mode 100644 tests/unit/src/test/scala/tests/telemetry/SourceCodeSanitizerSuite.scala diff --git a/build.sbt b/build.sbt index 4374d928423..293f49df7a4 100644 --- a/build.sbt +++ b/build.sbt @@ -266,6 +266,8 @@ lazy val telemetryInterfaces = project else Nil }, libraryDependencies := List( + "com.softwaremill.sttp.tapir" %% "tapir-core" % "1.10.0", + "com.softwaremill.sttp.tapir" %% "tapir-jsoniter-scala" % "1.10.0", "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-core" % "2.27.7", "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-macros" % "2.27.7" % "compile-internal", ), @@ -346,7 +348,6 @@ val mtagsSettings = List( Compile / doc / sources := Seq.empty, libraryDependencies ++= Seq( "com.lihaoyi" %% "geny" % V.genyVersion, - "com.lihaoyi" %% "requests" % V.requests, "com.thoughtworks.qdox" % "qdox" % V.qdox, // for java mtags "org.scala-lang.modules" %% "scala-java8-compat" % V.java8Compat, "org.jsoup" % "jsoup" % V.jsoup, // for extracting HTML from javadocs @@ -528,8 +529,9 @@ lazy val metals = project // for JSON formatted doctor "com.lihaoyi" %% "ujson" % "3.1.5", // For fetching projects' templates - // For remote language server - "com.lihaoyi" %% "requests" % V.requests, + // telemetry client + "com.softwaremill.sttp.client3" %% "core" % "3.9.5", + "com.softwaremill.sttp.tapir" %% "tapir-sttp-client" % "1.10.0", // for producing SemanticDB from Scala source files, to be sure we want the same version of scalameta "org.scalameta" %% "scalameta" % V.semanticdb(scalaVersion.value), "org.scalameta" % "semanticdb-scalac-core" % V.semanticdb( @@ -832,6 +834,7 @@ lazy val unit = project "io.get-coursier" %% "coursier" % V.coursier, // for jars "ch.epfl.scala" %% "bloop-config" % V.bloopConfig, "org.scalameta" %% "munit" % V.munit, + "com.softwaremill.sttp.tapir" %% "tapir-sttp-stub-server" % "1.10.0", ), buildInfoPackage := "tests", Compile / resourceGenerators += InputProperties diff --git a/metals/src/main/scala/scala/meta/internal/builds/NewProjectProvider.scala b/metals/src/main/scala/scala/meta/internal/builds/NewProjectProvider.scala index c4092f5dc4a..dab5326a2cc 100644 --- a/metals/src/main/scala/scala/meta/internal/builds/NewProjectProvider.scala +++ b/metals/src/main/scala/scala/meta/internal/builds/NewProjectProvider.scala @@ -23,6 +23,7 @@ import scala.meta.internal.process.ExitCodes import scala.meta.io.AbsolutePath import coursierapi._ +import sttp.client3._ class NewProjectProvider( client: MetalsLanguageClient, @@ -34,13 +35,15 @@ class NewProjectProvider( )(implicit context: ExecutionContext) { private val templatesUrl = - "https://github.com/foundweekends/giter8/wiki/giter8-templates.md" + uri"https://github.com/foundweekends/giter8/wiki/giter8-templates.md" private val giterDependency = Dependency .of("org.foundweekends.giter8", "giter8_2.12", BuildInfo.gitter8Version) // equal to cmd's: g8 playframework/play-scala-seed.g8 --name=../<> private val giterMain = "giter8.Giter8" + val backend = HttpClientSyncBackend() private var allTemplates = Seq.empty[MetalsQuickPickItem] + def allTemplatesFromWeb: Seq[MetalsQuickPickItem] = synchronized { if (allTemplates.nonEmpty) { @@ -53,14 +56,17 @@ class NewProjectProvider( // - [jimschubert/finatra.g8](https://github.com/jimschubert/finatra.g8) // (A simple Finatra 2.5 template with sbt-revolver and sbt-native-packager) val all = for { - result <- Try(requests.get(templatesUrl)).toOption.toIterable - _ = if (result.statusCode != 200) + result <- Try( + basicRequest.get(templatesUrl).send(backend) + ).toOption.toIterable + _ = if (!result.is200) client.showMessage( - NewScalaProject.templateDownloadFailed(result.statusMessage) + NewScalaProject.templateDownloadFailed(result.statusText) ) - if result.statusCode == 200 + if result.is200 + text <- result.body.toOption } yield { - NewProjectProvider.templatesFromText(result.text(), icons.github) + NewProjectProvider.templatesFromText(text, icons.github) } allTemplates = all.flatten.toSeq } diff --git a/metals/src/main/scala/scala/meta/internal/implementation/ImplementationProvider.scala b/metals/src/main/scala/scala/meta/internal/implementation/ImplementationProvider.scala index 6e910f5215a..5dfc847ccdc 100644 --- a/metals/src/main/scala/scala/meta/internal/implementation/ImplementationProvider.scala +++ b/metals/src/main/scala/scala/meta/internal/implementation/ImplementationProvider.scala @@ -15,7 +15,6 @@ import scala.meta.internal.metals.BuildTargets import scala.meta.internal.metals.Compilers import scala.meta.internal.metals.DefinitionProvider import scala.meta.internal.metals.MetalsEnrichments._ -import scala.meta.internal.metals.Report import scala.meta.internal.metals.ScalaVersionSelector import scala.meta.internal.metals.ScalaVersions import scala.meta.internal.metals.SemanticdbFeatureProvider @@ -46,6 +45,7 @@ import scala.meta.pc.ReportContext import ch.epfl.scala.bsp4j.BuildTargetIdentifier import org.eclipse.lsp4j.Location import org.eclipse.lsp4j.TextDocumentPositionParams +import scala.meta.internal.pc.StandardReport final class ImplementationProvider( semanticdbs: Semanticdbs, @@ -176,7 +176,7 @@ final class ImplementationProvider( if (sourceFiles.isEmpty) { rc.unsanitized.create( - Report( + StandardReport( "missing-definition", s"""|Missing definition symbol for: |$dealisedSymbol diff --git a/metals/src/main/scala/scala/meta/internal/metals/Compilers.scala b/metals/src/main/scala/scala/meta/internal/metals/Compilers.scala index d90786efb0c..3085668695c 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/Compilers.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/Compilers.scala @@ -1225,19 +1225,20 @@ class Compilers( ): ReportContext = { val logger = ju.logging.Logger.getLogger(classOf[TelemetryReportContext].getName) + + val loggerAccess = LoggerAccess( + debug = logger.fine(_), + info = logger.info(_), + warning = logger.warning(_), + error = logger.severe(_), + ) + new TelemetryReportContext( telemetryLevel = () => userConfig().telemetryLevel, reporterContext = createTelemetryReporterContext, - sanitizers = new TelemetryReportContext.Sanitizers( - workspace = Some(workspace.toNIO), - sourceCodeTransformer = Some(ScalametaSourceCodeTransformer), - ), - logger = LoggerAccess( - debug = logger.fine(_), - info = logger.info(_), - warning = logger.warning(_), - error = logger.severe(_), - ), + workspaceSanitizer = new WorkspaceSanitizer(Some(workspace.toNIO)), + telemetryClient = new telemetry.TelemetryClient(logger = loggerAccess), + logger = loggerAccess, ) } diff --git a/metals/src/main/scala/scala/meta/internal/metals/InlayHintResolveProvider.scala b/metals/src/main/scala/scala/meta/internal/metals/InlayHintResolveProvider.scala index 1803d1e1133..42052210e5d 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/InlayHintResolveProvider.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/InlayHintResolveProvider.scala @@ -12,6 +12,8 @@ import org.eclipse.lsp4j.InlayHint import org.eclipse.lsp4j.InlayHintLabelPart import org.eclipse.lsp4j.TextDocumentIdentifier import org.eclipse.{lsp4j => l} +import scala.meta.pc.ReportContext +import scala.meta.internal.pc.StandardReport final class InlayHintResolveProvider( definitionProvider: DefinitionProvider, @@ -30,7 +32,7 @@ final class InlayHintResolveProvider( resolve(inlayHint, labelParts, path, token) case Left(error) => scribe.warn(s"Failed to resolve inlay hint: $error") - rc.unsanitized.create(report(inlayHint, path, error), ifVerbose = true) + rc.unsanitized.create(report(inlayHint, path, error), true) Future.successful(inlayHint) } } @@ -109,7 +111,7 @@ final class InlayHintResolveProvider( error: Throwable, ) = { val pos = inlayHint.getPosition() - Report( + StandardReport( "inlayHint-resolve", s"""|pos: $pos | diff --git a/metals/src/main/scala/scala/meta/internal/metals/MetalsLspService.scala b/metals/src/main/scala/scala/meta/internal/metals/MetalsLspService.scala index dfdb1e661eb..650012eda7b 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/MetalsLspService.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/MetalsLspService.scala @@ -195,22 +195,24 @@ class MetalsLspService( }, ReportLevel.fromString(MetalsServerConfig.default.loglevel), ) + private val logger = logging.MetalsLogger.default + + private val loggerAccess = + LoggerAccess( + debug = logger.debug(_), + info = logger.info(_), + warning = logger.warn(_), + error = logger.error(_), + ) + + val client = new telemetry.TelemetryClient(logger = loggerAccess) + private val remoteTelemetryReports = new TelemetryReportContext( telemetryLevel = () => userConfig.telemetryLevel, reporterContext = createTelemetryReporterContext, - sanitizers = new TelemetryReportContext.Sanitizers( - workspace = Some(folder.toNIO), - sourceCodeTransformer = Some(ScalametaSourceCodeTransformer), - ), - logger = { - val logger = logging.MetalsLogger.default - LoggerAccess( - debug = logger.debug(_), - info = logger.info(_), - warning = logger.warn(_), - error = logger.error(_), - ) - }, + workspaceSanitizer = new WorkspaceSanitizer(Some(folder.toNIO)), + telemetryClient = client, + logger = loggerAccess, ) implicit val reports: ReportContext = diff --git a/metals/src/main/scala/scala/meta/internal/metals/ReferenceProvider.scala b/metals/src/main/scala/scala/meta/internal/metals/ReferenceProvider.scala index 50fe2a65667..68cd1c39c29 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/ReferenceProvider.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/ReferenceProvider.scala @@ -31,6 +31,8 @@ import com.google.common.hash.BloomFilter import com.google.common.hash.Funnels import org.eclipse.lsp4j.Location import org.eclipse.lsp4j.ReferenceParams +import scala.meta.internal.pc.StandardReport +import scala.meta.pc.ReportContext final class ReferenceProvider( workspace: AbsolutePath, @@ -174,7 +176,7 @@ final class ReferenceProvider( s"No references found, index size ${index.size}\n" + fileInIndex ) report.unsanitized.create( - Report( + StandardReport( "empty-references", index .map { case (path, entry) => diff --git a/metals/src/main/scala/scala/meta/internal/metals/UserConfiguration.scala b/metals/src/main/scala/scala/meta/internal/metals/UserConfiguration.scala index eb93baa4062..bcdd69f790f 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/UserConfiguration.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/UserConfiguration.scala @@ -57,6 +57,7 @@ case class UserConfiguration( verboseCompilation: Boolean = false, automaticImportBuild: AutoImportBuildKind = AutoImportBuildKind.Off, scalaCliLauncher: Option[String] = None, + defaultBspToBuildTool: Boolean = false, telemetryLevel: TelemetryLevel = TelemetryLevel.default, ) { @@ -362,8 +363,8 @@ object UserConfiguration { "off", "all", "Import build when changes detected without prompting", - """|Automatically import builds rather than prompting the user to choose. "initial" will - |only automatically import a build when a project is first opened, "all" will automate + """|Automatically import builds rather than prompting the user to choose. "initial" will + |only automatically import a build when a project is first opened, "all" will automate |build imports after subsequent changes as well.""".stripMargin, ), UserConfigurationOption( @@ -380,8 +381,8 @@ object UserConfiguration { "off", "all", "Import build when changes detected without prompting", - """|Automatically import builds rather than prompting the user to choose. "initial" will - |only automatically import a build when a project is first opened, "all" will automate + """|Automatically import builds rather than prompting the user to choose. "initial" will + |only automatically import a build when a project is first opened, "all" will automate |build imports after subsequent changes as well.""".stripMargin, ), UserConfigurationOption( @@ -654,10 +655,9 @@ object UserConfiguration { scalafixRulesDependencies = scalafixRulesDependencies, customProjectRoot = customProjectRoot, verboseCompilation = verboseCompilation, - scalaCliLauncher = None, telemetryLevel = telemetryLevel, - autoImportBuilds = autoImportBuilds, scalaCliLauncher = scalaCliLauncher, + automaticImportBuild = autoImportBuilds, defaultBspToBuildTool = defaultBspToBuildTool, ) ) diff --git a/metals/src/main/scala/scala/meta/internal/rename/RenameProvider.scala b/metals/src/main/scala/scala/meta/internal/rename/RenameProvider.scala index 94db85f3934..2bb0597e591 100644 --- a/metals/src/main/scala/scala/meta/internal/rename/RenameProvider.scala +++ b/metals/src/main/scala/scala/meta/internal/rename/RenameProvider.scala @@ -16,7 +16,6 @@ import scala.meta.internal.metals.Compilers import scala.meta.internal.metals.DefinitionProvider import scala.meta.internal.metals.MetalsEnrichments._ import scala.meta.internal.metals.ReferenceProvider -import scala.meta.internal.metals.ReportContext import scala.meta.internal.metals.TextEdits import scala.meta.internal.metals.clients.language.MetalsLanguageClient import scala.meta.internal.parsing.Trees @@ -30,6 +29,7 @@ import scala.meta.internal.semanticdb.TextDocument import scala.meta.internal.{semanticdb => s} import scala.meta.io.AbsolutePath import scala.meta.pc.CancelToken +import scala.meta.pc.ReportContext import org.eclipse.lsp4j.Location import org.eclipse.lsp4j.MessageParams diff --git a/metals/src/main/scala/scala/meta/internal/telemetry/TelemetryClient.scala b/metals/src/main/scala/scala/meta/internal/telemetry/TelemetryClient.scala index c365c29c8bc..ea64349d006 100644 --- a/metals/src/main/scala/scala/meta/internal/telemetry/TelemetryClient.scala +++ b/metals/src/main/scala/scala/meta/internal/telemetry/TelemetryClient.scala @@ -1,84 +1,46 @@ package scala.meta.internal.telemetry -import scala.concurrent.ExecutionContext -import scala.concurrent.Future -import scala.util.Random -import scala.util.Success +import sttp.client3._ import scala.meta.internal.metals.LoggerAccess -import scala.meta.internal.metals.TelemetryLevel -import scala.meta.internal.telemetry -import requests.Response +import sttp.tapir.client.sttp.SttpClientInterpreter +import sttp.model.Uri +import com.google.common.util.concurrent.RateLimiter object TelemetryClient { - case class Config(serverHost: String) + case class Config(serverHost: Uri) object Config { // private final val DefaultTelemetryEndpoint = // "https://scala3.westeurope.cloudapp.azure.com/telemetry" - private final val DefaultTelemetryEndpoint = "http://localhost:8081" + private final val DefaultTelemetryEndpoint = uri"http://localhost:8081" val default: Config = Config(DefaultTelemetryEndpoint) } - private class TelemetryRequest[In]( - endpoint: telemetry.FireAndForgetEndpoint[In], - logger: LoggerAccess, - )(implicit config: Config, ec: ExecutionContext) { - private val endpointURL = s"${config.serverHost}${endpoint.uri}" - private val requester = requests.send(endpoint.method) - println(s"TelemetryClient: sending $endpointURL") + private val interpreter = SttpClientInterpreter() - def apply(data: In): Unit = { - val json = endpoint.encodeInput(data) - val response = execute(json) - acknowledgeResponse(response) - } - - private def execute( - data: String, - retries: Int = 3, - backoffMillis: Int = 100, - ): Future[Response] = Future { - requester( - url = endpointURL, - data = data, - keepAlive = false, - check = false, - ) - }.recoverWith { - case _: requests.TimeoutException | _: requests.UnknownHostException - if retries > 0 => - Thread.sleep(backoffMillis) - execute(data, retries - 1, backoffMillis + Random.nextInt(1000)) - } - - private def acknowledgeResponse(response: Future[Response]): Unit = - response.onComplete { - case Success(value) if value.is2xx => - case _ => - logger.debug( - s"${endpoint.method}:${endpoint.uri} should never result in error, got ${response}" - ) - } - } } -private[meta] class TelemetryClient( - telemetryLevel: () => TelemetryLevel, +class TelemetryClient( config: TelemetryClient.Config = TelemetryClient.Config.default, logger: LoggerAccess = LoggerAccess.system, -)(implicit ec: ExecutionContext) - extends telemetry.TelemetryService { +) { import TelemetryClient._ - import telemetry.TelemetryService._ - - implicit private def clientConfig: Config = config - - private val sendErrorReport0 = - new TelemetryRequest(sendErrorReportEndpoint, logger) - - def sendErrorReport(report: telemetry.ErrorReport): Unit = - if (telemetryLevel().enabled) sendErrorReport0(report) - + val rateLimiter = RateLimiter.create(1.0 / 5.0) + + val backend = HttpClientFutureBackend() + val sendReport: ErrorReport => Unit = report => { + if (rateLimiter.tryAcquire()) { + logger.debug("Sending telemetry report.") + interpreter + .toClient( + TelemetryEndpoints.sendReport, + baseUri = Some(config.serverHost), + backend = backend, + ) + .apply(report) + } else logger.debug("Report was omitted, because of quota") + () + } } diff --git a/metals/src/main/scala/scala/meta/internal/telemetry/TelemetryReportContext.scala b/metals/src/main/scala/scala/meta/internal/telemetry/TelemetryReportContext.scala index 4e8fa5ddf2c..348ea6f6d84 100644 --- a/metals/src/main/scala/scala/meta/internal/telemetry/TelemetryReportContext.scala +++ b/metals/src/main/scala/scala/meta/internal/telemetry/TelemetryReportContext.scala @@ -3,12 +3,7 @@ package scala.meta.internal.telemetry import java.nio.file.Path import java.{util => ju} -import scala.concurrent.ExecutionContext - import scala.meta.internal.metals.LoggerAccess -import scala.meta.internal.metals.ReportSanitizer -import scala.meta.internal.metals.SourceCodeSanitizer -import scala.meta.internal.metals.SourceCodeTransformer import scala.meta.internal.metals.TelemetryLevel import scala.meta.internal.metals.WorkspaceSanitizer import scala.meta.internal.mtags.CommonMtagsEnrichments.XtensionOptionalJava @@ -18,26 +13,8 @@ import scala.meta.pc.ReportContext import scala.meta.pc.Reporter import scala.meta.pc.TimestampedFile import scala.meta.internal.metals.EmptyReporter - -object TelemetryReportContext { - case class Sanitizers( - workspaceSanitizer: WorkspaceSanitizer, - sourceCodeSanitizer: Option[SourceCodeSanitizer[_, _]], - ) { - def canSanitizeSources = sourceCodeSanitizer.isDefined - def this( - workspace: Option[Path], - sourceCodeTransformer: Option[SourceCodeTransformer[_, _]], - ) = - this( - workspaceSanitizer = new WorkspaceSanitizer(workspace), - sourceCodeSanitizer = - sourceCodeTransformer.map(new SourceCodeSanitizer(_)), - ) - val all: Seq[ReportSanitizer] = - Seq(workspaceSanitizer) ++ sourceCodeSanitizer - } -} +import com.google.common.collect.EvictingQueue +import java.util.concurrent.atomic.AtomicReference /** * A remote reporter sending reports to telemetry server aggregating the results. Operates in a best-effort manner. Created reporter does never reutrn any values. @@ -48,35 +25,27 @@ object TelemetryReportContext { class TelemetryReportContext( telemetryLevel: () => TelemetryLevel, reporterContext: () => telemetry.ReporterContext, - sanitizers: TelemetryReportContext.Sanitizers, - telemetryClientConfig: TelemetryClient.Config = - TelemetryClient.Config.default, + workspaceSanitizer: WorkspaceSanitizer, + telemetryClient: TelemetryClient, logger: LoggerAccess = LoggerAccess.system, -)(implicit ec: ExecutionContext) - extends ReportContext { +) extends ReportContext { val telemetryLevel0 = telemetryLevel() // Don't send reports with fragile user data - sources etc override lazy val unsanitized: Reporter = - if (telemetryLevel0 == TelemetryLevel.Full) reporter("incognito") + if (telemetryLevel0 == TelemetryLevel.Full) reporter("unsanitized") else EmptyReporter override lazy val incognito: Reporter = if (telemetryLevel0.enabled) reporter("incognito") else EmptyReporter override lazy val bloop: Reporter = if (telemetryLevel0.enabled) reporter("bloop") else EmptyReporter - private val client = new TelemetryClient( - config = telemetryClientConfig, - telemetryLevel = telemetryLevel, - logger = logger, - )(ec) - private def reporter(name: String) = new TelemetryReporter( name = name, - client = client, + client = telemetryClient, reporterContext = reporterContext, - sanitizers = sanitizers, + sanitizers = workspaceSanitizer, logger = logger, ) } @@ -85,10 +54,17 @@ private class TelemetryReporter( override val name: String, client: TelemetryClient, reporterContext: () => telemetry.ReporterContext, - sanitizers: TelemetryReportContext.Sanitizers, + sanitizers: WorkspaceSanitizer, logger: LoggerAccess, ) extends Reporter { + val previousTraces: AtomicReference[EvictingQueue[ExceptionSummary]] = + new AtomicReference(EvictingQueue.create(10)) + + def alreadyReported(report: ErrorReport): Boolean = { + report.error.exists(previousTraces.get.contains) + } + override def getReports(): ju.List[TimestampedFile] = ju.Collections.emptyList() @@ -100,7 +76,7 @@ private class TelemetryReporter( override def deleteAll(): Unit = () override def sanitize(message: String): String = - sanitizers.all.foldRight(message)(_.apply(_)) + sanitizers(message) private def createSanitizedReport(report: Report) = { new telemetry.ErrorReport( @@ -108,7 +84,7 @@ private class TelemetryReporter( reporterName = name, reporterContext = reporterContext(), id = report.id.asScala, - text = Option.when(sanitizers.canSanitizeSources)(sanitize(report.text)), + text = report.text, error = report.error .map(telemetry.ExceptionSummary.from(_, sanitize(_))) .asScala, @@ -120,13 +96,15 @@ private class TelemetryReporter( ifVerbose: Boolean, ): ju.Optional[Path] = { val report = createSanitizedReport(unsanitizedReport) - if (report.text.isDefined || report.error.isDefined) - client.sendErrorReport(report) - else - logger.info( - "Skipped reporting remotely unmeaningful report, no context or error, reportId=" + + if (!alreadyReported(report)) { + report.error.foreach(a => previousTraces.get.add(a)) + client.sendReport(report) + } else { + logger.debug( + "Skipped reporting remotely duplicated report, reportId=" + unsanitizedReport.id.orElse("null") ) + } ju.Optional.empty() } } diff --git a/metals/src/main/scala/scala/meta/internal/tvp/MetalsTreeViewProvider.scala b/metals/src/main/scala/scala/meta/internal/tvp/MetalsTreeViewProvider.scala index 791b83b8d10..88f63892171 100644 --- a/metals/src/main/scala/scala/meta/internal/tvp/MetalsTreeViewProvider.scala +++ b/metals/src/main/scala/scala/meta/internal/tvp/MetalsTreeViewProvider.scala @@ -11,7 +11,6 @@ import scala.collection.concurrent.TrieMap import scala.meta.Dialect import scala.meta.dialects import scala.meta.internal.metals.MetalsEnrichments._ -import scala.meta.internal.metals.ReportContext import scala.meta.internal.metals._ import scala.meta.internal.metals.clients.language.MetalsLanguageClient import scala.meta.internal.mtags.GlobalSymbolIndex diff --git a/mtags-shared/src/main/scala/scala/meta/internal/metals/SourceCodeSanitizer.scala b/mtags-shared/src/main/scala/scala/meta/internal/metals/SourceCodeSanitizer.scala deleted file mode 100644 index d5524456a8d..00000000000 --- a/mtags-shared/src/main/scala/scala/meta/internal/metals/SourceCodeSanitizer.scala +++ /dev/null @@ -1,112 +0,0 @@ -package scala.meta.internal.metals - -import java.util.regex.Pattern - -import scala.meta.internal.metals.SourceCodeSanitizer.Language - -/** - * Sanitizer ensuring that no original source code can leak through the reports. - * First it would treat input as the markdown source snippet with 1 or more code snipets. - * If the snippet contains parsable code it would erase all the original names, replacing them with synthetic symbols of the same length. - * If the code is not parsable or the transformed code is would not be parsable after transformation it would be replaced with an failure reason tag. - * If no code snipets are found the input is treated as a raw source code. - */ -class SourceCodeSanitizer[ParserCtx, ParserAST]( - parser: SourceCodeTransformer[ParserCtx, ParserAST] -) extends ReportSanitizer { - - override def sanitize(text: String): String = { - anonimizeMarkdownSnippets(text) - .getOrElse(tryAnonimize(text, languageHint = Some(Language.Scala)).merge) - } - - private final val OffsetMarker = "@@" - - private final val MarkdownCodeSnippet = java.util.regex.Pattern - .compile( - raw"```(\w+)?\s*\R([\s\S]*?)```", - Pattern.MULTILINE | Pattern.CASE_INSENSITIVE - ) - private final val StackTraceLine = - raw"(?:\s*(?:at\s*))?(\S+)\((?:(?:\S+\.(?:scala|java)\:\d+)|(?:Native Method))\)".r - - private type FailureReason = String - private def tryAnonimize( - source: String, - languageHint: Option[Language] - ): Either[FailureReason, String] = { - Option(source) - .map(_.trim()) - .filter(_.nonEmpty) - .map(_.replaceAll(OffsetMarker, "")) - .fold[Either[String, String]](Left("")) { source => - if (StackTraceLine.findFirstIn(source).isDefined) - Right(source) - else if (languageHint.forall(_ == Language.Scala)) { - val maybeParseResult = parser.parse(source) - if (maybeParseResult.isEmpty) Left("") - else { - val (ctx, tree) = maybeParseResult.get - val maybeSanitizedTree = parser.transformer.sanitizeSymbols(tree) - if (maybeSanitizedTree.isEmpty) Left("") - else { - val sanitizedTree = maybeSanitizedTree.get - val sourceString = parser.toSourceString(sanitizedTree, ctx) - val isReparsable = parser.parse(sourceString, ctx).isDefined - if (isReparsable) Right(sourceString) - else Left("") - } - } - } else Left("") - } - } - - private def anonimizeMarkdownSnippets(source: String): Option[String] = { - // Check if we have even number of markdown snipets markers, if not discard whole input - val snipetMarkers = source.linesIterator.count(_.startsWith("```")) - if (snipetMarkers == 0 || snipetMarkers % 2 != 0) None - else { - val matcher = MarkdownCodeSnippet.matcher(source) - val sourceResult = new java.lang.StringBuffer(source.size) - while (matcher.find()) { - val matchResult = matcher.toMatchResult() - val language = - Option(matchResult.group(1)).map(_.trim()).flatMap(Language.unapply) - val result = tryAnonimize( - languageHint = language, - source = matchResult.group(2) - ) - val sanitizedOrFailureReason: String = result.merge.replace("$", "\\$") - val updatedSnippet = - s"""```${language.map(_.stringValue).getOrElse("")} - |$sanitizedOrFailureReason - |``` - |""".stripMargin - - matcher.appendReplacement( - sourceResult, - updatedSnippet - ) - } - if (sourceResult.length() == 0) None // not found any snipets - else - Some { - matcher.appendTail(sourceResult) - sourceResult.toString() - } - } - } -} - -object SourceCodeSanitizer { - sealed abstract class Language(val stringValue: String) - object Language { - def unapply(v: String): Option[Language] = v.toLowerCase() match { - case "scala" => Some(Scala) - case "java" => Some(Java) - case _ => None - } - case object Scala extends Language("scala") - case object Java extends Language("java") - } -} diff --git a/mtags-shared/src/main/scala/scala/meta/internal/metals/SourceCodeTransformer.scala b/mtags-shared/src/main/scala/scala/meta/internal/metals/SourceCodeTransformer.scala deleted file mode 100644 index 55454679135..00000000000 --- a/mtags-shared/src/main/scala/scala/meta/internal/metals/SourceCodeTransformer.scala +++ /dev/null @@ -1,222 +0,0 @@ -package scala.meta.internal.metals - -import scala.annotation.tailrec -import scala.collection.mutable -import scala.util.Random - -// Needs to be implemented in user of the reporting, eg. MetalsLSPService using scalameta or compiler -trait SourceCodeTransformer[Context, Tree] { - - /** Try parse using any available dialects/contexts and return the context that can be used for validation of sanitized source */ - def parse(source: String): Option[(Context, Tree)] - - /** Parse once using dedicated context, used for validation */ - def parse(source: String, context: Context): Option[Tree] - - def toSourceString(value: Tree, ctx: Context): String - def transformer: ASTTransformer - - trait ASTTransformer { - protected type Name - protected type TermName <: Name - protected type TypeName <: Name - protected type UnclasifiedName <: Name - - protected def toTermName(name: String): TermName - protected def toTypeName(name: String): TypeName - protected def toUnclasifiedName(name: String): UnclasifiedName - protected def toSymbol(name: Name): String - - private final val SymbolToPrefixLength = 3 - private final val ShortSymbolLength = 2 - - def sanitizeSymbols(tree: Tree): Option[Tree] - - protected def isCommonScalaName(name: Name): Boolean = { - val symbol = toSymbol(name) - SourceCodeTransformer.CommonNames.types.contains(symbol) || - SourceCodeTransformer.CommonNames.methods.contains(symbol) - } - - // Computes all interim packages with exclusion of the top level one - // E.g. foo.bar.baz produces Seq("foo.bar", "foo.bar.baz") - private def interimPackages(packageName: String): Seq[String] = { - val segments = packageName.split('.') - require(segments.nonEmpty, s"Invalid package name $packageName") - val minPackageSegments = 2 - segments.toList - .drop(minPackageSegments) - .scanLeft(segments.take(minPackageSegments).mkString("."))(_ + "." + _) - } - // All Scala 2.13/3 stdlib packages - private val commonScalaPackages = Seq( - "scala.annotation", "scala.beans", "scala.collection.concurrent", - "scala.collection.convert", "scala.collection.generic", - "scala.collection.immutable", "scala.collection.mutable", - "scala.collection.compat", "scala.compiletime.ops", - "scala.compiletime.testing", "scala.concurrent.duration", - "scala.deriving", "scala.io", "scala.jdk.javaapi", "scala.math", - "scala.quoted.runtime", "scala.ref", "scala.reflect", "scala.runtime", - "scala.sys.process", "scala.util.control", "scala.util.hashing", - "scala.util.matching" - ).flatMap(interimPackages).distinct - - // All java.base packages - private val commonJavaPackages = Seq( - "java.io", "java.lang", "java.lang.annotation", "java.lang.constant", - "java.lang.foreign", "java.lang.invoke", "java.lang.module", - "java.lang.ref", "java.lang.reflect", "java.lang.runtime", "java.math", - "java.net", "java.net.spi", "java.nio", "java.nio.channels", - "java.nio.channels.spi", "java.nio.charset", "java.nio.charset.spi", - "java.nio.file", "java.nio.file.attribute", "java.nio.file.spi", - "java.security", "java.security.cert", "java.security.interfaces", - "java.security.spec", "java.text", "java.text.spi", "java.time", - "java.time.chrono", "java.time.format", "java.time.temporal", - "java.time.zone", "java.util", "java.util.concurrent", - "java.util.concurrent.atomic", "java.util.concurrent.locks", - "java.util.function", "java.util.jar", "java.util.random", - "java.util.regex", "java.util.spi", "java.util.stream", "java.util.zip", - "javax.crypto", "javax.crypto.interfaces", "javax.crypto.spec", - "javax.net", "javax.net.ssl", "javax.security.auth", - "javax.security.auth.callback", "javax.security.auth.login", - "javax.security.auth.spi", "javax.security.auth.x500", - "javax.security.cert" - ).flatMap(interimPackages).distinct - - private val wellKnownPackages = commonScalaPackages ++ commonJavaPackages - def isWellKnownPackageSelector(v: String): Boolean = { - val normalized = v.stripPrefix("_root_.") - wellKnownPackages.exists(normalized.startsWith(_)) - } - - protected def sanitizeTermName(name: TermName): TermName = - termNames.getOrElseUpdate( - name, - toTermName(sanitizeSymbolOf(toSymbol(name), termNames.size.toString)) - ) - - protected def sanitizeTypeName(name: TypeName): TypeName = - typeNames.getOrElseUpdate( - name, - toTypeName(sanitizeSymbolOf(toSymbol(name), typeNames.size.toString)) - ) - - protected def sanitizeUnclasifiedName( - name: UnclasifiedName - ): UnclasifiedName = - unclasifiedNames.getOrElseUpdate( - name, - toUnclasifiedName( - sanitizeSymbolOf(toSymbol(name), unclasifiedNames.size.toString) - ) - ) - - protected def sanitizeStringLiteral(original: String): String = - original.map(c => if (c.isLetterOrDigit) '-' else c) - - protected def santitizeScalaSymbol(original: scala.Symbol): scala.Symbol = - scala.Symbol( - toSymbol( - sanitizeUnclasifiedName(toUnclasifiedName(original.name)) - ) - ) - - private def cacheOf[T] = mutable.Map.empty[T, T] - private val symbols = cacheOf[String] - private val termNames = cacheOf[TermName] - private val typeNames = cacheOf[TypeName] - private val unclasifiedNames = cacheOf[UnclasifiedName] - - private def sanitizeSymbolOf( - originalSymbol: String, - suffix: => String - ): String = symbols.getOrElseUpdate( - originalSymbol, { - if (originalSymbol.length() <= ShortSymbolLength) originalSymbol - else generateSymbol(originalSymbol, suffix) - } - ) - - private def generateSymbol( - originalSymbol: String, - suffix: => String - ): String = - if (originalSymbol.forall(isAsciiLetterOrDigit)) - newSimpleSymbol(originalSymbol, suffix, fillChar = 'x') - else - newSymbolsWithSpecialCharacters(originalSymbol) - - private def isAsciiLetterOrDigit(c: Char) = - (c >= 'a' && c <= 'z') || - (c >= 'A' && c <= 'Z') || - (c >= '0' && c <= '9') - - private def newSymbolsWithSpecialCharacters( - originalSymbol: String - ): String = { - val rnd = new Random(originalSymbol.##) - - def nextAsciiLetter(original: Char) = { - val c = (rnd.nextInt('z' - 'a') + 'a').toChar - val next = if (original.isUpper) c.toUpper else c - next match { - case 'O' => 'P' // To easiliy distinquish O from 0 - case 'I' => 'J' // I vs l - case 'l' => 'k' // l vs I - case c => c - } - } - @tailrec def generate(): String = { - val newSymbol = originalSymbol.map(c => - if (!isAsciiLetterOrDigit(c)) c - else if (c.isDigit) c - else nextAsciiLetter(c) - ) - if (symbols.values.exists(_ == newSymbol)) generate() - else newSymbol - } - generate() - } - - private def newSimpleSymbol( - originalSymbol: String, - suffix: String, - fillChar: Char - ): String = { - val prefix = originalSymbol.take(SymbolToPrefixLength) - val prefixHead = - if (originalSymbol.head.isUpper) prefix.head.toUpper - else prefix.head.toLower - val fillInLength = - originalSymbol.length() - prefix.length() - suffix.length() - val prefixTail = - if (fillInLength < 0) prefix.tail.dropRight(-fillInLength) - else prefix.tail - - val sb = new java.lang.StringBuilder(originalSymbol.length()) - sb.append(prefixHead) - sb.append(prefixTail) - 0.until(fillInLength).foreach(_ => sb.append(fillChar)) - sb.append(suffix) - sb.toString() - } - - } -} - -private object SourceCodeTransformer { - object CommonNames { - final val types: Seq[String] = Seq("Byte", "Short", "Int", "Long", "String", - "Unit", "Nothing", "Class", "Option", "Some", "None", "List", "Nil", - "Set", "Seq", "Array", "Vector", "Stream", "LazyList", "Map", "Future", - "Try", "Success", "Failure", "mutable", "immutable") - - final val methods: Seq[String] = Seq("get", "getOrElse", "orElse", "map", - "left", "right", "flatMap", "flatten", "apply", "unapply", "fold", - "foldLeft", "foldRight", "reduce", "reduceLeft", "reduceRight", "scan", - "scanLeft", "scanRight", "recover", "recoverWith", "size", "length", - "exists", "contains", "forall", "value", "underlying", "classOf", - "toOption", "toEither", "toLeft", "toRight", "toString", "to", - "stripMargin", "empty") - } -} diff --git a/mtags/src/main/scala-3/scala/meta/internal/mtags/MtagsEnrichments.scala b/mtags/src/main/scala-3/scala/meta/internal/mtags/MtagsEnrichments.scala index f2a124acd48..193b6bedcaf 100644 --- a/mtags/src/main/scala-3/scala/meta/internal/mtags/MtagsEnrichments.scala +++ b/mtags/src/main/scala-3/scala/meta/internal/mtags/MtagsEnrichments.scala @@ -53,7 +53,7 @@ object MtagsEnrichments extends ScalametaCommonEnrichments: new SourcePosition(source, span) end sourcePosition - def latestRun = + def latestRun = if driver.currentCtx.run.units.nonEmpty then driver.currentCtx.run.units.head else diff --git a/mtags/src/main/scala-3/scala/meta/internal/pc/PcInlayHintsProvider.scala b/mtags/src/main/scala-3/scala/meta/internal/pc/PcInlayHintsProvider.scala index 4d398274c67..f01eddbb902 100644 --- a/mtags/src/main/scala-3/scala/meta/internal/pc/PcInlayHintsProvider.scala +++ b/mtags/src/main/scala-3/scala/meta/internal/pc/PcInlayHintsProvider.scala @@ -2,7 +2,6 @@ package scala.meta.internal.pc import java.nio.file.Paths -import scala.meta.internal.metals.ReportContext import scala.meta.internal.mtags.MtagsEnrichments.* import scala.meta.internal.pc.printer.MetalsPrinter import scala.meta.internal.pc.printer.ShortenedNames @@ -24,6 +23,7 @@ import dotty.tools.dotc.util.Spans.Span import org.eclipse.lsp4j as l import org.eclipse.lsp4j.InlayHint import org.eclipse.lsp4j.InlayHintKind +import scala.meta.pc.ReportContext class PcInlayHintsProvider( driver: InteractiveDriver, diff --git a/mtags/src/main/scala-3/scala/meta/internal/pc/ScalaPresentationCompiler.scala b/mtags/src/main/scala-3/scala/meta/internal/pc/ScalaPresentationCompiler.scala index 369689e62b6..67cc529b176 100644 --- a/mtags/src/main/scala-3/scala/meta/internal/pc/ScalaPresentationCompiler.scala +++ b/mtags/src/main/scala-3/scala/meta/internal/pc/ScalaPresentationCompiler.scala @@ -9,7 +9,6 @@ import java.util.concurrent.CompletableFuture import java.util.concurrent.ExecutorService import java.util.concurrent.ScheduledExecutorService import java.util.logging.Logger -import java.{util as ju} import scala.collection.JavaConverters.* import scala.concurrent.ExecutionContext diff --git a/mtags/src/main/scala-3/scala/meta/internal/pc/completions/MatchCaseCompletions.scala b/mtags/src/main/scala-3/scala/meta/internal/pc/completions/MatchCaseCompletions.scala index 208f0892b05..f0bfb3daa5b 100644 --- a/mtags/src/main/scala-3/scala/meta/internal/pc/completions/MatchCaseCompletions.scala +++ b/mtags/src/main/scala-3/scala/meta/internal/pc/completions/MatchCaseCompletions.scala @@ -7,7 +7,6 @@ import scala.collection.JavaConverters.* import scala.collection.mutable import scala.collection.mutable.ListBuffer -import scala.meta.internal.metals.ReportContext import scala.meta.internal.mtags.MtagsEnrichments.* import scala.meta.internal.pc.AutoImports.AutoImportsGenerator import scala.meta.internal.pc.AutoImports.SymbolImport @@ -32,6 +31,7 @@ import dotty.tools.dotc.core.Types.Type import dotty.tools.dotc.core.Types.TypeRef import dotty.tools.dotc.util.SourcePosition import org.eclipse.lsp4j as l +import scala.meta.pc.ReportContext object CaseKeywordCompletion: diff --git a/mtags/src/main/scala/scala/meta/internal/metals/ScalametaSourceCodeTransformer.scala b/mtags/src/main/scala/scala/meta/internal/metals/ScalametaSourceCodeTransformer.scala deleted file mode 100644 index c0d1f5679a0..00000000000 --- a/mtags/src/main/scala/scala/meta/internal/metals/ScalametaSourceCodeTransformer.scala +++ /dev/null @@ -1,82 +0,0 @@ -package scala.meta.internal.metals - -import scala.meta._ - -object ScalametaSourceCodeTransformer - extends SourceCodeTransformer[Dialect, Tree] { - private val availableDialects = { - import scala.meta.dialects._ - val mainDialects = Seq( - Dialect.current, - Scala3, - Scala213, - Sbt1, - Scala3Future, - Scala212 - ) - val auxilaryDialects = Seq( - Scala213Source3, - Scala212Source3, - Scala31, - Scala32, - Scala33, - Scala211, - Scala210, - Sbt0137, - Sbt0136 - ) - (mainDialects ++ auxilaryDialects) - } - - override def parse(source: String): Option[(Dialect, Tree)] = - availableDialects.iterator - .map { implicit dialect: meta.Dialect => - dialect -> parse(source, dialect) - } - .collectFirst { case (dialect, Some(tree)) => dialect -> tree } - - override def parse(source: String, context: Dialect): Option[Tree] = - context(source).parse[Source].toOption - - override def toSourceString(value: Tree, ctx: Dialect): String = - value.syntax - - override def transformer: ASTTransformer = ScalaMetaTransformer - - private object ScalaMetaTransformer extends Transformer with ASTTransformer { - override protected type Name = meta.Name - override protected type TermName = meta.Term.Name - override protected type TypeName = meta.Type.Name - override protected type UnclasifiedName = meta.Name.Indeterminate - - override protected def toTermName(name: String): TermName = - meta.Term.Name(name) - override protected def toTypeName(name: String): TypeName = - meta.Type.Name(name) - override protected def toUnclasifiedName(name: String): UnclasifiedName = - meta.Name.Indeterminate(name) - override protected def toSymbol(name: Name): String = name.value - - override def sanitizeSymbols(tree: Tree): Option[Tree] = Option( - this.apply(tree) - ) - - override def apply(tree: Tree): Tree = { - tree match { - case name: Name if isCommonScalaName(name) => name - case node: Term.Select if isWellKnownPackageSelector(node.toString()) => - node - case node: Type.Select if isWellKnownPackageSelector(node.toString()) => - node - case node: Type.Name => sanitizeTypeName(node) - case node: Term.Name => sanitizeTermName(node) - case node: Name.Indeterminate => sanitizeUnclasifiedName(node) - case lit: Lit.String => Lit.String(sanitizeStringLiteral(lit.value)) - case lit: Lit.Symbol => Lit.Symbol(santitizeScalaSymbol(lit.value)) - case x => super.apply(x) - } - } - - } - -} diff --git a/telemetry-interfaces/src/main/scala/scala/meta/internal/telemetry/Reports.scala b/telemetry-interfaces/src/main/scala/scala/meta/internal/telemetry/Reports.scala index 459e3446bb0..f1e8e81a078 100644 --- a/telemetry-interfaces/src/main/scala/scala/meta/internal/telemetry/Reports.scala +++ b/telemetry-interfaces/src/main/scala/scala/meta/internal/telemetry/Reports.scala @@ -20,8 +20,8 @@ case class ErrorReport( reporterName: String, reporterContext: ReporterContext, env: Environment = Environment.instance, + text: String, id: Option[String] = None, - text: Option[String] = None, error: Option[ExceptionSummary] = None, ) diff --git a/telemetry-interfaces/src/main/scala/scala/meta/internal/telemetry/TelemetryEndpoints.scala b/telemetry-interfaces/src/main/scala/scala/meta/internal/telemetry/TelemetryEndpoints.scala new file mode 100644 index 00000000000..626727da0ed --- /dev/null +++ b/telemetry-interfaces/src/main/scala/scala/meta/internal/telemetry/TelemetryEndpoints.scala @@ -0,0 +1,14 @@ +package scala.meta.internal.telemetry + +import sttp.tapir._ +import sttp.tapir.json.jsoniter._ +import sttp.tapir.generic.auto._ + +object TelemetryEndpoints { + val baseEndpoint = endpoint.in("v1.0") + + val sendReport: PublicEndpoint[ErrorReport, Unit, Unit, Any] = + baseEndpoint.post + .in("reporting" / "sendReport") + .in(jsonBody[ErrorReport]) +} diff --git a/telemetry-interfaces/src/main/scala/scala/meta/internal/telemetry/TelemetryService.scala b/telemetry-interfaces/src/main/scala/scala/meta/internal/telemetry/TelemetryService.scala deleted file mode 100644 index 9ffc2c6d5fa..00000000000 --- a/telemetry-interfaces/src/main/scala/scala/meta/internal/telemetry/TelemetryService.scala +++ /dev/null @@ -1,24 +0,0 @@ -package scala.meta.internal.telemetry - -import com.github.plokhotnyuk.jsoniter_scala.core._ -import scala.util.Try - -class FireAndForgetEndpoint[In: JsonValueCodec]( - val method: String, - val uri: String, -) { - def encodeInput(request: In): String = writeToString(request) - def decodeInput(request: String): Try[In] = Try { readFromString(request) } -} - -// This will be migrated to tapir endpoints in the next Commit -object TelemetryService { - val sendErrorReportEndpoint = new FireAndForgetEndpoint[ErrorReport]( - "POST", - "/v1/telemetry/sendErrorReport", - ) -} - -trait TelemetryService { - def sendErrorReport(errorReport: ErrorReport): Unit -} diff --git a/tests/unit/src/test/scala/tests/telemetry/SampleReports.scala b/tests/unit/src/test/scala/tests/telemetry/SampleReports.scala index ede10dc9966..b37b87781a7 100644 --- a/tests/unit/src/test/scala/tests/telemetry/SampleReports.scala +++ b/tests/unit/src/test/scala/tests/telemetry/SampleReports.scala @@ -15,7 +15,7 @@ object SampleReports { private def reportOf(ctx: telemetry.ReporterContext): telemetry.ErrorReport = new telemetry.ErrorReport( name = "name", - text = Some("text"), + text = "text", reporterContext = ctx, id = Some("id"), error = Some( diff --git a/tests/unit/src/test/scala/tests/telemetry/SourceCodeSanitizerSuite.scala b/tests/unit/src/test/scala/tests/telemetry/SourceCodeSanitizerSuite.scala deleted file mode 100644 index ecab1f4027d..00000000000 --- a/tests/unit/src/test/scala/tests/telemetry/SourceCodeSanitizerSuite.scala +++ /dev/null @@ -1,162 +0,0 @@ -package tests.telemetry - -import scala.meta.internal.metals.ScalametaSourceCodeTransformer -import scala.meta.internal.metals.SourceCodeSanitizer - -import tests.BaseSuite - -class SourceCodeSanitizerSuite extends BaseSuite { - - val sanitizer = new SourceCodeSanitizer(ScalametaSourceCodeTransformer) - - val sampleScalaInput: String = - """ - |package some.namespace.of.my.app - |class Foo{ - | def myFoo: Int = 42 - |} - |trait Bar{ - | def myBarSecret: String = "my_super-secret-code" - |} - |object FooBar extends Foo with Bar{ - | def compute(input: String, other: Bar): Unit = - | if(myBarSecret.contains("super-secret-code") || this.myBarSecret == other.myBarSecret) myFoo * 42 - | else -1 - |} - """.stripMargin - val sampleScalaOutput: String = - """package som0.namxxxxx1.of.my.ap4 - |class Fo0 { def myFx5: Int = 42 } - |trait Ba1 { def myBxxxxxxx6: String = "--_-----------------" } - |object Fooxx7 extends Fo0 with Ba1 { def comxxx8(inpx9: String, oth10: Ba1): Unit = if (myBxxxxxxx6.contains("-----------------") || this.myBxxxxxxx6 == oth10.myBxxxxxxx6) myFx5 * 42 else -1 } - """.stripMargin - - val sampleStackTraceElements: String = - """ - |scala.meta.internal.pc.completions.OverrideCompletions.scala$meta$internal$pc$completions$OverrideCompletions$$getMembers(OverrideCompletions.scala:180) - | scala.meta.internal.pc.completions.OverrideCompletions$OverrideCompletion.contribute(OverrideCompletions.scala:79) - | scala.meta.internal.pc.CompletionProvider.expected$1(CompletionProvider.scala:439) - | scala.meta.internal.pc.CompletionProvider.safeCompletionsAt(CompletionProvider.scala:499) - | scala.meta.internal.pc.CompletionProvider.completions(CompletionProvider.scala:58) - | scala.meta.internal.pc.ScalaPresentationCompiler.$anonfun$complete$1(ScalaPresentationCompiler.scala:169) - | - |""".stripMargin - - val sampleJavaInput: String = - """ - |package scala.meta.internal.telemetry; - | - |public class ServiceEndpoint { - | final private String uri; - | final private String method; - | final private Class inputType; - | final private Class outputType; - | - | public ServiceEndpoint(String method, String uri, Class inputType, Class outputType) { - | this.uri = uri; - | this.method = method; - | this.inputType = inputType; - | this.outputType = outputType; - | } - | - | public String getUri() { - | return uri; - | } - | - | public String getMethod() { - | return method; - } - """.stripMargin - - val sampleStackTrace: String = - """ - |java.lang.RuntimeException - | at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - | at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) - | at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - | at java.base/java.lang.reflect.Method.invoke(Method.java:568) - | at dotty.tools.repl.Rendering.$anonfun$4(Rendering.scala:110) - | at scala.Option.flatMap(Option.scala:283) - | at dotty.tools.repl.Rendering.valueOf(Rendering.scala:110) - | at dotty.tools.repl.Rendering.renderVal(Rendering.scala:152) - | at dotty.tools.repl.ReplDriver.$anonfun$7(ReplDriver.scala:388) - | at scala.runtime.function.JProcedure1.apply(JProcedure1.java:15) - """.stripMargin - - test("erases names from sources in Scala") { - val input = sampleScalaInput - val expected = sampleScalaOutput - assertNoDiff(sanitizer(input), expected) - } - - test("erases sources in non parsable sources") { // TODO: Java parsing - val input = sampleJavaInput - assertNoDiff(sanitizer(input), "") - } - - test("erases names from markdown snippets") { - val input = - s""" - |## Source code: - |``` - |$sampleScalaInput - |``` - | - |## Scala source code - |```scala - |$sampleScalaInput - |``` - | - |## Java source code - |``` - |${sampleJavaInput} - |``` - | - |## Stacktrace: - |``` - |$sampleStackTrace - |``` - | - |## Stack trace elements - |```scala - |$sampleStackTraceElements - |``` - | - """.stripMargin - - val expected = - s""" - |## Source code: - |``` - |$sampleScalaOutput - |``` - | - |## Scala source code - |```scala - |$sampleScalaOutput - |``` - | - |## Java source code - |``` - | - |``` - | - |## Stacktrace: - |``` - |$sampleStackTrace - |``` - | - |## Stack trace elements - |```scala - |$sampleStackTraceElements - |``` - | - """.stripMargin - def trimLines(string: String) = string.linesIterator - .map(_.trim()) - .filterNot(_.isEmpty()) - .mkString(System.lineSeparator()) - assertNoDiff(trimLines(sanitizer(input)), trimLines(expected)) - } - -} diff --git a/tests/unit/src/test/scala/tests/telemetry/TelemetryReporterSuite.scala b/tests/unit/src/test/scala/tests/telemetry/TelemetryReporterSuite.scala index 687fa535bae..a0f0be7873d 100644 --- a/tests/unit/src/test/scala/tests/telemetry/TelemetryReporterSuite.scala +++ b/tests/unit/src/test/scala/tests/telemetry/TelemetryReporterSuite.scala @@ -1,24 +1,12 @@ package tests.telemetry -import java.io.IOException -import java.net.InetSocketAddress -import java.net.ServerSocket -import java.nio.file.Path -import java.util.Optional - import scala.collection.mutable -import scala.concurrent.ExecutionContext.Implicits.global -import scala.util.control.NonFatal -import scala.meta.internal.jdk.CollectionConverters._ import scala.meta.internal.metals -import scala.meta.internal.mtags.CommonMtagsEnrichments.XtensionOptionalJava import scala.meta.internal.pc.StandardReport import scala.meta.internal.telemetry._ import scala.meta.pc.Report -import io.undertow.server.handlers.BlockingHandler -import io.undertow.server.handlers.PathHandler import tests.BaseSuite import tests.telemetry.SampleReports @@ -38,189 +26,44 @@ class TelemetryReporterSuite extends BaseSuite { assertEquals(metals.TelemetryLevel.Off, getDefault) } - // Remote telemetry reporter should be treated as best effort, ensure that logging - test("ignore connectiviy failures") { - val reporter = new TelemetryReportContext( - telemetryClientConfig = TelemetryClient.Config.default.copy(serverHost = - "https://not.existing.endpoint.for.metals.tests:8081" - ), - telemetryLevel = () => metals.TelemetryLevel.Full, - reporterContext = () => SampleReports.metalsLspReport().reporterContext, - sanitizers = new TelemetryReportContext.Sanitizers(None, None), - ) - - assertEquals( - Optional.empty[Path](), - reporter.incognito.create(simpleReport("")), - ) - } + def testCase(level: metals.TelemetryLevel, expected: Set[String]): Unit = + test( + s"Telemetry level: ${level} sends telemetry for ${expected.mkString("(", ", ", ")")}" + ) { + val client = new TestTelemetryClient() + val reportContexts = Seq( + SampleReports.metalsLspReport(), + SampleReports.scalaPresentationCompilerReport(), + ).map(_.reporterContext) - // Test end-to-end connection and event serialization using local http server implementing TelemetryService endpoints - test("connect with local server") { - implicit val ctx = new MockTelemetryServer.Context() - val server = MockTelemetryServer("127.0.0.1", 8081) - server.start() - try { - val serverEndpoint = MockTelemetryServer.address(server) for { - reporterCtx <- Seq( - SampleReports.metalsLspReport(), - SampleReports.scalaPresentationCompilerReport(), - ).map(_.reporterContext) + reporterCtx <- reportContexts reporter = new TelemetryReportContext( - telemetryClientConfig = TelemetryClient.Config.default - .copy(serverHost = serverEndpoint), - telemetryLevel = () => metals.TelemetryLevel.Full, + telemetryLevel = () => level, reporterContext = () => reporterCtx, - sanitizers = new TelemetryReportContext.Sanitizers( - None, - Some(metals.ScalametaSourceCodeTransformer), - ), + workspaceSanitizer = new metals.WorkspaceSanitizer(None), + telemetryClient = client, ) } { - val createdReport = simpleReport(reporterCtx.toString()) - reporter.incognito.create(createdReport) - Thread.sleep(1000) // wait for the server to receive the event - val received = ctx.errors.filter(_.id == createdReport.id.asScala) - assert(received.nonEmpty, "Not received matching id") - assert(received.size == 1, "Found more then 1 received event") - } - } finally server.stop() - } - - locally { - implicit val ctx = new MockTelemetryServer.Context() - val server = MockTelemetryServer("127.0.0.1", 8081) - - def testCase(level: metals.TelemetryLevel, expected: Set[String]): Unit = - test( - s"Telemetry level: ${level} sends telemetry for ${expected.mkString("(", ", ", ")")}" - ) { - ctx.errors.clear() - try { - server.start() - val serverEndpoint = MockTelemetryServer.address(server) - for { - reporterCtx <- Seq( - SampleReports.metalsLspReport(), - SampleReports.scalaPresentationCompilerReport(), - ).map(_.reporterContext) - reporter = new TelemetryReportContext( - telemetryClientConfig = TelemetryClient.Config.default - .copy(serverHost = serverEndpoint), - telemetryLevel = () => level, - reporterContext = () => reporterCtx, - sanitizers = new TelemetryReportContext.Sanitizers( - None, - Some(metals.ScalametaSourceCodeTransformer), - ), - ) - } { + reporter.incognito.create(simpleReport("incognito")) + reporter.bloop.create(simpleReport("bloop")) + reporter.unsanitized.create(simpleReport("unsanitized")) - reporter.incognito.create(simpleReport("incognito")) - reporter.bloop.create(simpleReport("bloop")) - reporter.unsanitized.create(simpleReport("unsanitized")) - - def received = ctx.errors.map(_.id.get).toSet - Thread.sleep(1000) - assertEquals(received, expected) - } - } finally { - server.stop() - } + val received = client.reportsBuffer.map(_.id.get).toSet + assertEquals(received, expected) } + } - testCase(metals.TelemetryLevel.Off, Set()) - testCase(metals.TelemetryLevel.Anonymous, Set("incognito", "bloop")) - testCase( - metals.TelemetryLevel.Full, - Set("incognito", "bloop", "unsanitized"), - ) - } - + testCase(metals.TelemetryLevel.Off, Set()) + testCase(metals.TelemetryLevel.Anonymous, Set("incognito", "bloop")) + testCase(metals.TelemetryLevel.Full, Set("incognito", "bloop", "unsanitized")) } -object MockTelemetryServer { - import io.undertow.Handlers.path - import io.undertow.Undertow - import io.undertow.server.HttpHandler - import io.undertow.server.HttpServerExchange - import io.undertow.util.Headers - - case class Context( - errors: mutable.ListBuffer[ErrorReport] = mutable.ListBuffer.empty - ) - - def apply( - host: String, - preferredPort: Int, - )(implicit ctx: Context): Undertow = { - val port = freePort(host, preferredPort) +class TestTelemetryClient extends TelemetryClient { + val reportsBuffer = mutable.ListBuffer.empty[ErrorReport] - val baseHandler = path() - .withEndpoint( - TelemetryService.sendErrorReportEndpoint, - _.errors, - ) - Undertow.builder - .addHttpListener(port, host) - .setHandler(baseHandler) - .build() - } - - implicit class EndpointOps(private val handler: PathHandler) extends AnyVal { - def withEndpoint[In]( - endpoint: FireAndForgetEndpoint[In], - eventCollectionsSelector: Context => mutable.ListBuffer[In], - )(implicit ctx: Context): PathHandler = handler.addExactPath( - endpoint.uri, - new BlockingHandler( - new SimpleHttpHandler[In]( - endpoint, - eventCollectionsSelector(ctx), - ) - ), - ) - } - - private class SimpleHttpHandler[In]( - endpoint: FireAndForgetEndpoint[In], - receivedEvents: mutable.ListBuffer[In], - ) extends HttpHandler { - override def handleRequest(exchange: HttpServerExchange): Unit = { - exchange.getRequestReceiver().receiveFullString { - (exchange: HttpServerExchange, json: String) => - receivedEvents += endpoint.decodeInput(json).get - exchange - .getResponseHeaders() - .put(Headers.CONTENT_TYPE, "application/json") - exchange - .getResponseSender() - .send("") - } - } - } - - def address(server: Undertow): String = - server.getListenerInfo.asScala.headOption match { - case Some(listener) => - s"${listener.getProtcol}:/" + listener.getAddress.toString - case None => "" - } - - final def freePort(host: String, port: Int, maxRetries: Int = 20): Int = { - try { - val socket = new ServerSocket() - try { - socket.bind(new InetSocketAddress(host, port)) - socket.getLocalPort() - } finally { - socket.close() - } - } catch { - case NonFatal(_: IOException) if maxRetries > 0 => - freePort(host, port + 1, maxRetries - 1) - } + override val sendReport: ErrorReport => Unit = error => { + reportsBuffer += error } }