From 4e8da48b2e9855b1ac98bea6f345e7699a0aa555 Mon Sep 17 00:00:00 2001 From: Matt Bovel Date: Thu, 18 Dec 2025 18:15:51 +0000 Subject: [PATCH] Cache file bytes in a configurable cache --- .../src/dotty/tools/dotc/GlobalCache.scala | 59 +++++++++++++++++++ .../tools/dotc/config/JavaPlatform.scala | 2 +- .../src/dotty/tools/dotc/core/Contexts.scala | 7 ++- .../dotty/tools/dotc/core/SymbolLoaders.scala | 4 +- .../dotc/core/classfile/ClassfileParser.scala | 2 +- .../classfile/ClassfileTastyUUIDParser.scala | 2 +- .../core/classfile/ReusableDataReader.scala | 53 ++++------------- .../dotty/tools/dotc/util/SourceFile.scala | 6 +- compiler/test/dotty/tools/DottyTest.scala | 1 + .../test/dotty/tools/TestGlobalCache.scala | 17 ++++++ .../dotty/tools/dotc/CompilationTests.scala | 1 + .../dotc/config/ScalaSettingsTests.scala | 5 +- .../dotty/tools/vulpix/ParallelTesting.scala | 24 +++++--- .../dotty/tools/scaladoc/site/templates.scala | 3 +- 14 files changed, 126 insertions(+), 60 deletions(-) create mode 100644 compiler/src/dotty/tools/dotc/GlobalCache.scala create mode 100644 compiler/test/dotty/tools/TestGlobalCache.scala diff --git a/compiler/src/dotty/tools/dotc/GlobalCache.scala b/compiler/src/dotty/tools/dotc/GlobalCache.scala new file mode 100644 index 000000000000..f7825548fb50 --- /dev/null +++ b/compiler/src/dotty/tools/dotc/GlobalCache.scala @@ -0,0 +1,59 @@ +package dotty.tools.dotc + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.LongAdder + +import scala.jdk.CollectionConverters.* + +import dotty.tools.dotc.core.Contexts.Context +import dotty.tools.io.{AbstractFile, FileExtension} + +trait GlobalCache: + /** Get the content of a file, possibly caching it globally. + * + * Implementations must be thread-safe. + */ + def getFileContent(file: AbstractFile): Array[Byte] + +object GlobalCache: + /** A global cache that keeps file contents in memory without any size limit. + * + * @param shouldCache + * A predicate that determines whether an [[AbstracFile]] should be cached. + */ + class ConcurrentGlobalCache(shouldCache: AbstractFile => Boolean) extends GlobalCache: + private val cache = ConcurrentHashMap[AbstractFile, Array[Byte]]() + private val totalByExt = ConcurrentHashMap[FileExtension, LongAdder]() + private val missesByExt = ConcurrentHashMap[FileExtension, LongAdder]() + private val uncachedByExt = ConcurrentHashMap[FileExtension, LongAdder]() + + override def getFileContent(file: AbstractFile): Array[Byte] = + totalByExt.computeIfAbsent(file.ext, _ => LongAdder()).increment() + if shouldCache(file) then + cache.computeIfAbsent(file, f => + missesByExt.computeIfAbsent(file.ext, _ => LongAdder()).increment() + //println(s"Caching file: ${file.canonicalPath}") + f.toByteArray + ) + else + uncachedByExt.computeIfAbsent(file.ext, _ => LongAdder()).increment() + file.toByteArray + + final def printCacheStats(): Unit = + println(this.getClass.getSimpleName + " statistics:") + totalByExt.forEach: (ext, totalAdder) => + val misses = missesByExt.computeIfAbsent(ext, _ => LongAdder()).longValue() + val uncached = uncachedByExt.computeIfAbsent(ext, _ => LongAdder()).longValue() + val total = totalAdder.longValue() + val hits = total - misses - uncached + val files = cache.asScala.filter(_._1.ext == ext) + val sizeMB = files.map(_._2.length.toLong).sum.toDouble / (1024 * 1024) + println(f"- *.$ext: hits: $hits, misses: $misses, uncached: $uncached, total: $total, cache size: $sizeMB%.2f MB") + + /** A global cache that does not cache anything. + * + * This is the default value for [[GlobalCache]]. + */ + object NoGlobalCache extends GlobalCache: + override def getFileContent(file: AbstractFile): Array[Byte] = + file.toByteArray diff --git a/compiler/src/dotty/tools/dotc/config/JavaPlatform.scala b/compiler/src/dotty/tools/dotc/config/JavaPlatform.scala index afa701283eb6..ae71afa5ac50 100644 --- a/compiler/src/dotty/tools/dotc/config/JavaPlatform.scala +++ b/compiler/src/dotty/tools/dotc/config/JavaPlatform.scala @@ -78,5 +78,5 @@ class JavaPlatform extends Platform { new ClassfileLoader(bin) def newTastyLoader(bin: AbstractFile)(using Context): SymbolLoader = - new TastyLoader(bin) + new TastyLoader(bin, ctx) } diff --git a/compiler/src/dotty/tools/dotc/core/Contexts.scala b/compiler/src/dotty/tools/dotc/core/Contexts.scala index 9896cfaa6a1d..ebc1c156a41f 100644 --- a/compiler/src/dotty/tools/dotc/core/Contexts.scala +++ b/compiler/src/dotty/tools/dotc/core/Contexts.scala @@ -57,8 +57,9 @@ object Contexts { private val (importInfoLoc, store9) = store8.newLocation[ImportInfo | Null]() private val (typeAssignerLoc, store10) = store9.newLocation[TypeAssigner](TypeAssigner) private val (progressCallbackLoc, store11) = store10.newLocation[ProgressCallback | Null]() + private val (globalCacheLoc, store12) = store11.newLocation[GlobalCache]() - private val initialStore = store11 + private val initialStore = store12 /** The current context */ inline def ctx(using ctx: Context): Context = ctx @@ -189,6 +190,8 @@ object Contexts { val local = progressCallback if local != null then op(local) + def globalCache: GlobalCache = store(globalCacheLoc) + /** The current plain printer */ def printerFn: Context => Printer = store(printerFnLoc) @@ -712,6 +715,7 @@ object Contexts { def setCompilerCallback(callback: CompilerCallback): this.type = updateStore(compilerCallbackLoc, callback) def setIncCallback(callback: IncrementalCallback): this.type = updateStore(incCallbackLoc, callback) def setProgressCallback(callback: ProgressCallback): this.type = updateStore(progressCallbackLoc, callback) + def setGlobalCache(globalCache: GlobalCache): this.type = updateStore(globalCacheLoc, globalCache) def setPrinterFn(printer: Context => Printer): this.type = updateStore(printerFnLoc, printer) def setSettings(settingsState: SettingsState): this.type = updateStore(settingsStateLoc, settingsState) def setRun(run: Run | Null): this.type = updateStore(runLoc, run) @@ -775,6 +779,7 @@ object Contexts { .updated(notNullInfosLoc, Nil) .updated(compilationUnitLoc, NoCompilationUnit) .updated(profilerLoc, Profiler.NoOp) + .updated(globalCacheLoc, GlobalCache.NoGlobalCache) c._searchHistory = new SearchRoot c._gadtState = GadtState(GadtConstraint.empty) c diff --git a/compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala b/compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala index 96f0d0cb9d20..008e1cb3f3eb 100644 --- a/compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala +++ b/compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala @@ -471,10 +471,10 @@ class ClassfileLoader(val classfile: AbstractFile) extends SymbolLoader { classfileParser.run() } -class TastyLoader(val tastyFile: AbstractFile) extends SymbolLoader { +class TastyLoader(val tastyFile: AbstractFile, creationContext: Context) extends SymbolLoader { val isBestEffortTasty = tastyFile.hasBetastyExtension - lazy val tastyBytes = tastyFile.toByteArray + private def tastyBytes = creationContext.globalCache.getFileContent(tastyFile) private lazy val unpickler: tasty.DottyUnpickler = handleUnpicklingExceptions: diff --git a/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala b/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala index ea8a74d18192..2ad0a9c726ae 100644 --- a/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala +++ b/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala @@ -295,7 +295,7 @@ class ClassfileParser( throw new IOException(s"class file '${classfile.canonicalPath}' has location not matching its contents: contains class $className") def run()(using Context): Option[Embedded] = try ctx.base.reusableDataReader.withInstance { reader => - implicit val reader2 = reader.reset(classfile) + implicit val reader2: ReusableDataReader = reader.reset(classfile)(using ctx) report.debuglog("[class] >> " + classRoot.fullName) classfileVersion = parseHeader(classfile) this.pool = new ConstantPool diff --git a/compiler/src/dotty/tools/dotc/core/classfile/ClassfileTastyUUIDParser.scala b/compiler/src/dotty/tools/dotc/core/classfile/ClassfileTastyUUIDParser.scala index e2220e40c6b4..016b89422f82 100644 --- a/compiler/src/dotty/tools/dotc/core/classfile/ClassfileTastyUUIDParser.scala +++ b/compiler/src/dotty/tools/dotc/core/classfile/ClassfileTastyUUIDParser.scala @@ -28,7 +28,7 @@ class ClassfileTastyUUIDParser(classfile: AbstractFile)(ictx: Context) { private var classfileVersion: Header.Version = Header.Version.Unknown def checkTastyUUID(tastyUUID: UUID)(using Context): Unit = try ctx.base.reusableDataReader.withInstance { reader => - implicit val reader2 = reader.reset(classfile) + implicit val reader2: ReusableDataReader = reader.reset(classfile) this.classfileVersion = ClassfileParser.parseHeader(classfile) this.pool = new ConstantPool checkTastyAttr(tastyUUID) diff --git a/compiler/src/dotty/tools/dotc/core/classfile/ReusableDataReader.scala b/compiler/src/dotty/tools/dotc/core/classfile/ReusableDataReader.scala index 35716feb7a42..efd30f8cfa08 100644 --- a/compiler/src/dotty/tools/dotc/core/classfile/ReusableDataReader.scala +++ b/compiler/src/dotty/tools/dotc/core/classfile/ReusableDataReader.scala @@ -6,6 +6,9 @@ package classfile import java.io.{DataInputStream, InputStream} import java.nio.{BufferUnderflowException, ByteBuffer} +import dotty.tools.io.AbstractFile +import dotty.tools.dotc.core.Contexts.{ctx, Context} + final class ReusableDataReader() extends DataReader { private var data = new Array[Byte](32768) private var bb: ByteBuffer = ByteBuffer.wrap(data) @@ -33,49 +36,17 @@ final class ReusableDataReader() extends DataReader { private def nextPositivePowerOfTwo(target: Int): Int = 1 << -Integer.numberOfLeadingZeros(target - 1) - def reset(file: dotty.tools.io.AbstractFile): this.type = { + def reset(file: AbstractFile)(using Context): this.type = { this.size = 0 - file.sizeOption match { - case Some(size) => - if (size > data.length) { - data = new Array[Byte](nextPositivePowerOfTwo(size)) - } else { - java.util.Arrays.fill(data, 0.toByte) - } - val input = file.input - try { - var endOfInput = false - while (!endOfInput) { - val remaining = data.length - this.size - if (remaining == 0) endOfInput = true - else { - val read = input.read(data, this.size, remaining) - if (read < 0) endOfInput = true - else this.size += read - } - } - bb = ByteBuffer.wrap(data, 0, size) - } finally { - input.close() - } - case None => - val input = file.input - try { - var endOfInput = false - while (!endOfInput) { - val remaining = data.length - size - if (remaining == 0) { - data = java.util.Arrays.copyOf(data, nextPositivePowerOfTwo(size)) - } - val read = input.read(data, this.size, data.length - this.size) - if (read < 0) endOfInput = true - else this.size += read - } - bb = ByteBuffer.wrap(data, 0, size) - } finally { - input.close() - } + val bytes = ctx.globalCache.getFileContent(file) + val size = bytes.length + if (size > data.length) { + data = new Array[Byte](nextPositivePowerOfTwo(size)) + } else { + java.util.Arrays.fill(data, 0.toByte) } + System.arraycopy(bytes, 0, data, 0, size) + bb = ByteBuffer.wrap(data, 0, size) this } diff --git a/compiler/src/dotty/tools/dotc/util/SourceFile.scala b/compiler/src/dotty/tools/dotc/util/SourceFile.scala index 8800743c4f1c..7afc19e780cc 100644 --- a/compiler/src/dotty/tools/dotc/util/SourceFile.scala +++ b/compiler/src/dotty/tools/dotc/util/SourceFile.scala @@ -304,11 +304,13 @@ object SourceFile { def isScript(file: AbstractFile | Null, content: Array[Char]): Boolean = ScriptSourceFile.hasScriptHeader(content) - def apply(file: AbstractFile | Null, codec: Codec): SourceFile = + def apply(file: AbstractFile | Null, codec: Codec)(using Context): SourceFile = // Files.exists is slow on Java 8 (https://rules.sonarsource.com/java/tag/performance/RSPEC-3725), // so cope with failure. val chars = - try new String(file.toByteArray, codec.charSet).toCharArray + try + val bytes = ctx.globalCache.getFileContent(file) + new String(bytes, codec.charSet).toCharArray catch case _: FileSystemException => Array.empty[Char] diff --git a/compiler/test/dotty/tools/DottyTest.scala b/compiler/test/dotty/tools/DottyTest.scala index 76d2fdcb6d26..33181f137f1f 100644 --- a/compiler/test/dotty/tools/DottyTest.scala +++ b/compiler/test/dotty/tools/DottyTest.scala @@ -42,6 +42,7 @@ trait DottyTest extends ContextEscapeDetection { fc.setSetting(fc.settings.classpath, TestConfiguration.basicClasspath) fc.setSetting(fc.settings.language, List("experimental.erasedDefinitions").asInstanceOf) fc.setProperty(ContextDoc, new ContextDocstrings) + fc.setGlobalCache(TestGlobalCache) } protected def defaultCompiler: Compiler = new Compiler() diff --git a/compiler/test/dotty/tools/TestGlobalCache.scala b/compiler/test/dotty/tools/TestGlobalCache.scala new file mode 100644 index 000000000000..2ba0a1c81395 --- /dev/null +++ b/compiler/test/dotty/tools/TestGlobalCache.scala @@ -0,0 +1,17 @@ +package dotty.tools + +import dotty.tools.dotc.GlobalCache +import dotty.tools.dotc.core.Contexts.Context +import dotty.tools.io.{AbstractFile, FileExtension} +import java.io.File + +object TestGlobalCache extends GlobalCache.ConcurrentGlobalCache( + file => { + if file.ext == FileExtension.Class || file.ext == FileExtension.Tasty then + !file.canonicalPath.startsWith("out") + else if file.ext == FileExtension.Scala then + file.canonicalPath.startsWith("library/src") + else + false + } +) diff --git a/compiler/test/dotty/tools/dotc/CompilationTests.scala b/compiler/test/dotty/tools/dotc/CompilationTests.scala index 74ff53b9ebf3..6e29e2334d69 100644 --- a/compiler/test/dotty/tools/dotc/CompilationTests.scala +++ b/compiler/test/dotty/tools/dotc/CompilationTests.scala @@ -406,6 +406,7 @@ object CompilationTests extends ParallelTesting { implicit val summaryReport: SummaryReporting = new SummaryReport @AfterClass def tearDown(): Unit = { + //TestGlobalCache.printCacheStats() super.cleanup() summaryReport.echoSummary() } diff --git a/compiler/test/dotty/tools/dotc/config/ScalaSettingsTests.scala b/compiler/test/dotty/tools/dotc/config/ScalaSettingsTests.scala index c36e15a0eb36..43a08815942b 100644 --- a/compiler/test/dotty/tools/dotc/config/ScalaSettingsTests.scala +++ b/compiler/test/dotty/tools/dotc/config/ScalaSettingsTests.scala @@ -6,6 +6,7 @@ import Settings._ import dotty.tools.dotc.config.ScalaSettingCategories._ import org.junit.Test import org.junit.Assert._ +import core.Contexts.NoContext import core.Decorators.toMessage import dotty.tools.io.{Path, PlainFile} @@ -248,7 +249,7 @@ class ScalaSettingsTests: warning = reporting.Diagnostic.Warning( "A warning".toMessage, util.SourcePosition( - source = util.SourceFile(new PlainFile(Path(file)), "UTF-8"), + source = util.SourceFile(new PlainFile(Path(file)), "UTF-8")(using NoContext), span = util.Spans.Span(1L) ) ) @@ -263,7 +264,7 @@ class ScalaSettingsTests: warning = reporting.Diagnostic.Warning( "A warning".toMessage, util.SourcePosition( - source = util.SourceFile(new PlainFile(Path(file)), "UTF-8"), + source = util.SourceFile(new PlainFile(Path(file)), "UTF-8")(using NoContext), span = util.Spans.Span(1L) ) ) diff --git a/compiler/test/dotty/tools/vulpix/ParallelTesting.scala b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala index 3a2f261a8e16..41be6f0b76ec 100644 --- a/compiler/test/dotty/tools/vulpix/ParallelTesting.scala +++ b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala @@ -63,6 +63,14 @@ trait ParallelTesting extends RunnerOrchestration: protected def testPlatform: TestPlatform = TestPlatform.JVM + + def setupTestContext(initCtx: FreshContext): FreshContext = + initCtx.setGlobalCache(TestGlobalCache) + initCtx + + private class TestDriver extends Driver: + override protected def initCtx = setupTestContext(super.initCtx.fresh) + /** A test source whose files or directory of files is to be compiled * in a specific way defined by the `Test` */ @@ -533,8 +541,8 @@ trait ParallelTesting extends RunnerOrchestration: val reporter = mkReporter val driver = - if (times == 1) new Driver - else new Driver { + if (times == 1) TestDriver() + else new TestDriver { private def ntimes(n: Int)(op: Int => Reporter): Reporter = (1 to n).foldLeft(emptyReporter) ((_, i) => op(i)) @@ -581,7 +589,7 @@ trait ParallelTesting extends RunnerOrchestration: reporter } - private def parseErrors(errorsText: String, compilerVersion: String, pageWidth: Int) = + private def parseErrors(errorsText: String, compilerVersion: String, pageWidth: Int)(using Context) = val errorPattern = """^.*Error: (.*\.scala):(\d+):(\d+).*""".r val brokenClassPattern = """^class file (.*) is broken.*""".r val warnPattern = """^.*Warning: (.*\.scala):(\d+):(\d+).*""".r @@ -674,9 +682,9 @@ trait ParallelTesting extends RunnerOrchestration: val reporter = mkReporter val errorsText = Source.fromInputStream(process.getErrorStream).mkString if process.waitFor() != 0 then - val diagnostics = parseErrors(errorsText, compiler, pageWidth) + val context = (new ContextBase).initialCtx + val diagnostics = parseErrors(errorsText, compiler, pageWidth)(using context) diagnostics.foreach { diag => - val context = (new ContextBase).initialCtx reporter.report(diag)(using context) } @@ -686,7 +694,7 @@ trait ParallelTesting extends RunnerOrchestration: val classes = flattenFiles(targetDir).filter(isBestEffortTastyFile).map(_.toString) val flags = flags0 `and` "-from-tasty" `and` "-Ywith-best-effort-tasty" val reporter = mkReporter - val driver = new Driver + val driver = TestDriver() driver.process(flags.all ++ classes, reporter = reporter) @@ -698,7 +706,7 @@ trait ParallelTesting extends RunnerOrchestration: .and("-Ywith-best-effort-tasty") .and("-d", targetDir.getPath) val reporter = mkReporter - val driver = new Driver + val driver = TestDriver() val args = Array("-classpath", flags.defaultClassPath + JFile.pathSeparator + bestEffortDir.toString) ++ flags.options @@ -716,7 +724,7 @@ trait ParallelTesting extends RunnerOrchestration: val reporter = mkReporter - val driver = new Driver + val driver = TestDriver() driver.process(flags.all ++ classes, reporter = reporter) diff --git a/scaladoc/src/dotty/tools/scaladoc/site/templates.scala b/scaladoc/src/dotty/tools/scaladoc/site/templates.scala index 6d5eeb06267a..f6f6b0a1942d 100644 --- a/scaladoc/src/dotty/tools/scaladoc/site/templates.scala +++ b/scaladoc/src/dotty/tools/scaladoc/site/templates.scala @@ -80,7 +80,8 @@ case class TemplateFile( lazy val snippetCheckingFunc: SnippetChecker.SnippetCheckingFunc = val path = Some(Paths.get(file.getAbsolutePath)) val pathBasedArg = ssctx.snippetCompilerArgs.get(path) - val sourceFile = dotty.tools.dotc.util.SourceFile(dotty.tools.io.AbstractFile.getFile(path.get), scala.io.Codec.UTF8) + val initDottyContext = (new dotty.tools.dotc.core.Contexts.ContextBase).initialCtx + val sourceFile = dotty.tools.dotc.util.SourceFile(dotty.tools.io.AbstractFile.getFile(path.get), scala.io.Codec.UTF8)(using initDottyContext) (str: String, lineOffset: SnippetChecker.LineOffset, argOverride: Option[SnippetCompilerArg]) => { val arg = argOverride.fold(pathBasedArg)(pathBasedArg.merge(_)) val compilerData = SnippetCompilerData(