Merge pull request #986 from ephemerist/feature/direct-to-jar

Enable direct-to-jar compilation for local java

internal/compiler-interface/src/main/java/xsbti/compile/JavaCompiler.java

@@ -12,4 +12,8 @@
 package xsbti.compile;
 
 /** Represent the interface of a Java compiler. */
-public interface JavaCompiler extends JavaTool {}
+public interface JavaCompiler extends JavaTool {
+    default boolean supportsDirectToJar() {
+        return false;
+    }
+}

internal/zinc-classfile/src/main/scala/sbt/internal/inc/JarUtils.scala

@@ -332,8 +332,8 @@ object JarUtils {
   }
 
   /**
-   * As javac does not support compiling directly to jar it is required to
-   * change its output to a directory that is temporary, as after compilation
+   * As some javac implementations do not support compiling directly to jar it is
+   * required to change its output to a directory that is temporary, as after compilation
    * the plain classes are put into a zip file and merged with the output jar.
    *
    * This method returns path to this directory based on output jar. The result

internal/zinc-classfile/src/main/scala/sbt/internal/inc/classfile/JavaAnalyze.scala

@@ -44,8 +44,9 @@ private[sbt] object JavaAnalyze {
       .toSet[VirtualFile]
       .groupBy(_.name)
     // For performance reasons, precompute these as they are static throughout this analysis
-    val outputJarOrNull: Path = finalJarOutput.getOrElse(null)
     val singleOutputOrNull: Path = output.getSingleOutputAsPath.orElse(null)
+    val directOutputJarOrNull: Path = JarUtils.getOutputJar(output).getOrElse(null)
+    val mappedOutputJarOrNull: Path = finalJarOutput.getOrElse(null)
 
     def load(tpe: String, errMsg: => Option[String]): Option[Class[_]] = {
       if (tpe.endsWith("module-info")) None
@@ -58,8 +59,16 @@ private[sbt] object JavaAnalyze {
     }
 
     def remapClassFile(classFile: Path) =
-      if (singleOutputOrNull == null || outputJarOrNull == null) classFile
-      else resolveFinalClassFile(classFile, singleOutputOrNull, outputJarOrNull, log)
+      if (directOutputJarOrNull != null && classFile.getFileSystem.provider.getScheme == "jar")
+        // convert to the class-in-jar path format that zinc uses. we make an assumption here that
+        // if we've got a jar-based path, it's referring to a class in the output jar.
+        JarUtils
+          .ClassInJar(directOutputJarOrNull, classFile.getRoot.relativize(classFile).toString)
+          .toPath
+      else if (singleOutputOrNull != null && mappedOutputJarOrNull != null)
+        resolveFinalClassFile(classFile, singleOutputOrNull, mappedOutputJarOrNull, log)
+      else
+        classFile
 
     val sourceToClassFiles = mutable.HashMap[VirtualFile, Buffer[ClassFile]](
       sources.map(vf => vf -> new ArrayBuffer[ClassFile]): _*
@@ -85,7 +94,6 @@ private[sbt] object JavaAnalyze {
       binaryClassNameToLoadedClass.update(binaryClassName, loadedClass)
 
       val srcClassName = loadEnclosingClass(loadedClass)
-
       val finalClassFile: Path = remapClassFile(newClass)
       srcClassName match {
         case Some(className) =>
@@ -201,8 +209,8 @@ private[sbt] object JavaAnalyze {
   }
 
   /**
-   * When straight-to-jar compilation is enabled, classes are compiled to a temporary directory
-   * because javac cannot compile to jar directly. The paths to class files that can be observed
+   * When straight-to-jar compilation is enabled on a javac which doesn't support it, classes are compiled to a
+   * temporary directory because javac cannot compile to jar directly. The paths to class files that can be observed
    * here through the file system or class loaders are located in temporary output directory for
    * javac. As this output will be eventually included in the output jar (`finalJarOutput`), the
    * analysis (products) have to be changed accordingly.

internal/zinc-classfile/src/main/scala/sbt/internal/inc/classfile/Parser.scala

@@ -15,6 +15,7 @@ package inc
 package classfile
 
 import java.net.URL
+import java.nio.file.Files
 import java.nio.file.Path
 import java.io.{ BufferedInputStream, InputStream, File, DataInputStream }
 
@@ -32,7 +33,7 @@ import Constants._
 
 private[sbt] object Parser {
   def apply(file: Path): ClassFile =
-    Using.fileInputStream(file.toFile)(parse(file.toString)).right.get
+    Using.bufferedInputStream(Files.newInputStream(file))(parse(file.toString)).right.get
 
   def apply(file: File): ClassFile =
     Using.fileInputStream(file)(parse(file.toString)).right.get

internal/zinc-compile-core/src/main/scala/sbt/internal/inc/javac/LocalJava.scala

@@ -16,11 +16,12 @@ package inc
 package javac
 
 import java.net.{ URI, URLClassLoader }
-import java.io.{ InputStream, OutputStream, PrintWriter, Writer }
+import java.io.{ InputStream, OutputStream, OutputStreamWriter, PrintWriter, Reader, Writer }
 import java.util.Locale
 import java.nio.{ ByteBuffer, CharBuffer }
 import java.nio.charset.{ Charset, CodingErrorAction }
-import java.nio.file.{ Files, Path, Paths }
+import java.nio.file.{ Files, FileSystems, Path, Paths }
+import javax.lang.model.element.{ Modifier, NestingKind }
 import javax.tools.JavaFileManager.Location
 import javax.tools.JavaFileObject.Kind
 import javax.tools.{
@@ -31,11 +32,13 @@ import javax.tools.{
   JavaFileObject,
   SimpleJavaFileObject,
   StandardJavaFileManager,
+  StandardLocation,
   DiagnosticListener
 }
 import sbt.internal.util.LoggerWriter
 import sbt.util.{ Level, Logger }
 
+import scala.collection.JavaConverters._
 import scala.util.control.NonFatal
 import xsbti.{ Reporter, Logger => XLogger, PathBasedFile, VirtualFile, VirtualFileRef }
 import xsbti.compile.{
@@ -276,6 +279,8 @@ final class LocalJavadoc() extends XJavadoc {
  * resident Java compiler.
  */
 final class LocalJavaCompiler(compiler: javax.tools.JavaCompiler) extends XJavaCompiler {
+  override def supportsDirectToJar: Boolean = true
+
   override def run(
       sources: Array[VirtualFile],
       options: Array[String],
@@ -297,26 +302,35 @@ final class LocalJavaCompiler(compiler: javax.tools.JavaCompiler) extends XJavaC
       log.warn("Javac is running in 'local' mode. These flags have been removed:")
       log.warn(invalidOptions.mkString("\t", ", ", ""))
     }
-    val outputOption = CompilerArguments.outputOption(output)
-    output.getSingleOutputAsPath match {
-      case p if p.isPresent => Files.createDirectories(p.get)
-      case _                =>
-    }
 
-    val fileManager = {
-      if (cleanedOptions.contains("-XDuseOptimizedZip=false")) {
-        fileManagerWithoutOptimizedZips(diagnostics)
-      } else {
-        compiler.getStandardFileManager(diagnostics, null, null)
-      }
+    def standardFileManager = compiler.getStandardFileManager(diagnostics, null, null)
+
+    val (fileManager, javacOptions) = JarUtils.getOutputJar(output) match {
+      case Some(outputJar) =>
+        (new DirectToJarFileManager(outputJar, standardFileManager), cleanedOptions.toSeq)
+      case None =>
+        output.getSingleOutputAsPath match {
+          case p if p.isPresent => Files.createDirectories(p.get)
+          case _                =>
+        }
+        val fileManager = {
+          if (cleanedOptions.contains("-XDuseOptimizedZip=false")) {
+            fileManagerWithoutOptimizedZips(diagnostics)
+          } else {
+            standardFileManager
+          }
+        }
+        val outputOption = CompilerArguments.outputOption(output)
+        (fileManager, outputOption ++ cleanedOptions)
     }
 
     val jfiles = sources.toList.map(LocalJava.toFileObject)
     val customizedFileManager = {
       val maybeClassFileManager = incToolOptions.classFileManager()
       if (incToolOptions.useCustomizedFileManager && maybeClassFileManager.isPresent)
         new WriteReportingFileManager(fileManager, maybeClassFileManager.get)
-      else new SameFileFixFileManager(fileManager)
+      else
+        new SameFileFixFileManager(fileManager)
     }
 
     var compileSuccess = false
@@ -326,7 +340,7 @@ final class LocalJavaCompiler(compiler: javax.tools.JavaCompiler) extends XJavaC
           logWriter,
           customizedFileManager,
           diagnostics,
-          (outputOption ++ cleanedOptions).toList.asJava,
+          javacOptions.toList.asJava,
           null,
           jfiles.asJava
         )
@@ -461,3 +475,86 @@ class WriterOutputStream(writer: Writer) extends OutputStream {
   }
   override def toString: String = charBuffer.toString
 }
+
+final class DirectToJarFileManager(
+    outputJar: Path,
+    delegate: JavaFileManager
+) extends ForwardingJavaFileManager[JavaFileManager](delegate) {
+  private val jarFs = {
+    val uri = URI.create("jar:file:" + outputJar.toUri.getPath)
+    def newFs() = FileSystems.newFileSystem(uri, Map("create" -> "true").asJava)
+    // work around java 8 bug which results in ZipFileSystem being initially registered with a null key on the provider
+    newFs().close()
+    newFs()
+  }
+  private val jarRoot = jarFs.getRootDirectories.iterator.next()
+
+  override def getFileForOutput(
+      location: JavaFileManager.Location,
+      packageName: String,
+      relativeName: String,
+      sibling: FileObject
+  ): FileObject =
+    if (location == StandardLocation.CLASS_OUTPUT) {
+      val packagePath = packageName.replace('.', '/')
+      val filePath = jarRoot.resolve(packagePath).resolve(relativeName)
+      new DirectToJarFileObject(filePath, filePath.toUri)
+    } else super.getFileForOutput(location, packageName, relativeName, sibling)
+
+  override def getJavaFileForOutput(
+      location: JavaFileManager.Location,
+      className: String,
+      kind: JavaFileObject.Kind,
+      sibling: FileObject
+  ): JavaFileObject =
+    if (location == StandardLocation.CLASS_OUTPUT) {
+      val relativeFilePath = className.replace('.', '/') + kind.extension
+      val filePath = jarRoot.resolve(relativeFilePath)
+      new DirectToJarJavaFileObject(filePath, filePath.toUri, kind)
+    } else super.getJavaFileForOutput(location, className, kind, sibling)
+
+  override def isSameFile(a: FileObject, b: FileObject): Boolean = a == b
+
+  override def close(): Unit = {
+    // super also holds a reference to outputJar, so we need to close it before closing jarFs
+    super.close()
+    jarFs.close()
+  }
+}
+
+class DirectToJarFileObject(path: Path, uri: URI) extends FileObject {
+  override def getName: String = path.toString
+  override def toUri: URI = uri
+
+  override def getLastModified: Long = 0L
+
+  override def getCharContent(ignoreEncodingErrors: Boolean): CharSequence =
+    throw new UnsupportedOperationException
+  override def openInputStream(): InputStream =
+    throw new UnsupportedOperationException
+  override def openReader(ignoreEncodingErrors: Boolean): Reader =
+    throw new UnsupportedOperationException
+
+  override def openOutputStream(): OutputStream = {
+    Files.createDirectories(path.getParent)
+    Files.newOutputStream(path)
+  }
+  override def openWriter(): Writer = new OutputStreamWriter(openOutputStream())
+
+  override def delete(): Boolean = false
+}
+
+final class DirectToJarJavaFileObject(path: Path, uri: URI, kind: JavaFileObject.Kind)
+    extends DirectToJarFileObject(path, uri)
+    with JavaFileObject {
+
+  override def getKind: JavaFileObject.Kind = kind
+
+  override def isNameCompatible(simpleName: String, kind: JavaFileObject.Kind): Boolean = {
+    val baseName = s"$simpleName${kind.extension}"
+    kind == this.kind && (path.toString == baseName || path.endsWith(baseName))
+  }
+
+  override def getNestingKind: NestingKind = null
+  override def getAccessLevel: Modifier = null
+}

zinc/src/main/scala/sbt/internal/inc/MixedAnalyzingCompiler.scala

@@ -67,12 +67,15 @@ final class MixedAnalyzingCompiler(
           )
         val joptions = config.currentSetup.options.javacOptions
 
+        def toVirtualFile(p: Path) = config.converter.toVirtualFile(p.toAbsolutePath)
+
         JarUtils.getOutputJar(output) match {
-          case Some(outputJar) =>
+          case Some(outputJar) if !javac.supportsDirectToJar =>
             val outputDir = JarUtils.javacTempOutput(outputJar)
             Files.createDirectories(outputDir)
             javac.compile(
               javaSrcs,
+              Seq(toVirtualFile(outputJar)),
               config.converter,
               joptions,
               CompileOutput(outputDir),
@@ -84,19 +87,22 @@ final class MixedAnalyzingCompiler(
               config.progress
             )
             putJavacOutputInJar(outputJar.toFile, outputDir.toFile)
-          case None =>
-            javac.compile(
-              javaSrcs,
-              config.converter,
-              joptions,
-              output,
-              finalJarOutput = None,
-              callback,
-              incToolOptions,
-              config.reporter,
-              log,
-              config.progress
-            )
+          case _ =>
+            JarUtils.withPreviousJar(output) { extraClasspath: Seq[Path] =>
+              javac.compile(
+                javaSrcs,
+                extraClasspath map toVirtualFile,
+                config.converter,
+                joptions,
+                output,
+                None,
+                callback,
+                incToolOptions,
+                config.reporter,
+                log,
+                config.progress
+              )
+            }
         }
       } // timed
     else ()
@@ -389,7 +395,6 @@ object MixedAnalyzingCompiler {
     searchClasspathAndLookup(
       config.converter,
       config.classpath,
-      config.currentSetup.output,
       config.currentSetup.options.scalacOptions,
       config.perClasspathEntryLookup,
       config.compiler
@@ -399,25 +404,14 @@ object MixedAnalyzingCompiler {
   def searchClasspathAndLookup(
       converter: FileConverter,
       classpath: Seq[VirtualFile],
-      output: Output,
       scalacOptions: Array[String],
       perClasspathEntryLookup: PerClasspathEntryLookup,
       compiler: xsbti.compile.ScalaCompiler
   ): (Seq[VirtualFile], String => Option[VirtualFile]) = {
-    // If we are compiling straight to jar, as javac does not support this,
-    // it will be compiled to a temporary directory (with deterministic name)
-    // and then added to the final jar. This temporary directory has to be
-    // available for sbt.internal.inc.classfile.Analyze to work correctly.
-    val tempJavacOutput =
-      JarUtils
-        .getOutputJar(output)
-        .map(JarUtils.javacTempOutput)
-        .toSeq
-        .map(converter.toVirtualFile(_))
     val absClasspath = classpath.map(toAbsolute(_))
     val cArgs =
       new CompilerArguments(compiler.scalaInstance, compiler.classpathOptions)
-    val searchClasspath: Seq[VirtualFile] = tempJavacOutput ++ explicitBootClasspath(
+    val searchClasspath: Seq[VirtualFile] = explicitBootClasspath(
       scalacOptions,
       converter
     ) ++