The best IDE is

your favourite IDE

The best IDE is in your favourite IDE

- Scala3 Compiler Engineer

- IntelliJ Plugin Developer

- Scala.js Frontend Developer

- Tooling Engineer

- Backend Developer

I work on tooling.

Scastie

Metals

IntelliJ

But this is not your average project

Right ?

import quoted.*

class Config(map: Map[String, Any]) extends Selectable:
  def selectDynamic(name: String): Any = map(name)

transparent inline def typesafeConfig(inline pairs: (String, Any)*) = 
  ${ typesafeConfigImpl('pairs) }

def typesafeConfigImpl(pairs: Expr[Seq[(String, Any)]])(using Quotes): Expr[Any] =
  import quotes.reflect.*
  val unpacked = Varargs.unapply(pairs).getOrElse(Nil).map:
    case '{ ($k: String) -> ($v: t) } => k.valueOrError -> v

  val typ = unpacked.foldLeft(TypeRepr.of[Config]): (acc, entry) =>
    Refinement(acc, entry._1, entry._2.asTerm.tpe.widen)

  val params = unpacked.map: (k, v) =>
    '{ ${Expr(k)} -> $v }
  typ.asType match
    case '[t] => '{ Config(Map(${Varargs(params)}: _*)).asInstanceOf[t] }

MACROS

🥁

Metals

IntelliJ

LSP with the rescue

But why did IntelliJ fail ?

Compilation 101*

* The following slides are simplification

Select

"head"

Ident

"godfatherSeries"

val godfatherSeries: List[String] = ???
godfatherSeries.head

Parser

Select

"get"

Ident

Apply

Literal

Parser

val godfatherSeries: List[String] = ???
godfatherSeries.get(0)

"godfatherSeries"

Select

"get"

Ident

Apply

Literal

Typer

val godfatherSeries: List[String] = ???
godfatherSeries.get(0)

"godfatherSeries"

Select

"get"

Ident

Apply

Literal

Typer

val godfatherSeries: List[String] = ???
godfatherSeries.get(0)

"godfatherSeries"

?(...)

Select

"get"

Ident: godfatherSeries.type

Apply

Literal

Typer

val godfatherSeries: List[String] = ???
godfatherSeries.get(0)

"godfatherSeries"

?(...)

?.get

Select

"get"

Ident: godfatherSeries.type

Apply

Literal

Typer

val godfatherSeries: List[String] = ???
godfatherSeries.get(0)

"godfatherSeries"

?(...)

Select: godfatherSeries.get.type

"get"

Ident: godfatherSeries.type

Apply

Literal

Typer

val godfatherSeries: List[String] = ???
godfatherSeries.get(0)

"godfatherSeries"

?(...)

Select: godfatherSeries.get.type

"get"

Ident: godfatherSeries.type

Apply

Literal: 0.type

Typer

val godfatherSeries: List[String] = ???
godfatherSeries.get(0)

"godfatherSeries"

?(...)

Select: godfatherSeries.get.type

"get"

Ident: godfatherSeries.type

Apply

Literal: 0.type

Typer

val godfatherSeries: List[String] = ???
godfatherSeries.get(0)

"godfatherSeries"

?(Int)

Select: godfatherSeries.get.type (Int => T)

"get"

Ident: godfatherSeries.type

Apply

Literal: 0.type

Typer

val godfatherSeries: List[String] = ???
godfatherSeries.get(0)

"godfatherSeries"

?(Int)

Select: godfatherSeries.get.type (Int => String)

"get"

Ident: godfatherSeries.type

Apply

Literal: 0.type

Typer

val godfatherSeries: List[String] = ???
godfatherSeries.get(0)

"godfatherSeries"

?(Int)

Select: godfatherSeries.get.type

"get"

Ident: godfatherSeries.type

Apply: String

Literal: 0.type

Typer

val godfatherSeries: List[String] = ???
godfatherSeries.get(0)

"godfatherSeries"

Abstract Syntax Trees

Select

"get"

Ident

Apply

Literal

"godfatherSeries"

ReferenceExpression

"get"

ReferenceExpression

MethodCall

ArgumentList

"godfatherSeries"

PsiElement(.)

PSI - Program Structure Interface

AST - Abstract Syntax Tree

This is not bad

It's just a tradeoff

Error recovery

Correctness

Error recovery

Correctness

Compiler diagnostics

Error recovery

Correctness

Compiler diagnostics

Error recovery

Correctness

BTasty format

Error recovery

Correctness

BTasty format

Error recovery

Correctness

Where am I going with this ?

Not all features require typer

But those that do, should rely on true representation

Completions

Hovers

Signature help

And others are more reliant on indexes

Go-to definition

Find usages

Indexes can enrich IDE
IDE can enrich indexes

I did a thing

But this has a very big overhead

Resource usage

Completions

public abstract CompletableFuture<CompletionList> complete(OffsetParams params);

Hovers

public abstract CompletableFuture<Optional<HoverSignature>> hover(OffsetParams params);

Signature help

public abstract CompletableFuture<CompletionList> complete(OffsetParams params);

Go-to definition

public abstract CompletableFuture<SignatureHelpResult> signatureHelp(OffsetParams params);

Find usages

IntelliJ plugin creation 101

build.sbt

import org.jetbrains.sbtidea.IntelliJPlatform.IdeaCommunity

ThisBuild / intellijPluginName := "intellij-metals"
ThisBuild / intellijPlatform := IdeaCommunity
ThisBuild / intellijBuild := "241.14494.240"

lazy val root = project
  .enablePlugins(SbtIdeaPlugin)
  .settings(
    name := "intellij-next",
    scalaVersion := 2.13.12,
    intellijPlugins := Seq(
      "com.intellij.java".toPlugin,
      "org.intellij.scala".toPlugin
    )
  )

build.sbt

import org.jetbrains.sbtidea.IntelliJPlatform.IdeaCommunity

ThisBuild / intellijPluginName := "intellij-metals"
ThisBuild / intellijPlatform := IdeaCommunity
ThisBuild / intellijBuild := "241.14494.240"

lazy val root = project
  .enablePlugins(SbtIdeaPlugin)
  .settings(
    name := "intellij-next",
    scalaVersion := 2.13.12,
    intellijPlugins := Seq(
      "com.intellij.java".toPlugin,
      "org.intellij.scala".toPlugin
    ),
    libraryDependencies += 
      "org.scala-lang" % "scala3-presentation-compiler_3" % "3.3.3",
  )

<idea-plugin>
    <id>intellij-metals</id>
    <name>Scala LSP (Metals) for IntelliJ</name>
    <version>replaced-by-build</version>

    <idea-version since-build="241" until-build="241.*"/>

    <depends>com.intellij.modules.platform</depends>
    <depends>org.intellij.scala</depends>

    <extensions defaultExtensionNs="com.intellij">

    </extensions>

</idea-plugin>

plugin.xml

final class PcCompletionProvider() extends CompletionContributor {
  // normal person would fetch it from database or fire a request to imdb
  val paramountMovies = List("The Godfather" -> (1972)) 

  override def fillCompletionVariants(
    parameters: CompletionParameters, result: CompletionResultSet
  ): Unit = {
    paramountMovies.foreach { case (title, year) => {
      val elem = LookupElementBuilder.create(title)
      val elemWithDate = elem.withTypeText(year.toString)
      result.addElement(elemWithDate)
    }}
    result.stopHere()
    super.fillCompletionVariants(parameters, result)
  }

Completions

<idea-plugin>
    <id>intellij-next</id>
    <name>IntelliJ Next</name>
    <version>4.0.0</version>

    <idea-version since-build="241" until-build="241.*"/>

    <depends>com.intellij.modules.platform</depends>
    <depends>org.intellij.scala</depends>

    <extensions defaultExtensionNs="com.intellij">
        <completion.contributor
                language="Scala"
                implementationClass="intellij.next.PcCompletionProvider"
                order="first"/>
    </extensions>

</idea-plugin>

plugin.xml

final class PcCompletionProvider() extends CompletionContributor {
  override def fillCompletionVariants(
    parameters: CompletionParameters, result: CompletionResultSet
  ): Unit = {
    // compute real completions
    result.stopHere()
    super.fillCompletionVariants(parameters, result)
  }

Completions

public abstract CompletableFuture[CompletionList] complete(OffsetParams params)
// This is completion api ^^^

final class PcCompletionProvider() extends CompletionContributor {
  override def fillCompletionVariants(
    parameters: CompletionParameters, result: CompletionResultSet
  ): Unit = {
    // compute real completions
    val presentationCompiler: PresentationCompiler = ???
    val offsetParams: OffsetParams = ???
    val completions = presentationCompiler.complete(offsetParams)
    result.stopHere()
    super.fillCompletionVariants(parameters, result)
  }

Completions

public abstract CompletableFuture[CompletionList] complete(OffsetParams params)
// This is completion api ^^^

final class PcCompletionProvider() extends CompletionContributor {
  override def fillCompletionVariants(
    parameters: CompletionParameters, result: CompletionResultSet
  ): Unit = {
    // compute real completions
    val presentationCompiler: PresentationCompiler = ???
    val offsetParams: OffsetParams = ???
    val completions = presentationCompiler.complete(offsetParams)
    completions.get().getItems.foreach { item =>
      val elem = LookupElementBuilder.create(item.getLabel)
      result.addElement(elem)
    }
    result.stopHere()
    super.fillCompletionVariants(parameters, result)
  }

Completions

public abstract CompletableFuture[CompletionList] complete(OffsetParams params)
// This is completion api ^^^

final class PcCompletionProvider() extends CompletionContributor {
  override def fillCompletionVariants(
    parameters: CompletionParameters, result: CompletionResultSet
  ): Unit = {
    // compute real completions
    val presentationCompiler: PresentationCompiler = ???
    val file = parameters.getOriginalFile.getVirtualFile
    val offsetParams: OffsetParams = CompilerOffsetParams(
      file.toUri, parameters.getOriginalFile.getText, parameters.getOffset)
    val completions = presentationCompiler.complete(offsetParams)
    completions.get().getItems.foreach { item =>
      val elem = LookupElementBuilder.create(item.getLabel)
      result.addElement(elem)
    }
    result.stopHere()
    super.fillCompletionVariants(parameters, result)
  }

Completions

public abstract CompletableFuture[CompletionList] complete(OffsetParams params)
// This is completion api ^^^

final class PcCompletionProvider() extends CompletionContributor {
  override def fillCompletionVariants(
    parameters: CompletionParameters, result: CompletionResultSet
  ): Unit = {
    // compute real completions
    val presentationCompiler: PresentationCompiler = 
      Compilers.getPresentationCompiler(module)
    val file = parameters.getOriginalFile.getVirtualFile
    val offsetParams: OffsetParams = CompilerOffsetParams(
      file.toUri, parameters.getOriginalFile.getText, parameters.getOffset)
    val completions = presentationCompiler.complete(offsetParams)
    completions.get().getItems.foreach { item =>
      val elem = LookupElementBuilder.create(item.getLabel)
      result.addElement(elem)
    }
    result.stopHere()
    super.fillCompletionVariants(parameters, result)
  }

Completions

final class Compilers(val project: Project) {
  private val compilers: TrieMap[module.Module, PresentationCompiler] = new TrieMap()

  def startPc(module0: module.Module): PresentationCompiler = {

    val pc = new ScalaPresentationCompiler()
      .newInstance(module0.getName, fullClasspath.asJava, Nil.asJava)
    compilers.addOne(module0 -> pc)
    pc

  }

  def getPresentationCompiler(module0: module.Module): PresentationCompiler =
    compilers.getOrElse(module0, startPc(module0))
}

Compilers

final class Compilers(val project: Project) {
  private val compilers: TrieMap[module.Module, PresentationCompiler] = new TrieMap()

  def startPc(module0: module.Module): PresentationCompiler = {
    val originalClasspath = OrderEnumerator.orderEntries(module0).recursively()
      .withoutSdk().getClassesRoots.map(_.getPresentableUrl)
    val fullClasspath = originalClasspath.toList.map(Paths.get(_))

    val pc = new ScalaPresentationCompiler()
      .newInstance(module0.getName, fullClasspath.asJava, Nil.asJava)
    compilers.addOne(module0 -> pc)
    pc

  }

  def getPresentationCompiler(module0: module.Module): PresentationCompiler =
    compilers.getOrElse(module0, startPc(module0))
}

Compilers

libraryDependencies:
/Users/jrochala/.../scala-library-2.13.12.jar
/Users/jrochala/.../scala3-library_3/3.3.3/scala3-library_3-3.3.3.jar

moduleDependencies:
/Users/jrochala/IdeaProjects/testProject/target/scala-3.3.3/classes

Wait, it was that simple ?

Integrating symbol index

/**
 * The interface for the presentation compile
 * to extract symbol documentation and perform fuzzy symbol search.
 */
public interface SymbolSearch {

    Optional<SymbolDocumentation> documentation(String symbol, ParentSymbols parents);

    List<Location> definition(String symbol, URI sourceUri);

    List<String> definitionSourceToplevels(String symbol, URI sourceUri);

    Result search(String query,
                  String buildTargetIdentifier,
                  SymbolSearchVisitor visitor);
                  
    Result searchMethods(String query,
                  String buildTargetIdentifier,
                  SymbolSearchVisitor visitor);
    
    enum Result {
        COMPLETE,
        INCOMPLETE
    }
}

Symbol search

/**
 * Consumer of symbol search results.
 *
 * Search results can come from two different sources: classpath or workspace.
 * Classpath results are symbols defined in library dependencies while workspace
 * results are symbols that are defined by the user.
 */
public abstract class SymbolSearchVisitor {

   
    abstract public boolean shouldVisitPackage(String pkg);
    
    abstract public int visitClassfile(String pkg, String filename);

    abstract public int visitWorkspaceSymbol(
      Path path, String symbol, SymbolKind kind, Range range
    );

    abstract public boolean isCancelled();

}

Symbol Search Visitor


public abstract class SymbolSearchVisitor {

    abstract public int visitWorkspaceSymbol(
      Path path, String symbol, SymbolKind kind, Range range
    );
}

public abstract class SymbolSeach {
    Result search(String query,
                  String buildTargetIdentifier,
                  SymbolSearchVisitor visitor);
}

Implementation


public abstract class SymbolSearchVisitor {

    abstract public int visitWorkspaceSymbol(
      Path path, String symbol, SymbolKind kind, Range range
    );
}

public abstract class SymbolSeach {
    Result search(String query,
                  String buildTargetIdentifier,
                  SymbolSearchVisitor visitor) =
    
    val symbols: List[PsiElement] = IntelliJCache.getSymbols(query)
    symbols.foreach(visitor.visitWorkspaceSymbol(null, _, null, null))
}

Implementation

SemanticDB

object Test {
  def main(args: Array[String]): Unit = {
    println("hello world")
  }
}


Symbols:
_empty_/Test.
_empty_/Test.main().
_empty_/Test.main().(args)

Occurrences:
[0:7..0:11) <= _empty_/Test.
[1:6..1:10) <= _empty_/Test.main().
[1:11..1:15) <= _empty_/Test.main().(args)
[1:17..1:22) => scala/Array#
[1:23..1:29) => scala/Predef.String#
[1:33..1:37) => scala/Unit#
[2:4..2:11) => scala/Predef.println(+1).

scala/Predef.String#

scala/Predef.String#

IntelliJ Symbol

scala/Predef.String#

PsiElement

scala/Predef.String#

PsiElement

Compiler Internal Symbol

  private def toSemanticdbSymbol(psiClass: PsiClass): String = {
    val suffix = if (psiClass.getLanguage.is(ScalaLanguage.INSTANCE)) "."
    else "#"
    Option(psiClass.getQualifiedName)
      .map(_.replace(".", "/") + suffix)
      .getOrElse("")
  }

* this method will not work in most cases, it is simplification for presentation purposes

final class IntelliJSymbolSearch(
  cache: PsiShortNamesCache, searchScope: GlobalSearchScope) extends SymbolSearch {

  override def search(
    query: String, buildTargetIdentifier: String, visitor: SymbolSearchVisitor
  ): SymbolSearch.Result = {

    if (query.length > 3) {
      val allMatchingClasses = cache.getAllClassNames.filter(_.startsWith(query))
      allMatchingClasses.map { classname =>
        val classes = cache.getClassesByName(classname, searchScope)
        classes.flatMap { psiClass =>
          SemanticDbSymbolCreator.createSemanticDbSymbol(psiClass)
        }.map { semanticDbSymbol =>
          visitor.visitWorkspaceSymbol(
            Paths.get(""), semanticDbSymbol, SymbolKind.Null, new lsp4j.Range()
          )
        }
      }
    }
    SymbolSearch.Result.COMPLETE
  }
}

IntelliJ Symbol Search

* this code handles only class names, it won't provide us with methods, extensions etc

final class Compilers(val project: Project) {
  private val compilers: TrieMap[module.Module, PresentationCompiler] = new TrieMap()

  def startPc(module0: module.Module): PresentationCompiler = {
    val originalClasspath = OrderEnumerator.orderEntries(module0).recursively()
      .withoutSdk().getClassesRoots.map(_.getPresentableUrl)
    val fullClasspath = originalClasspath.toList.map(Paths.get(_))

    val pc = new ScalaPresentationCompiler()
      .newInstance(module0.getName, fullClasspath.asJava, Nil.asJava)
    compilers.addOne(module0 -> pc)
    pc

  }

  def getPresentationCompiler(module0: module.Module): PresentationCompiler =
    compilers.getOrElse(module0, startPc(module0))
}

Compilers

final class Compilers(val project: Project) {
  private val compilers: TrieMap[module.Module, PresentationCompiler] = new TrieMap()

  def startPc(module0: module.Module): PresentationCompiler = {
    val originalClasspath = OrderEnumerator.orderEntries(module0).recursively()
      .withoutSdk().getClassesRoots.map(_.getPresentableUrl)
    val fullClasspath = originalClasspath.toList.map(Paths.get(_))

    val cache = PsiShortNamesCache.getInstance(project)
    val searchScope = GlobalSearchScope
      .moduleWithDependenciesAndLibrariesScope(module0, false)
    val intelliJSymbolSearch = new IntelliJSymbolSearch(cache, searchScope)

    val pc = new ScalaPresentationCompiler()
      .withSearch(intelliJSymbolSearch)
      .newInstance(module0.getName, fullClasspath.asJava, Nil.asJava)
    compilers.addOne(module0 -> pc)
    pc

  }

  def getPresentationCompiler(module0: module.Module): PresentationCompiler =
    compilers.getOrElse(module0, startPc(module0))
}

Compilers

Thank you

https://slides.com/rochala/the-best-ide-in-your-favourite-ide

The best IDE is your favourite IDE

By rochala

The best IDE is your favourite IDE

The best IDE is

your favourite IDE

The best IDE is in your favourite IDE

The best IDE is your favourite IDE

More from rochala