-
Notifications
You must be signed in to change notification settings - Fork 93
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Semantic Tokens extension point (#1683)
Offers extension point to contribute semantic tokens into the JDT Java editor for syntax highlighting. #1594
- Loading branch information
Showing
15 changed files
with
743 additions
and
95 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
137 changes: 137 additions & 0 deletions
137
....tests/src/org/eclipse/jdt/text/tests/semantictokens/SampleSqlSemanticTokensProvider.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,137 @@ | ||
| /******************************************************************************* | ||
| * Copyright (c) 2024 Broadcom Inc. and others. | ||
| * This program and the accompanying materials are made | ||
| * available under the terms of the Eclipse Public License 2.0 | ||
| * which is available at https://www.eclipse.org/legal/epl-2.0/ | ||
| * | ||
| * SPDX-License-Identifier: EPL-2.0 | ||
| * | ||
| * Contributors: | ||
| * Alex Boyko (Broadcom Inc.) - Initial implementation | ||
| *******************************************************************************/ | ||
| package org.eclipse.jdt.text.tests.semantictokens; | ||
|
|
||
| import java.text.NumberFormat; | ||
| import java.text.ParseException; | ||
| import java.util.ArrayList; | ||
| import java.util.Collection; | ||
| import java.util.Collections; | ||
| import java.util.List; | ||
| import java.util.regex.Matcher; | ||
| import java.util.regex.Pattern; | ||
|
|
||
| import org.eclipse.jdt.core.dom.ASTVisitor; | ||
| import org.eclipse.jdt.core.dom.CompilationUnit; | ||
| import org.eclipse.jdt.core.dom.Expression; | ||
| import org.eclipse.jdt.core.dom.StringLiteral; | ||
| import org.eclipse.jdt.core.dom.TextBlock; | ||
|
|
||
| import org.eclipse.jdt.ui.text.java.ISemanticTokensProvider; | ||
|
|
||
| /** | ||
| * Semantic tokens are computed for string literals or text blocks starting with "SQL:" prefix. | ||
| * <ul> | ||
| * <li>SELECT, WHERE, IN, FROM are KEYWORD</li> | ||
| * <li>*, <, >, ==, != etc are OPEARATOR</li> | ||
| * <li>Words starting from capital are CLASS</li> | ||
| * <li>Numbers are NUMBER</li> | ||
| * <li>all other lower case starting words are LOCAL_VARIABLE</li> | ||
| * </ul> | ||
| */ | ||
| public class SampleSqlSemanticTokensProvider implements ISemanticTokensProvider { | ||
|
|
||
| private static final String SQL_PREFIX = "SQL:"; | ||
|
|
||
| @Override | ||
| public Collection<SemanticToken> computeSemanticTokens(CompilationUnit ast) { | ||
| List<SemanticToken> tokens = new ArrayList<>(); | ||
| ast.accept(new ASTVisitor() { | ||
|
|
||
| @Override | ||
| public boolean visit(StringLiteral node) { | ||
| tokens.addAll(reconileEmbeddedExpression(node)); | ||
| return super.visit(node); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean visit(TextBlock node) { | ||
| tokens.addAll(reconileEmbeddedExpression(node)); | ||
| return super.visit(node); | ||
| } | ||
|
|
||
| }); | ||
| return tokens; | ||
| } | ||
|
|
||
| private List<SemanticToken> reconileEmbeddedExpression(Expression valueExp) { | ||
| String text = null; | ||
| int offset = 0; | ||
| if (valueExp instanceof StringLiteral sl && sl.getLiteralValue().startsWith(SQL_PREFIX)) { | ||
| text = sl.getEscapedValue(); | ||
| int skip = 1 + SQL_PREFIX.length(); | ||
| text = text.substring(skip, text.length() - 1); | ||
| offset = sl.getStartPosition() + skip; // +1 to skip over opening " and over "SQL:" | ||
| } else if (valueExp instanceof TextBlock tb && tb.getLiteralValue().startsWith(SQL_PREFIX)) { | ||
| text = tb.getEscapedValue(); | ||
| int skip = 3 + SQL_PREFIX.length(); | ||
| text = text.substring(skip, text.length() - 3); | ||
| offset = tb.getStartPosition() + skip; // +3 to skip over opening """ and over "SQL:" | ||
| } | ||
| return compute(text, offset); | ||
| } | ||
|
|
||
| private List<SemanticToken> compute(String text, int offset) { | ||
| if (text == null) { | ||
| return Collections.emptyList(); | ||
| } | ||
| List<SemanticToken> tokens = new ArrayList<>(); | ||
| Matcher matcher= Pattern.compile("[\\w*=><!]+").matcher(text); | ||
| while (matcher.find()) { | ||
| String token = matcher.group(); | ||
| if (!token.isBlank()) { | ||
| int start = matcher.start(); | ||
| int end = matcher.end(); | ||
| tokens.add(new SemanticToken(start + offset, end - start, getTokenType(token))); | ||
| } | ||
| } | ||
| return tokens; | ||
| } | ||
|
|
||
| private TokenType getTokenType(String token) { | ||
| try { | ||
| NumberFormat.getInstance().parse(token); | ||
| return TokenType.NUMBER; | ||
| } catch (ParseException e) { | ||
| switch (token) { | ||
| case "SELECT": | ||
| case "WHERE": | ||
| case "WHEN": | ||
| case "ALL": | ||
| case "BY": | ||
| case "ORDER": | ||
| case "LIKE": | ||
| case "IN": | ||
| case "FROM": | ||
| case "NOT": | ||
| return TokenType.KEYWORD; | ||
| case "*": | ||
| case "(": | ||
| case "-": | ||
| case ">": | ||
| case "<": | ||
| case ">=": | ||
| case "<=": | ||
| case "==": | ||
| case "!=": | ||
| return TokenType.OPERATOR; | ||
| default: | ||
| if (token.length() > 0 && Character.isUpperCase(token.charAt(0))) { | ||
| return TokenType.CLASS; | ||
| } | ||
| return TokenType.LOCAL_VARIABLE; | ||
| } | ||
| } | ||
| } | ||
|
|
||
|
|
||
| } |
50 changes: 50 additions & 0 deletions
50
....text.tests/src/org/eclipse/jdt/text/tests/semantictokens/SemanticTokensProviderTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| /******************************************************************************* | ||
| * Copyright (c) 2024 Broadcom Inc. and others. | ||
| * This program and the accompanying materials are made | ||
| * available under the terms of the Eclipse Public License 2.0 | ||
| * which is available at https://www.eclipse.org/legal/epl-2.0/ | ||
| * | ||
| * SPDX-License-Identifier: EPL-2.0 | ||
| * | ||
| * Contributors: | ||
| * Alex Boyko (Broadcom Inc.) - Initial implementation | ||
| *******************************************************************************/ | ||
| package org.eclipse.jdt.text.tests.semantictokens; | ||
|
|
||
| import org.junit.Rule; | ||
| import org.junit.Test; | ||
|
|
||
| import org.eclipse.jdt.text.tests.AbstractSemanticHighlightingTest; | ||
|
|
||
| import org.eclipse.jface.text.Position; | ||
|
|
||
| import org.eclipse.jdt.internal.ui.javaeditor.SemanticHighlightings; | ||
|
|
||
| public class SemanticTokensProviderTest extends AbstractSemanticHighlightingTest { | ||
|
|
||
| @Rule | ||
| public SemanticHighlightingTestSetup shts= new SemanticHighlightingTestSetup( "/SHTest/src/STTest.java"); | ||
|
|
||
| @Test | ||
| public void contributedHighlighting() throws Exception { | ||
| setUpSemanticHighlighting(SemanticHighlightings.CLASS); | ||
| setUpSemanticHighlighting(SemanticHighlightings.NUMBER); | ||
| setUpSemanticHighlighting(SemanticHighlightings.LOCAL_VARIABLE); | ||
| Position[] actual= getSemanticHighlightingPositions(); | ||
| Position[] expected= new Position[] { | ||
| createPosition(0, 6, 1), | ||
| createPosition(1, 1, 6), | ||
| createPosition(1, 20, 6), | ||
| createPosition(1, 27, 1), | ||
| createPosition(1, 29, 4), | ||
| createPosition(1, 34, 1), | ||
| createPosition(1, 36, 5), | ||
| createPosition(1, 42, 1), | ||
| createPosition(1, 44, 2), | ||
| createPosition(1, 47, 3), | ||
| }; | ||
| assertEqualPositions(expected, actual); | ||
| } | ||
|
|
||
|
|
||
| } |
3 changes: 3 additions & 0 deletions
3
org.eclipse.jdt.text.tests/testResources/semanticHighlightingTest1/STTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| class A { | ||
| String sql = "SQL: SELECT * FROM T WHERE a == 567"; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.