Perfected coloring

angel_vscode/snippets/jael.json

@@ -3,7 +3,7 @@
     "block": {
       "prefix": "block",
       "description": "Insert a named <block> tag.",
-      "body": ["<block name=\"$1\">", "   $2", "<block>"]
+      "body": ["<block name=\"$1\">", "   $2", "</block>"]
     },
     "comment": {
       "prefix": "comment",
@@ -13,22 +13,22 @@
     "declare": {
       "prefix": "declare",
       "description": "Insert a <declare> tag.",
-      "body": ["<declare $1=$2>", "   $3", "<declare>"]
+      "body": ["<declare $1=$2>", "   $3", "</declare>"]
     },
     "element": {
       "prefix": "element",
       "description": "Insert a custom <element> tag.",
-      "body": ["<element name=\"$1\">", "   $2", "<element>"]
+      "body": ["<element name=\"$1\">", "   $2", "</element>"]
     },
     "extend": {
       "prefix": "extend",
       "description": "Insert an <extend> tag.",
-      "body": ["<extend src=\"$1\">", "   $2", "<extend>"]
+      "body": ["<extend src=\"$1\">", "   $2", "</extend>"]
     },
     "for-each": {
       "prefix": "for-each",
       "description": "Insert a <for-each> tag.",
-      "body": ["<${1:div} for-each=$2 as=\"$3\">", "   $4", "<$1>"]
+      "body": ["<${1:div} for-each=$2 as=\"$3\">", "   $4", "</$1>"]
     },
     "include": {
       "prefix": "include",
@@ -46,7 +46,7 @@
         "   <default>",
         "       $4",
         "   </default>",
-        "<switch>"
+        "</switch>"
       ]
     }
   }

angel_vscode/syntaxes/jael.json

@@ -6,6 +6,17 @@
     {
       "include": "#expressions"
     },
+    {
+      "match": "!(DOCTYPE|doctype)([>]*)",
+      "captures": {
+        "1": {
+          "name": "keyword.doctype.jael"
+        },
+        "2": {
+          "name": "support.doctype-value.jael"
+        }
+      }
+    },
     {
       "match": "[:,\\.]",
       "name": "punctuation.jael"
@@ -30,7 +41,7 @@
       "name": "keyword.control.jael"
     },
     {
-      "match": "<\\s*/?\\s*([A-Za-z_][A-Za-z0-9_]*)\\b",
+      "match": "<\\s*/?\\s*([A-Za-z_][A-Za-z0-9_-]*)\\b",
       "captures": {
         "1": {
           "name": "keyword.tag.jael"
@@ -42,7 +53,7 @@
       "name": "storage.argument.jael"
     },
     {
-      "match": "\\b(@?[A-Za-z_][A-Za-z0-9_]*)=",
+      "match": "\\b(@?[A-Za-z_][A-Za-z0-9_]*)=?",
       "captures": {
         "1": {
           "name": "storage.name.jael"

jael_language_server/bin/jael_language_server.dart

@@ -23,6 +23,19 @@ main(List<String> args) async {
       return;
     } else {
       var jaelServer = new JaelLanguageServer();
+
+      jaelServer.logger.onRecord.listen((rec) async {
+        // TODO: Remove this
+        var f = new File(
+            '/Users/thosakwe/Source/Angel/vscode/jael_language_server/.dart_tool/log.txt');
+        await f.create(recursive: true);
+        var sink = await f.openWrite(mode: FileMode.append);
+        sink.writeln(rec);
+        if (rec.error != null) sink.writeln(rec.error);
+        if (rec.stackTrace != null) sink.writeln(rec.stackTrace);
+        await sink.close();
+      });
+
       var stdio = new StdIOLanguageServer.start(jaelServer);
       await stdio.onDone;
     }

jael_language_server/lib/src/analyzer.dart

@@ -0,0 +1,106 @@
+import 'package:jael/jael.dart';
+import 'package:logging/logging.dart';
+import 'package:symbol_table/symbol_table.dart';
+import 'object.dart';
+
+class Analyzer extends Parser {
+  final Logger logger;
+  Analyzer(Scanner scanner, this.logger) : super(scanner);
+
+  final errors = <JaelError>[];
+  var _scope = new SymbolTable<JaelObject>();
+  var allDefinitions = <Variable<JaelObject>>[];
+
+  SymbolTable<JaelObject> get parentScope =>
+      _scope.isRoot ? _scope : _scope.parent;
+
+  SymbolTable<JaelObject> get scope => _scope;
+
+  bool ensureAttributeIsPresent(Element element, String name) {
+    if (element.getAttribute(name) == null) {
+      errors.add(new JaelError(JaelErrorSeverity.error,
+          'Missing required attribute `$name`.', element.span));
+      return false;
+    }
+    return true;
+  }
+
+  @override
+  Element parseElement() {
+    try {
+      _scope = _scope.createChild();
+      var element = super.parseElement();
+      if (element == null) {
+        // TODO: ???
+        if (next(TokenType.lt)) {
+          var tagName = parseIdentifier();
+          if (tagName != null) {
+            errors.add(
+                new JaelError(JaelErrorSeverity.error, "Hmm", tagName.span));
+          }
+        }
+        return null;
+      }
+
+      logger.info('!!! ${element.tagName.name}');
+
+      // Check if any custom element exists.
+      _scope
+          .resolve(element.tagName.name)
+          ?.value
+          ?.usages
+          ?.add(new SymbolUsage(SymbolUsageType.read, element.span));
+
+      // Validate attrs
+      // TODO: if, for-each
+
+      // Validate the tag itself
+      if (element is RegularElement) {
+        if (element.tagName.name == 'block') {
+          ensureAttributeIsPresent(element, 'name');
+          logger.info('Found <block> at ${element.span.start.toolString}');
+        } else if (element.tagName.name == 'case') {
+          ensureAttributeIsPresent(element, 'value');
+          logger.info('Found <case> at ${element.span.start.toolString}');
+        } else if (element.tagName.name == 'element') {
+          if (ensureAttributeIsPresent(element, 'name')) {
+            var nameCtx = element.getAttribute('name').value;
+
+            if (nameCtx is! StringLiteral) {
+              errors.add(new JaelError(
+                  JaelErrorSeverity.warning,
+                  "`name` attribute should be a constant string literal.",
+                  nameCtx.span));
+            } else {
+              var name = (nameCtx as StringLiteral).value;
+              logger.info(
+                  'Found custom element $name at ${element.span.start.toolString}');
+              try {
+                var symbol = parentScope.create(name,
+                    value: new JaelCustomElement(name, element.tagName.span),
+                    constant: true);
+                allDefinitions.add(symbol);
+              } on StateError catch (e) {
+                errors.add(new JaelError(
+                    JaelErrorSeverity.error, e.message, element.tagName.span));
+              }
+            }
+          }
+        } else if (element.tagName.name == 'extend') {
+          ensureAttributeIsPresent(element, 'src');
+          logger.info('Found <extend> at ${element.span.start.toolString}');
+        }
+      } else if (element is SelfClosingElement) {
+        if (element.tagName.name == 'include') {
+          logger.info('Found <include> at ${element.span.start.toolString}');
+          ensureAttributeIsPresent(element, 'src');
+        }
+      }
+
+      return element;
+    } finally {
+      _scope = _scope.parent;
+      return null;
+    }
+  }
+}

jael_language_server/lib/src/object.dart

@@ -0,0 +1,27 @@
+import 'dart:collection';
+
+import 'package:source_span/source_span.dart';
+
+abstract class JaelObject {
+  final FileSpan span;
+  final usages = <SymbolUsage>[];
+  String get name;
+
+  JaelObject(this.span);
+}
+
+class JaelCustomElement extends JaelObject {
+  final String name;
+  final attributes = new SplayTreeSet<String>();
+
+  JaelCustomElement(this.name, FileSpan span) : super(span);
+}
+
+class SymbolUsage {
+  final SymbolUsageType type;
+  final FileSpan span;
+
+  SymbolUsage(this.type, this.span);
+}
+
+enum SymbolUsageType { definition, read }

jael_language_server/lib/src/server.dart

@@ -4,16 +4,22 @@ import 'package:dart_language_server/src/protocol/language_server/messages.dart'
 import 'package:file/file.dart';
 import 'package:file/local.dart';
 import 'package:file/memory.dart';
+import 'package:jael/jael.dart';
 import 'package:logging/logging.dart';
 import 'package:path/path.dart' as p;
+import 'package:source_span/source_span.dart';
+import 'package:symbol_table/symbol_table.dart';
+import 'analyzer.dart';
+import 'object.dart';
 
 class JaelLanguageServer extends LanguageServer {
-  var _diagnostics = new StreamController<Diagnostics>();
+  var _diagnostics = new StreamController<Diagnostics>(sync: true);
   var _done = new Completer();
   var _memFs = new MemoryFileSystem();
   var _localFs = const LocalFileSystem();
   Directory _localRootDir;
-  var _logger = new Logger('jael');
+  var logger = new Logger('jael');
+  Uri _rootUri;
   var _workspaceEdits = new StreamController<ApplyWorkspaceEditParams>();
 
   @override
@@ -37,7 +43,7 @@ class JaelLanguageServer extends LanguageServer {
   Future<ServerCapabilities> initialize(int clientPid, String rootUri,
       ClientCapabilities clientCapabilities, String trace) async {
     // Find our real root dir.
-    _localRootDir = _localFs.directory(rootUri);
+    _localRootDir = _localFs.directory(_rootUri = Uri.parse(rootUri));
 
     // Copy all real files that end in *.jael (and *.jl for legacy) into the in-memory filesystem.
     await for (var entity in _localRootDir.list(recursive: true)) {
@@ -47,12 +53,33 @@ class JaelLanguageServer extends LanguageServer {
         var file = _memFs.file(relativePath);
         await file.create(recursive: true);
         await entity.openRead().pipe(file.openWrite(mode: FileMode.write));
-        _logger.info('Found Jael file ${file.path}');
+        logger.info('Found Jael file ${file.path}');
+
+        // Analyze it
+        var documentId = new TextDocumentIdentifier((b) {
+          b..uri = _rootUri.replace(path: relativePath).toString();
+        });
+
+        await analyzerForId(documentId);
       }
     }
 
     return new ServerCapabilities((b) {
       b
+      ..codeActionProvider = false
+        ..completionProvider = new CompletionOptions((b) {
+          b
+            ..resolveProvider = true
+            ..triggerCharacters =
+                'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdeghijklmnopqrstuvxwyz'
+                    .codeUnits
+                    .map((c) => new String.fromCharCode(c))
+                    .toList();
+        })
+        ..definitionProvider = true
+        ..documentHighlightProvider = true
+        ..documentRangeFormattingProvider = false
+        ..documentOnTypeFormattingProvider = null
         ..documentSymbolProvider = true
         ..documentFormattingProvider = true
         ..hoverProvider = true
@@ -62,78 +89,331 @@ class JaelLanguageServer extends LanguageServer {
         ..signatureHelpProvider = new SignatureHelpOptions((b) {})
         ..textDocumentSync = new TextDocumentSyncOptions((b) {
           b
+            ..openClose = true
             ..change = TextDocumentSyncKind.incremental
             ..save = new SaveOptions((b) {
-              b..includeText = true;
+              b..includeText = false;
             })
-            ..willSave = false;
+            ..willSave = false
+            ..willSaveWaitUntil = false;
+        })
+        ..workspaceSymbolProvider = true;
+    });
+  }
+
+  Future<File> fileForId(TextDocumentIdentifier documentId) async {
+    var uri = Uri.parse(documentId.uri);
+    var relativePath = p.relative(uri.path, from: _rootUri.path);
+    var file = _memFs.file(relativePath);
+
+    if (!await file.exists()) {
+      await file.create(recursive: true);
+      await _localFs.file(uri).openRead().pipe(file.openWrite());
+      logger.info('Opened Jael file ${file.path}');
+    }
+
+    return file;
+  }
+
+  Future<Scanner> scannerForId(TextDocumentIdentifier documentId) async {
+    var file = await fileForId(documentId);
+    return scan(await file.readAsString(), sourceUrl: file.uri);
+  }
+
+  Future<Analyzer> analyzerForId(TextDocumentIdentifier documentId) async {
+    var scanner = await scannerForId(documentId);
+    var analyzer = new Analyzer(scanner, logger)..errors.addAll(scanner.errors);
+    analyzer.parseDocument();
+    logger.info(
+        'Done ${documentId.uri} ${await (await fileForId(documentId)).readAsString()}');
+    logger.info(analyzer.errors);
+    emitDiagnostics(documentId.uri, analyzer.errors.map(toDiagnostic).toList());
+    return analyzer;
+  }
+
+  Diagnostic toDiagnostic(JaelError e) {
+    return new Diagnostic((b) {
+      b
+        ..message = e.message
+        ..range = toRange(e.span)
+        ..severity = toSeverity(e.severity)
+        ..source = e.span.start.sourceUrl.toString();
+    });
+  }
+
+  int toSeverity(JaelErrorSeverity s) {
+    switch (s) {
+      case JaelErrorSeverity.warning:
+        return DiagnosticSeverity.warning;
+      default:
+        return DiagnosticSeverity.error;
+    }
+  }
+
+  Range toRange(FileSpan span) {
+    return new Range((b) {
+      b
+        ..start = toPosition(span.start)
+        ..end = toPosition(span.end);
+    });
+  }
+
+  Range emptyRange() {
+    return new Range((b) => b
+      ..start = b.end = new Position((b) {
+        b
+          ..character = 1
+          ..line = 0;
+      }));
+  }
+
+  Position toPosition(SourceLocation location) {
+    return new Position((b) {
+      b
+        ..line = location.line
+        ..character = location.column;
+    });
+  }
+
+  Location toLocation(String uri, FileSpan span) {
+    return new Location((b) {
+      b
+        ..range = toRange(span)
+        ..uri = uri;
+    });
+  }
+
+  bool isReachable(JaelObject obj, Position position) {
+    return obj.span.start.line <= position.line &&
+        obj.span.start.column <= position.character;
+  }
+
+  CompletionItem toCompletion(Variable<JaelObject> symbol) {
+    var value = symbol.value;
+
+    if (value is JaelCustomElement) {
+      var name = value.name;
+      return new CompletionItem((b) {
+        b
+          ..kind = CompletionItemKind.classKind
+          ..label = symbol.name
+          ..textEdit = new TextEdit((b) {
+            b
+              ..range = emptyRange()
+              ..newText = '<$name\$1>\n    \$2\n</name>';
           });
       });
     }
 
+    return null;
+  }
+
+  void emitDiagnostics(String uri, Iterable<Diagnostic> diagnostics) {
+    if (diagnostics.isEmpty) return;
+    _diagnostics.add(new Diagnostics((b) {
+      b
+        ..diagnostics = diagnostics.toList()
+        ..uri = uri.toString();
+    }));
+  }
+
+  @override
+  Future textDocumentDidOpen(TextDocumentItem document) async {
+    await analyzerForId(
+        new TextDocumentIdentifier((b) => b..uri = document.uri));
+  }
+
+  @override
+  Future textDocumentDidChange(VersionedTextDocumentIdentifier documentId,
+      List<TextDocumentContentChangeEvent> changes) async {
+    var id = new TextDocumentIdentifier((b) => b..uri = documentId.uri);
+    var file = await fileForId(id);
+
+    for (var change in changes) {
+      if (change.range != null) {
+        await file.writeAsString(change.text);
+      } else {
+        var contents = await file.readAsString();
+
+        int findIndex(Position position) {
+          var lines = contents.split('\n');
+
+          // Sum the length of the previous lines.
+          int lineLength = lines
+              .take(position.line - 1)
+              .map((s) => s.length)
+              .reduce((a, b) => a + b);
+          return lineLength + position.character - 1;
+        }
+
+        if (change.range == null) {
+          contents = change.text;
+        } else {
+          var start = findIndex(change.range.start),
+              end = findIndex(change.range.end);
+          contents = contents.replaceRange(start, end, change.text);
+        }
+
+        await file.writeAsString(contents);
+      }
+    }
+
+    await analyzerForId(id);
+  }
+
   @override
   Future<List> textDocumentCodeAction(TextDocumentIdentifier documentId,
-      Range range, CodeActionContext context) {
+      Range range, CodeActionContext context) async {
     // TODO: implement textDocumentCodeAction
+    return [];
   }
 
   @override
   Future<CompletionList> textDocumentCompletion(
-      TextDocumentIdentifier documentId, Position position) {
-    // TODO: implement textDocumentCompletion
+      TextDocumentIdentifier documentId, Position position) async {
+    var analyzer = await analyzerForId(documentId);
+    var symbols = analyzer.scope.allVariables;
+    var reachable = symbols.where((s) => isReachable(s.value, position));
+    return new CompletionList((b) {
+      b
+        ..isIncomplete = false
+        ..items = reachable.map(toCompletion).toList();
+    });
+  }
+
+  Future<JaelObject> currentSymbol(
+      TextDocumentIdentifier documentId, Position position) async {
+    var analyzer = await analyzerForId(documentId);
+    var symbols = analyzer.allDefinitions; // analyzer.scope.allVariables;
+    logger.info('Current synmbols: ${symbols.map((v) => v.name)}');
+
+    for (var s in symbols) {
+      var v = s.value;
+
+      if (position.line == v.span.start.line &&
+          position.character == v.span.start.column) {
+        logger.info('Success ${s.name}');
+        return v;
+      } else {
+        logger.info(
+            'Nope ${s.name} (${v.span.start.toolString} vs ${position.line}:${position.character})');
+      }
+    }
+
+    return null;
   }
 
   @override
   Future<Location> textDocumentDefinition(
-      TextDocumentIdentifier documentId, Position position) {
-    // TODO: implement textDocumentDefinition
+      TextDocumentIdentifier documentId, Position position) async {
+    var symbol = await currentSymbol(documentId, position);
+    if (symbol != null) {
+      return toLocation(documentId.uri, symbol.span);
+    }
+    return null;
   }
 
   @override
   Future<List<DocumentHighlight>> textDocumentHighlight(
-      TextDocumentIdentifier documentId, Position position) {
-    // TODO: implement textDocumentHighlight
+      TextDocumentIdentifier documentId, Position position) async {
+    var symbol = await currentSymbol(documentId, position);
+    if (symbol != null) {
+      return symbol.usages.map((u) {
+        return new DocumentHighlight((b) {
+          b
+            ..range = toRange(u.span)
+            ..kind = u.type == SymbolUsageType.definition
+                ? DocumentHighlightKind.write
+                : DocumentHighlightKind.read;
+        });
+      }).toList();
+    }
+    return [];
   }
 
   @override
-  Future textDocumentHover(
-      TextDocumentIdentifier documentId, Position position) {
-    // TODO: implement textDocumentHover
+  Future<Hover> textDocumentHover(
+      TextDocumentIdentifier documentId, Position position) async {
+    var symbol = await currentSymbol(documentId, position);
+    if (symbol != null) {
+      return new Hover((b) {
+        b..range = toRange(symbol.span);
+      });
+    }
+    return null;
   }
 
   @override
   Future<List<Location>> textDocumentImplementation(
-      TextDocumentIdentifier documentId, Position position) {
-    // TODO: implement textDocumentImplementation
+      TextDocumentIdentifier documentId, Position position) async {
+    var defn = await textDocumentDefinition(documentId, position);
+    return defn == null ? [] : [defn];
   }
 
   @override
   Future<List<Location>> textDocumentReferences(
       TextDocumentIdentifier documentId,
       Position position,
-      ReferenceContext context) {
-    // TODO: implement textDocumentReferences
+      ReferenceContext context) async {
+    var symbol = await currentSymbol(documentId, position);
+    if (symbol != null) {
+      return symbol.usages.map((u) {
+        return toLocation(documentId.uri, u.span);
+      }).toList();
+    }
+
+    return [];
   }
 
   @override
-  Future<WorkspaceEdit> textDocumentRename(
-      TextDocumentIdentifier documentId, Position position, String newName) {
-    // TODO: implement textDocumentRename
+  Future<WorkspaceEdit> textDocumentRename(TextDocumentIdentifier documentId,
+      Position position, String newName) async {
+    var symbol = await currentSymbol(documentId, position);
+    if (symbol != null) {
+      return new WorkspaceEdit((b) {
+        b
+          ..changes = {
+            symbol.name: symbol.usages.map((u) {
+              return new TextEdit((b) {
+                b
+                  ..range = toRange(u.span)
+                  ..newText = newName;
+              });
+            }).toList()
+          };
+      });
+    }
+    return new WorkspaceEdit((b) {
+      b..changes = {};
+    });
   }
 
   @override
   Future<List<SymbolInformation>> textDocumentSymbols(
-      TextDocumentIdentifier documentId) {
-    // TODO: implement textDocumentSymbols
+      TextDocumentIdentifier documentId) async {
+    var analyzer = await analyzerForId(documentId);
+    return analyzer.allDefinitions.map((symbol) {
+      return new SymbolInformation((b) {
+        b
+          ..kind = SymbolKind.classSymbol
+          ..name = symbol.name
+          ..location = toLocation(documentId.uri, symbol.value.span);
+      });
+    }).toList();
   }
 
   @override
-  Future<void> workspaceExecuteCommand(String command, List arguments) {
+  Future<void> workspaceExecuteCommand(String command, List arguments) async {
     // TODO: implement workspaceExecuteCommand
   }
 
   @override
-  Future<List<SymbolInformation>> workspaceSymbol(String query) {
+  Future<List<SymbolInformation>> workspaceSymbol(String query) async {
     // TODO: implement workspaceSymbol
+    return [];
   }
 }
+
+abstract class DiagnosticSeverity {
+  static const int error = 0, warning = 1, information = 2, hint = 3;
+}

jael_language_server/pubspec.yaml

@@ -14,5 +14,7 @@ dependencies:
   jael_preprocessor: ^2.0.0
   logging: ^0.11.3
   path: ^1.0.0
+  symbol_table: ^2.0.0
+  source_span: ^1.0.0
 executables:
-  jael_language_server: main
+  jael_language_server: jael_language_server