diff --git a/packages/code_buffer/.travis.yml b/packages/code_buffer/.travis.yml new file mode 100644 index 0000000..de2210c --- /dev/null +++ b/packages/code_buffer/.travis.yml @@ -0,0 +1 @@ +language: dart \ No newline at end of file diff --git a/packages/code_buffer/AUTHORS.md b/packages/code_buffer/AUTHORS.md new file mode 100644 index 0000000..ac95ab5 --- /dev/null +++ b/packages/code_buffer/AUTHORS.md @@ -0,0 +1,12 @@ +Primary Authors +=============== + +* __[Thomas Hii](dukefirehawk.apps@gmail.com)__ + + Thomas is the current maintainer of the code base. He has refactored and migrated the + code base to support NNBD. + +* __[Tobe O](thosakwe@gmail.com)__ + + Tobe has written much of the original code prior to NNBD migration. He has moved on and + is no longer involved with the project. diff --git a/packages/code_buffer/CHANGELOG.md b/packages/code_buffer/CHANGELOG.md new file mode 100644 index 0000000..7d179c3 --- /dev/null +++ b/packages/code_buffer/CHANGELOG.md @@ -0,0 +1,26 @@ +# Change Log + +## 3.0.0 + +* Upgraded from `pendantic` to `lints` linter +* Published as `belatuk_code_buffer` package + +## 2.0.3 + +* Resolved static analysis warnings + +## 2.0.2 + +* Updated README + +## 2.0.1 + +* Fixed invalid homepage url in pubspec.yaml + +## 2.0.0 + +* Migrated to support Dart SDK 2.12.x NNBD + +## 1.0.1 + +* Added `CodeBuffer.noWhitespace()`. diff --git a/packages/code_buffer/LICENSE b/packages/code_buffer/LICENSE new file mode 100644 index 0000000..e37a346 --- /dev/null +++ b/packages/code_buffer/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2021, dukefirehawk.com +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/packages/code_buffer/README.md b/packages/code_buffer/README.md new file mode 100644 index 0000000..f79dfb1 --- /dev/null +++ b/packages/code_buffer/README.md @@ -0,0 +1,69 @@ +# Belatuk Code Buffer + +[![version](https://img.shields.io/badge/pub-v3.0.0-brightgreen)](https://pub.dartlang.org/packages/belatuk_code_buffer) +[![Null Safety](https://img.shields.io/badge/null-safety-brightgreen)](https://dart.dev/null-safety) +[![License](https://img.shields.io/github/license/dart-backend/belatuk-common-utilities)](https://github.com/dart-backend/belatuk-common-utilities/packages/code_buffer/LICENSE) + +**Replacement of `package:code_buffer` with breaking changes to support NNBD.** + +An advanced StringBuffer geared toward generating code, and source maps. + +## Installation + +In your `pubspec.yaml`: + +```yaml +dependencies: + belatuk_code_buffer: ^3.0.0 +``` + +## Usage + +Use a `CodeBuffer` just like any regular `StringBuffer`: + +```dart +String someFunc() { + var buf = CodeBuffer(); + buf + ..write('hello ') + ..writeln('world!'); + return buf.toString(); +} +``` + +However, a `CodeBuffer` supports indentation. + +```dart +void someOtherFunc() { + var buf = CodeBuffer(); + // Custom options... + var buf = CodeBuffer(newline: '\r\n', space: '\t', trailingNewline: true); + + // Any following lines will have an incremented indentation level... + buf.indent(); + + // And vice-versa: + buf.outdent(); +} +``` + +`CodeBuffer` instances keep track of every `SourceSpan` they create. +This makes them useful for codegen tools, or to-JS compilers. + +```dart +void someFunc(CodeBuffer buf) { + buf.write('hello'); + expect(buf.lastLine.text, 'hello'); + + buf.writeln('world'); + expect(buf.lastLine.lastSpan.start.column, 5); +} +``` + +You can copy a `CodeBuffer` into another, heeding indentation rules: + +```dart +void yetAnotherFunc(CodeBuffer a, CodeBuffer b) { + b.copyInto(a); +} +``` diff --git a/packages/code_buffer/analysis_options.yaml b/packages/code_buffer/analysis_options.yaml new file mode 100644 index 0000000..ea2c9e9 --- /dev/null +++ b/packages/code_buffer/analysis_options.yaml @@ -0,0 +1 @@ +include: package:lints/recommended.yaml \ No newline at end of file diff --git a/packages/code_buffer/example/main.dart b/packages/code_buffer/example/main.dart new file mode 100644 index 0000000..d3d02ba --- /dev/null +++ b/packages/code_buffer/example/main.dart @@ -0,0 +1,46 @@ +import 'package:belatuk_code_buffer/belatuk_code_buffer.dart'; +import 'package:test/test.dart'; + +/// Use a `CodeBuffer` just like any regular `StringBuffer`: +String someFunc() { + var buf = CodeBuffer(); + buf + ..write('hello ') + ..writeln('world!'); + return buf.toString(); +} + +/// However, a `CodeBuffer` supports indentation. +void someOtherFunc() { + var buf = CodeBuffer(); + + // Custom options... + // ignore: unused_local_variable + var customBuf = + CodeBuffer(newline: '\r\n', space: '\t', trailingNewline: true); + + // Without whitespace.. + // ignore: unused_local_variable + var minifyingBuf = CodeBuffer.noWhitespace(); + + // Any following lines will have an incremented indentation level... + buf.indent(); + + // And vice-versa: + buf.outdent(); +} + +/// `CodeBuffer` instances keep track of every `SourceSpan` they create. +//This makes them useful for codegen tools, or to-JS compilers. +void yetAnotherOtherFunc(CodeBuffer buf) { + buf.write('hello'); + expect(buf.lastLine!.text, 'hello'); + + buf.writeln('world'); + expect(buf.lastLine!.lastSpan!.start.column, 5); +} + +/// You can copy a `CodeBuffer` into another, heeding indentation rules: +void yetEvenAnotherFunc(CodeBuffer a, CodeBuffer b) { + b.copyInto(a); +} diff --git a/packages/code_buffer/lib/belatuk_code_buffer.dart b/packages/code_buffer/lib/belatuk_code_buffer.dart new file mode 100644 index 0000000..2e377c5 --- /dev/null +++ b/packages/code_buffer/lib/belatuk_code_buffer.dart @@ -0,0 +1,231 @@ +import 'package:source_span/source_span.dart'; + +/// An advanced StringBuffer geared toward generating code, and source maps. +class CodeBuffer implements StringBuffer { + /// The character sequence used to represent a line break. + final String newline; + + /// The character sequence used to represent a space/tab. + final String space; + + /// The source URL to be applied to all generated [SourceSpan] instances. + final dynamic sourceUrl; + + /// If `true` (default: `false`), then an additional [newline] will be inserted at the end of the generated string. + final bool trailingNewline; + + final List _lines = []; + CodeBufferLine? _currentLine, _lastLine; + int _indentationLevel = 0; + int _length = 0; + + CodeBuffer( + {this.space = ' ', + this.newline = '\n', + this.trailingNewline = false, + this.sourceUrl}); + + /// Creates a [CodeBuffer] that does not emit additional whitespace. + factory CodeBuffer.noWhitespace({sourceUrl}) => CodeBuffer( + space: '', newline: '', trailingNewline: false, sourceUrl: sourceUrl); + + /// The last line created within this buffer. + CodeBufferLine? get lastLine => _lastLine; + + /// Returns an immutable collection of the [CodeBufferLine]s within this instance. + List get lines => List.unmodifiable(_lines); + + @override + bool get isEmpty => _lines.isEmpty; + + @override + bool get isNotEmpty => _lines.isNotEmpty; + + @override + int get length => _length; + + CodeBufferLine _createLine() { + var start = SourceLocation( + _length, + sourceUrl: sourceUrl, + line: _lines.length, + column: _indentationLevel * space.length, + ); + var line = CodeBufferLine._(_indentationLevel, start).._end = start; + _lines.add(_lastLine = line); + return line; + } + + /// Increments the indentation level. + void indent() { + _indentationLevel++; + } + + /// Decrements the indentation level, if it is greater than `0`. + void outdent() { + if (_indentationLevel > 0) _indentationLevel--; + } + + /// Copies the contents of this [CodeBuffer] into another, preserving indentation and source mapping information. + void copyInto(CodeBuffer other) { + if (_lines.isEmpty) return; + var i = 0; + + for (var line in _lines) { + // To compute offset: + // 1. Find current length of other + // 2. Add length of its newline + // 3. Add indentation + var column = (other._indentationLevel + line.indentationLevel) * + other.space.length; + var offset = other._length + other.newline.length + column; + + // Re-compute start + end + var start = SourceLocation( + offset, + sourceUrl: other.sourceUrl, + line: other._lines.length + i, + column: column, + ); + + var end = SourceLocation( + offset + line.span.length, + sourceUrl: other.sourceUrl, + line: start.line, + column: column + line._buf.length, + ); + + var clone = CodeBufferLine._( + line.indentationLevel + other._indentationLevel, start) + .._end = end + .._buf.write(line._buf.toString()); + + // Adjust lastSpan + if (line._lastSpan != null) { + var s = line._lastSpan!.start; + var lastSpanColumn = + ((line.indentationLevel + other._indentationLevel) * + other.space.length) + + line.text.indexOf(line._lastSpan!.text); + clone._lastSpan = SourceSpan( + SourceLocation( + offset + s.offset, + sourceUrl: other.sourceUrl, + line: clone.span.start.line, + column: lastSpanColumn, + ), + SourceLocation( + offset + s.offset + line._lastSpan!.length, + sourceUrl: other.sourceUrl, + line: clone.span.end.line, + column: lastSpanColumn + line._lastSpan!.length, + ), + line._lastSpan!.text, + ); + } + + other._lines.add(other._currentLine = other._lastLine = clone); + + // Adjust length accordingly... + other._length = offset + clone.span.length; + i++; + } + + other.writeln(); + } + + @override + void clear() { + _lines.clear(); + _length = _indentationLevel = 0; + _currentLine = null; + } + + @override + void writeCharCode(int charCode) { + _currentLine ??= _createLine(); + + _currentLine!._buf.writeCharCode(charCode); + var end = _currentLine!._end; + _currentLine!._end = SourceLocation( + end.offset + 1, + sourceUrl: end.sourceUrl, + line: end.line, + column: end.column + 1, + ); + _length++; + _currentLine!._lastSpan = + SourceSpan(end, _currentLine!._end, String.fromCharCode(charCode)); + } + + @override + void write(Object? obj) { + var msg = obj.toString(); + _currentLine ??= _createLine(); + _currentLine!._buf.write(msg); + var end = _currentLine!._end; + _currentLine!._end = SourceLocation( + end.offset + msg.length, + sourceUrl: end.sourceUrl, + line: end.line, + column: end.column + msg.length, + ); + _length += msg.length; + _currentLine!._lastSpan = SourceSpan(end, _currentLine!._end, msg); + } + + @override + void writeln([Object? obj = '']) { + if (obj != null && obj != '') write(obj); + _currentLine = null; + _length++; + } + + @override + void writeAll(Iterable objects, [String separator = '']) { + write(objects.join(separator)); + } + + @override + String toString() { + var buf = StringBuffer(); + var i = 0; + + for (var line in lines) { + if (i++ > 0) buf.write(newline); + for (var j = 0; j < line.indentationLevel; j++) { + buf.write(space); + } + buf.write(line._buf.toString()); + } + + if (trailingNewline == true) buf.write(newline); + + return buf.toString(); + } +} + +/// Represents a line of text within a [CodeBuffer]. +class CodeBufferLine { + /// Mappings from one [SourceSpan] to another, to aid with generating dynamic source maps. + final Map sourceMappings = {}; + + /// The level of indentation preceding this line. + final int indentationLevel; + + final SourceLocation _start; + final StringBuffer _buf = StringBuffer(); + late SourceLocation _end; + SourceSpan? _lastSpan; + + CodeBufferLine._(this.indentationLevel, this._start); + + /// The [SourceSpan] corresponding to the last text written to this line. + SourceSpan? get lastSpan => _lastSpan; + + /// The [SourceSpan] corresponding to this entire line. + SourceSpan get span => SourceSpan(_start, _end, _buf.toString()); + + /// The text within this line. + String get text => _buf.toString(); +} diff --git a/packages/code_buffer/pubspec.yaml b/packages/code_buffer/pubspec.yaml new file mode 100644 index 0000000..93d25b2 --- /dev/null +++ b/packages/code_buffer/pubspec.yaml @@ -0,0 +1,12 @@ +name: belatuk_code_buffer +version: 3.0.0 +description: An advanced StringBuffer geared toward generating code, and source maps. +homepage: https://github.com/dart-backend/belatuk-common-utilities/tree/main/packages/code_buffer +environment: + sdk: '>=2.12.0 <3.0.0' +dependencies: + charcode: ^1.2.0 + source_span: ^1.8.1 +dev_dependencies: + test: ^1.17.3 + lints: ^1.0.0 diff --git a/packages/code_buffer/test/copy_test.dart b/packages/code_buffer/test/copy_test.dart new file mode 100644 index 0000000..7359a71 --- /dev/null +++ b/packages/code_buffer/test/copy_test.dart @@ -0,0 +1,47 @@ +import 'package:belatuk_code_buffer/belatuk_code_buffer.dart'; +import 'package:test/test.dart'; + +void main() { + var a = CodeBuffer(), b = CodeBuffer(); + + setUp(() { + a.writeln('outer block 1'); + b + ..writeln('inner block 1') + ..writeln('inner block 2'); + b.copyInto(a..indent()); + a + ..outdent() + ..writeln('outer block 2'); + }); + + tearDown(() { + a.clear(); + b.clear(); + }); + + test('sets correct text', () { + expect( + a.toString(), + [ + 'outer block 1', + ' inner block 1', + ' inner block 2', + 'outer block 2', + ].join('\n')); + }); + + test('sets lastLine+lastSpan', () { + var c = CodeBuffer() + ..indent() + ..write('>') + ..writeln('innermost'); + c.copyInto(a); + expect(a.lastLine!.text, '>innermost'); + expect(a.lastLine!.span.start.column, 2); + expect(a.lastLine!.lastSpan!.start.line, 4); + expect(a.lastLine!.lastSpan!.start.column, 3); + expect(a.lastLine!.lastSpan!.end.line, 4); + expect(a.lastLine!.lastSpan!.end.column, 12); + }); +} diff --git a/packages/code_buffer/test/span_test.dart b/packages/code_buffer/test/span_test.dart new file mode 100644 index 0000000..ecf7b6a --- /dev/null +++ b/packages/code_buffer/test/span_test.dart @@ -0,0 +1,46 @@ +import 'package:charcode/charcode.dart'; +import 'package:belatuk_code_buffer/belatuk_code_buffer.dart'; +import 'package:test/test.dart'; + +void main() { + var buf = CodeBuffer(); + tearDown(buf.clear); + + test('writeCharCode', () { + buf.writeCharCode($x); + expect(buf.lastLine!.lastSpan!.start.column, 0); + expect(buf.lastLine!.lastSpan!.start.line, 0); + expect(buf.lastLine!.lastSpan!.end.column, 1); + expect(buf.lastLine!.lastSpan!.end.line, 0); + }); + + test('write', () { + buf.write('foo'); + expect(buf.lastLine!.lastSpan!.start.column, 0); + expect(buf.lastLine!.lastSpan!.start.line, 0); + expect(buf.lastLine!.lastSpan!.end.column, 3); + expect(buf.lastLine!.lastSpan!.end.line, 0); + }); + + test('multiple writes in one line', () { + buf + ..write('foo') + ..write('baz'); + expect(buf.lastLine!.lastSpan!.start.column, 3); + expect(buf.lastLine!.lastSpan!.start.line, 0); + expect(buf.lastLine!.lastSpan!.end.column, 6); + expect(buf.lastLine!.lastSpan!.end.line, 0); + }); + + test('multiple lines', () { + buf + ..writeln('foo') + ..write('bar') + ..write('+') + ..writeln('baz'); + expect(buf.lastLine!.lastSpan!.start.column, 4); + expect(buf.lastLine!.lastSpan!.start.line, 1); + expect(buf.lastLine!.lastSpan!.end.column, 7); + expect(buf.lastLine!.lastSpan!.end.line, 1); + }); +} diff --git a/packages/code_buffer/test/write_test.dart b/packages/code_buffer/test/write_test.dart new file mode 100644 index 0000000..628310f --- /dev/null +++ b/packages/code_buffer/test/write_test.dart @@ -0,0 +1,90 @@ +import 'package:charcode/charcode.dart'; +import 'package:test/test.dart'; +import 'package:belatuk_code_buffer/belatuk_code_buffer.dart'; + +void main() { + var buf = CodeBuffer(); + tearDown(buf.clear); + + test('writeCharCode', () { + buf.writeCharCode($x); + expect(buf.toString(), 'x'); + }); + + test('write', () { + buf.write('hello world'); + expect(buf.toString(), 'hello world'); + }); + + test('custom space', () { + var b = CodeBuffer(space: '+') + ..writeln('foo') + ..indent() + ..writeln('baz'); + expect(b.toString(), 'foo\n+baz'); + }); + + test('custom newline', () { + var b = CodeBuffer(newline: 'N') + ..writeln('foo') + ..indent() + ..writeln('baz'); + expect(b.toString(), 'fooN baz'); + }); + + test('trailing newline', () { + var b = CodeBuffer(trailingNewline: true)..writeln('foo'); + expect(b.toString(), 'foo\n'); + }); + + group('multiple lines', () { + setUp(() { + buf + ..writeln('foo') + ..writeln('bar') + ..writeln('baz'); + expect(buf.lines, hasLength(3)); + expect(buf.lines[0].text, 'foo'); + expect(buf.lines[1].text, 'bar'); + expect(buf.lines[2].text, 'baz'); + }); + }); + + test('indent', () { + buf + ..writeln('foo') + ..indent() + ..writeln('bar') + ..indent() + ..writeln('baz') + ..outdent() + ..writeln('quux') + ..outdent() + ..writeln('end'); + expect(buf.toString(), 'foo\n bar\n baz\n quux\nend'); + }); + + group('sets lastLine text', () { + test('writeCharCode', () { + buf.writeCharCode($x); + expect(buf.lastLine!.text, 'x'); + }); + + test('write', () { + buf.write('hello world'); + expect(buf.lastLine!.text, 'hello world'); + }); + }); + + group('sets lastLine lastSpan', () { + test('writeCharCode', () { + buf.writeCharCode($x); + expect(buf.lastLine!.lastSpan!.text, 'x'); + }); + + test('write', () { + buf.write('hello world'); + expect(buf.lastLine!.lastSpan!.text, 'hello world'); + }); + }); +} diff --git a/packages/combinator/.travis.yml b/packages/combinator/.travis.yml new file mode 100644 index 0000000..2f22c5c --- /dev/null +++ b/packages/combinator/.travis.yml @@ -0,0 +1,4 @@ +language: dart +dart: + - stable + - dev \ No newline at end of file diff --git a/packages/combinator/AUTHORS.md b/packages/combinator/AUTHORS.md new file mode 100644 index 0000000..ac95ab5 --- /dev/null +++ b/packages/combinator/AUTHORS.md @@ -0,0 +1,12 @@ +Primary Authors +=============== + +* __[Thomas Hii](dukefirehawk.apps@gmail.com)__ + + Thomas is the current maintainer of the code base. He has refactored and migrated the + code base to support NNBD. + +* __[Tobe O](thosakwe@gmail.com)__ + + Tobe has written much of the original code prior to NNBD migration. He has moved on and + is no longer involved with the project. diff --git a/packages/combinator/CHANGELOG.md b/packages/combinator/CHANGELOG.md new file mode 100644 index 0000000..60bcaaa --- /dev/null +++ b/packages/combinator/CHANGELOG.md @@ -0,0 +1,33 @@ +# Change Log + +## 3.0.0 + +* Upgraded from `pendantic` to `lints` linter +* Published as `belatuk_combinator` package + +## 2.0.2 + +* Resolve static analysis warnings + +## 2.0.1 + +* Updated README + +## 2.0.0 + +* Migrated to support Dart SDK 2.12.x NNBD + +## 1.1.0 + +* Add `tupleX` parsers. Hooray for strong typing! + +## 1.0.0+3 + +* `then` now *always* returns `dynamic`. + +## 1.0.0+2 + +* `star` now includes with a call to `opt`. +* Added comments. +* Enforce generics on `separatedBy`. +* Enforce Dart 2 semantics. diff --git a/packages/combinator/LICENSE b/packages/combinator/LICENSE new file mode 100644 index 0000000..e37a346 --- /dev/null +++ b/packages/combinator/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2021, dukefirehawk.com +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/packages/combinator/README.md b/packages/combinator/README.md new file mode 100644 index 0000000..23ec825 --- /dev/null +++ b/packages/combinator/README.md @@ -0,0 +1,128 @@ +# Belatuk Combinator + +[![version](https://img.shields.io/badge/pub-v3.0.0-brightgreen)](https://pub.dartlang.org/packages/belatuk_combinator) +[![Null Safety](https://img.shields.io/badge/null-safety-brightgreen)](https://dart.dev/null-safety) +[![License](https://img.shields.io/github/license/dart-backend/belatuk-common-utilities)](https://github.com/dart-backend/belatuk-common-utilities/packages/combinator/LICENSE) + +Packrat parser combinators that support static typing, generics, file spans, memoization, and more. + +**RECOMMENDED:** +Check `example/` for examples. +The examples contain examples of using: + +* Generic typing +* Reading `FileSpan` from `ParseResult` +* More... + +## Basic Usage + +```dart +void main() { + // Parse a Pattern (usually String or RegExp). + var foo = match('foo'); + var number = match(RegExp(r'[0-9]+'), errorMessage: 'Expected a number.'); + + // Set a value. + var numWithValue = number.map((r) => int.parse(r.span.text)); + + // Expect a pattern, or nothing. + var optional = numWithValue.opt(); + + // Expect a pattern zero or more times. + var star = optional.star(); + + // Expect one or more times. + var plus = optional.plus(); + + // Expect an arbitrary number of times. + var threeTimes = optional.times(3); + + // Expect a sequence of patterns. + var doraTheExplorer = chain([ + match('Dora').space(), + match('the').space(), + match('Explorer').space(), + ]); + + // Choose exactly one of a set of patterns, whichever + // appears first. + var alt = any([ + match('1'), + match('11'), + match('111'), + ]); + + // Choose the *longest* match for any of the given alternatives. + var alt2 = longest([ + match('1'), + match('11'), + match('111'), + ]); + + // Friendly operators + var fooOrNumber = foo | number; + var fooAndNumber = foo & number; + var notFoo = ~foo; +} +``` + +## Error Messages + +Parsers without descriptive error messages can lead to frustrating dead-ends +for end-users. Fortunately, `belatuk_combinator` is built with error handling in mind. + +```dart +void main(Parser parser) { + // Append an arbitrary error message to a parser if it is not matched. + var withError = parser.error(errorMessage: 'Hey!!! Wrong!!!'); + + // You can also set the severity of an error. + var asHint = parser.error(severity: SyntaxErrorSeverity.hint); + + // Constructs like `any`, `chain`, and `longest` support this as well. + var foo = longest([ + parser.error(errorMessage: 'foo'), + parser.error(errorMessage: 'bar') + ], errorMessage: 'Expected a "foo" or a "bar"'); + + // If multiple errors are present at one location, + // it can create a lot of noise. + // + // Use `foldErrors` to only take one error at a given location. + var lessNoise = parser.foldErrors(); +} +``` + +## Whitespaces + +Handling optional whitespace is dead-easy: + +```dart +void main(Parser parser) { + var optionalSpace = parser.space(); +} +``` + +## For Programming Languages + +`belatuk_combinator` was conceived to make writing parsers for complex grammars easier, +namely programming languages. Thus, there are functions built-in to make common constructs +easier: + +```dart +void main(Parser parser) { + var array = parser + .separatedByComma() + .surroundedBySquareBrackets(defaultValue: []); + + var braces = parser.surroundedByCurlyBraces(); + + var sep = parser.separatedBy(match('!').space()); +} +``` + +## Differences between this and Petitparser + +* `belatuk_combinator` makes extensive use of Dart's dynamic typing +* `belatuk_combinator` supports detailed error messages (with configurable severity) +* `belatuk_combinator` keeps track of locations (ex. `line 1: 3`) diff --git a/packages/combinator/analysis_options.yaml b/packages/combinator/analysis_options.yaml new file mode 100644 index 0000000..ea2c9e9 --- /dev/null +++ b/packages/combinator/analysis_options.yaml @@ -0,0 +1 @@ +include: package:lints/recommended.yaml \ No newline at end of file diff --git a/packages/combinator/combinator.iml b/packages/combinator/combinator.iml new file mode 100644 index 0000000..75734c9 --- /dev/null +++ b/packages/combinator/combinator.iml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/packages/combinator/example/basic_auth.dart b/packages/combinator/example/basic_auth.dart new file mode 100644 index 0000000..3f12808 --- /dev/null +++ b/packages/combinator/example/basic_auth.dart @@ -0,0 +1,56 @@ +// Run this with "Basic QWxhZGRpbjpPcGVuU2VzYW1l" + +import 'dart:convert'; +import 'dart:io'; +import 'package:angel3_combinator/belatuk_combinator.dart'; +import 'package:string_scanner/string_scanner.dart'; + +/// Parse a part of a decoded Basic auth string. +/// +/// Namely, the `username` or `password` in `{username}:{password}`. +final Parser string = + match(RegExp(r'[^:$]+'), errorMessage: 'Expected a string.') + .value((r) => r.span!.text); + +/// Transforms `{username}:{password}` to `{"username": username, "password": password}`. +final Parser> credentials = chain([ + string.opt(), + match(':'), + string.opt(), +]).map>( + (r) => {'username': r.value![0], 'password': r.value![2]}); + +/// We can actually embed a parser within another parser. +/// +/// This is used here to BASE64URL-decode a string, and then +/// parse the decoded string. +final Parser credentialString = match?>( + RegExp(r'([^\n$]+)'), + errorMessage: 'Expected a credential string.') + .value((r) { + var decoded = utf8.decode(base64Url.decode(r.span!.text)); + var scanner = SpanScanner(decoded); + return credentials.parse(scanner).value; +}); + +final Parser basic = match('Basic').space(); + +final Parser basicAuth = basic.then(credentialString).index(1); + +void main() { + while (true) { + stdout.write('Enter a basic auth value: '); + var line = stdin.readLineSync()!; + var scanner = SpanScanner(line, sourceUrl: 'stdin'); + var result = basicAuth.parse(scanner); + + if (!result.successful) { + for (var error in result.errors) { + print(error.toolString); + print(error.span!.highlight(color: true)); + } + } else { + print(result.value); + } + } +} diff --git a/packages/combinator/example/calculator.dart b/packages/combinator/example/calculator.dart new file mode 100644 index 0000000..2f99afc --- /dev/null +++ b/packages/combinator/example/calculator.dart @@ -0,0 +1,71 @@ +import 'dart:math'; +import 'dart:io'; +import 'package:angel3_combinator/belatuk_combinator.dart'; +import 'package:string_scanner/string_scanner.dart'; + +/// Note: This grammar does not handle precedence, for the sake of simplicity. +Parser calculatorGrammar() { + var expr = reference(); + + var number = match(RegExp(r'-?[0-9]+(\.[0-9]+)?')) + .value((r) => num.parse(r.span!.text)); + + var hex = match(RegExp(r'0x([A-Fa-f0-9]+)')) + .map((r) => int.parse(r.scanner.lastMatch![1]!, radix: 16)); + + var binary = match(RegExp(r'([0-1]+)b')) + .map((r) => int.parse(r.scanner.lastMatch![1]!, radix: 2)); + + var alternatives = >[]; + + void registerBinary(String op, num Function(num, num) f) { + alternatives.add( + chain([ + expr.space(), + match(op).space() as Parser, + expr.space(), + ]).map((r) => f(r.value![0], r.value![2])), + ); + } + + registerBinary('**', (a, b) => pow(a, b)); + registerBinary('*', (a, b) => a * b); + registerBinary('/', (a, b) => a / b); + registerBinary('%', (a, b) => a % b); + registerBinary('+', (a, b) => a + b); + registerBinary('-', (a, b) => a - b); + registerBinary('^', (a, b) => a.toInt() ^ b.toInt()); + registerBinary('&', (a, b) => a.toInt() & b.toInt()); + registerBinary('|', (a, b) => a.toInt() | b.toInt()); + + alternatives.addAll([ + number, + hex, + binary, + expr.parenthesized(), + ]); + + expr.parser = longest(alternatives); + + return expr; +} + +void main() { + var calculator = calculatorGrammar(); + + while (true) { + stdout.write('Enter an expression: '); + var line = stdin.readLineSync()!; + var scanner = SpanScanner(line, sourceUrl: 'stdin'); + var result = calculator.parse(scanner); + + if (!result.successful) { + for (var error in result.errors) { + stderr.writeln(error.toolString); + stderr.writeln(error.span!.highlight(color: true)); + } + } else { + print(result.value); + } + } +} diff --git a/packages/combinator/example/delimiter.dart b/packages/combinator/example/delimiter.dart new file mode 100644 index 0000000..6876a55 --- /dev/null +++ b/packages/combinator/example/delimiter.dart @@ -0,0 +1,29 @@ +import 'dart:io'; +import 'package:angel3_combinator/belatuk_combinator.dart'; +import 'package:string_scanner/string_scanner.dart'; + +final Parser id = + match(RegExp(r'[A-Za-z]+')).value((r) => r.span!.text); + +// We can use `separatedBy` to easily construct parser +// that can be matched multiple times, separated by another +// pattern. +// +// This is useful for parsing arrays or map literals. +void main() { + while (true) { + stdout.write('Enter a string (ex "a,b,c"): '); + var line = stdin.readLineSync()!; + var scanner = SpanScanner(line, sourceUrl: 'stdin'); + var result = id.separatedBy(match(',').space()).parse(scanner); + + if (!result.successful) { + for (var error in result.errors) { + print(error.toolString); + print(error.span!.highlight(color: true)); + } + } else { + print(result.value); + } + } +} diff --git a/packages/combinator/example/json.dart b/packages/combinator/example/json.dart new file mode 100644 index 0000000..90f038f --- /dev/null +++ b/packages/combinator/example/json.dart @@ -0,0 +1,71 @@ +import 'dart:io'; +import 'package:angel3_combinator/belatuk_combinator.dart'; +import 'package:string_scanner/string_scanner.dart'; + +Parser jsonGrammar() { + var expr = reference(); + + // Parse a number + var number = match(RegExp(r'-?[0-9]+(\.[0-9]+)?'), + errorMessage: 'Expected a number.') + .value( + (r) => num.parse(r.span!.text), + ); + + // Parse a string (no escapes supported, because lazy). + var string = + match(RegExp(r'"[^"]*"'), errorMessage: 'Expected a string.').value( + (r) => r.span!.text.substring(1, r.span!.text.length - 1), + ); + + // Parse an array + var array = expr + .space() + .separatedByComma() + .surroundedBySquareBrackets(defaultValue: []); + + // KV pair + var keyValuePair = chain([ + string.space(), + match(':').space(), + expr.error(errorMessage: 'Missing expression.'), + ]).castDynamic().cast().value((r) => {r.value![0]: r.value![2]}); + + // Parse an object. + var object = keyValuePair + .separatedByComma() + .castDynamic() + .surroundedByCurlyBraces(defaultValue: {}); + + expr.parser = longest( + [ + array, + number, + string, + object.error(), + ], + errorMessage: 'Expected an expression.', + ).space(); + + return expr.foldErrors(); +} + +void main() { + var JSON = jsonGrammar(); + + while (true) { + stdout.write('Enter some JSON: '); + var line = stdin.readLineSync()!; + var scanner = SpanScanner(line, sourceUrl: 'stdin'); + var result = JSON.parse(scanner); + + if (!result.successful) { + for (var error in result.errors) { + print(error.toolString); + print(error.span!.highlight(color: true)); + } + } else { + print(result.value); + } + } +} diff --git a/packages/combinator/example/main.dart b/packages/combinator/example/main.dart new file mode 100644 index 0000000..ff07b89 --- /dev/null +++ b/packages/combinator/example/main.dart @@ -0,0 +1,38 @@ +import 'dart:io'; +import 'package:angel3_combinator/belatuk_combinator.dart'; +import 'package:string_scanner/string_scanner.dart'; + +final Parser minus = match('-'); + +final Parser digit = + match(RegExp(r'[0-9]'), errorMessage: 'Expected a number'); + +final Parser digits = digit.plus(); + +final Parser dot = match('.'); + +final Parser decimal = ( // digits, (dot, digits)? + digits & (dot & digits).opt() // + ); + +final Parser number = // + (minus.opt() & decimal) // minus?, decimal + .map((r) => num.parse(r.span!.text)); + +void main() { + while (true) { + stdout.write('Enter a number: '); + var line = stdin.readLineSync()!; + var scanner = SpanScanner(line, sourceUrl: 'stdin'); + var result = number.parse(scanner); + + if (!result.successful) { + for (var error in result.errors) { + stderr.writeln(error.toolString); + stderr.writeln(error.span!.highlight(color: true)); + } + } else { + print(result.value); + } + } +} diff --git a/packages/combinator/example/query_string.dart b/packages/combinator/example/query_string.dart new file mode 100644 index 0000000..d2525bc --- /dev/null +++ b/packages/combinator/example/query_string.dart @@ -0,0 +1,45 @@ +// For some reason, this cannot be run in checked mode??? + +import 'dart:io'; +import 'package:angel3_combinator/belatuk_combinator.dart'; +import 'package:string_scanner/string_scanner.dart'; + +final Parser key = + match(RegExp(r'[^=&\n]+'), errorMessage: 'Missing k/v') + .value((r) => r.span!.text); + +final Parser value = key.map((r) => Uri.decodeQueryComponent(r.value!)); + +final Parser pair = chain([ + key, + match('='), + value, +]).map((r) { + return { + r.value![0]: r.value![2], + }; +}); + +final Parser pairs = pair + .separatedBy(match(r'&')) + .map((r) => r.value!.reduce((a, b) => a..addAll(b))); + +final Parser queryString = pairs.opt(); + +void main() { + while (true) { + stdout.write('Enter a query string: '); + var line = stdin.readLineSync()!; + var scanner = SpanScanner(line, sourceUrl: 'stdin'); + var result = pairs.parse(scanner); + + if (!result.successful) { + for (var error in result.errors) { + print(error.toolString); + print(error.span!.highlight(color: true)); + } + } else { + print(result.value); + } + } +} diff --git a/packages/combinator/example/sexp.dart b/packages/combinator/example/sexp.dart new file mode 100644 index 0000000..93cb98e --- /dev/null +++ b/packages/combinator/example/sexp.dart @@ -0,0 +1,85 @@ +import 'dart:collection'; +import 'dart:io'; +import 'dart:math'; +import 'package:angel3_combinator/belatuk_combinator.dart'; +import 'package:string_scanner/string_scanner.dart'; +import 'package:tuple/tuple.dart'; + +void main() { + var expr = reference(); + var symbols = {}; + + void registerFunction(String name, int nArgs, Function(List) f) { + symbols[name] = Tuple2(nArgs, f); + } + + registerFunction('**', 2, (args) => pow(args[0], args[1])); + registerFunction('*', 2, (args) => args[0] * args[1]); + registerFunction('/', 2, (args) => args[0] / args[1]); + registerFunction('%', 2, (args) => args[0] % args[1]); + registerFunction('+', 2, (args) => args[0] + args[1]); + registerFunction('-', 2, (args) => args[0] - args[1]); + registerFunction('.', 1, (args) => args[0].toDouble()); + registerFunction('print', 1, (args) { + print(args[0]); + return args[0]; + }); + + var number = + match(RegExp(r'[0-9]+(\.[0-9]+)?'), errorMessage: 'Expected a number.') + .map((r) => num.parse(r.span!.text)); + + var id = match( + RegExp( + r'[A-Za-z_!\\$",\\+-\\./:;\\?<>%&\\*@\[\]\\{\}\\|`\\^~][A-Za-z0-9_!\\$",\\+-\\./:;\\?<>%&\*@\[\]\\{\}\\|`\\^~]*'), + errorMessage: 'Expected an ID') + .map((r) => symbols[r.span!.text] ??= + throw "Undefined symbol: '${r.span!.text}'"); + + var atom = number.castDynamic().or(id); + + var list = expr.space().times(2, exact: false).map((r) { + try { + var out = []; + var q = Queue.from(r.value!.reversed); + + while (q.isNotEmpty) { + var current = q.removeFirst(); + if (current is! Tuple2) { + out.insert(0, current); + } else { + var args = []; + for (var i = 0; i < (current.item1 as num); i++) { + args.add(out.removeLast()); + } + out.add(current.item2(args)); + } + } + + return out.length == 1 ? out.first : out; + } catch (_) { + return []; + } + }); + + expr.parser = longest([ + list, + atom, + expr.parenthesized(), + ]); //list | atom | expr.parenthesized(); + + while (true) { + stdout.write('> '); + var line = stdin.readLineSync()!; + var result = expr.parse(SpanScanner(line)); + + if (result.errors.isNotEmpty) { + for (var error in result.errors) { + print(error.toolString); + print(error.message); + } + } else { + print(result.value); + } + } +} diff --git a/packages/combinator/example/tuple.dart b/packages/combinator/example/tuple.dart new file mode 100644 index 0000000..68e4032 --- /dev/null +++ b/packages/combinator/example/tuple.dart @@ -0,0 +1,14 @@ +import 'package:angel3_combinator/belatuk_combinator.dart'; +import 'package:string_scanner/string_scanner.dart'; + +void main() { + var pub = match('pub').map((r) => r.span!.text).space(); + var dart = match('dart').map((r) => 24).space(); + var lang = match('lang').map((r) => true).space(); + + // Parses a Tuple3 + var grammar = tuple3(pub, dart, lang); + + var scanner = SpanScanner('pub dart lang'); + print(grammar.parse(scanner).value); +} diff --git a/packages/combinator/lib/belatuk_combinator.dart b/packages/combinator/lib/belatuk_combinator.dart new file mode 100644 index 0000000..79e4c07 --- /dev/null +++ b/packages/combinator/lib/belatuk_combinator.dart @@ -0,0 +1,2 @@ +export 'src/combinator/combinator.dart'; +export 'src/error.dart'; diff --git a/packages/combinator/lib/src/combinator/advance.dart b/packages/combinator/lib/src/combinator/advance.dart new file mode 100644 index 0000000..01506dc --- /dev/null +++ b/packages/combinator/lib/src/combinator/advance.dart @@ -0,0 +1,26 @@ +part of lex.src.combinator; + +class _Advance extends Parser { + final Parser parser; + final int amount; + + _Advance(this.parser, this.amount); + + @override + ParseResult __parse(ParseArgs args) { + var result = parser._parse(args.increaseDepth()).change(parser: this); + if (result.successful) args.scanner.position += amount; + return result; + } + + @override + void stringify(CodeBuffer buffer) { + buffer + ..writeln('advance($amount) (') + ..indent(); + parser.stringify(buffer); + buffer + ..outdent() + ..writeln(')'); + } +} diff --git a/packages/combinator/lib/src/combinator/any.dart b/packages/combinator/lib/src/combinator/any.dart new file mode 100644 index 0000000..45bc107 --- /dev/null +++ b/packages/combinator/lib/src/combinator/any.dart @@ -0,0 +1,85 @@ +part of lex.src.combinator; + +/// Matches any one of the given [parsers]. +/// +/// If [backtrack] is `true` (default), a failed parse will not modify the scanner state. +/// +/// You can provide a custom [errorMessage]. You can set it to `false` to not +/// generate any error at all. +Parser any(Iterable> parsers, + {bool backtrack = true, errorMessage, SyntaxErrorSeverity? severity}) { + return _Any(parsers, backtrack != false, errorMessage, + severity ?? SyntaxErrorSeverity.error); +} + +class _Any extends Parser { + final Iterable> parsers; + final bool backtrack; + final errorMessage; + final SyntaxErrorSeverity severity; + + _Any(this.parsers, this.backtrack, this.errorMessage, this.severity); + + @override + ParseResult _parse(ParseArgs args) { + var inactive = parsers + .where((p) => !args.trampoline.isActive(p, args.scanner.position)); + + if (inactive.isEmpty) { + return ParseResult(args.trampoline, args.scanner, this, false, []); + } + + var errors = []; + var replay = args.scanner.position; + + for (var parser in inactive) { + var result = parser._parse(args.increaseDepth()); + + if (result.successful) { + return result; + } else { + if (backtrack) args.scanner.position = replay; + if (parser is _Alt) errors.addAll(result.errors); + } + } + + if (errorMessage != false) { + errors.add( + SyntaxError( + severity, + errorMessage?.toString() ?? + 'No match found for ${parsers.length} alternative(s)', + args.scanner.emptySpan, + ), + ); + } + + return ParseResult(args.trampoline, args.scanner, this, false, errors); + } + + @override + ParseResult __parse(ParseArgs args) { + // Never called + throw ArgumentError('[Combinator] Invalid method call'); + } + + @override + void stringify(CodeBuffer buffer) { + buffer + ..writeln('any(${parsers.length}) (') + ..indent(); + var i = 1; + + for (var parser in parsers) { + buffer + ..writeln('#${i++}:') + ..indent(); + parser.stringify(buffer); + buffer.outdent(); + } + + buffer + ..outdent() + ..writeln(')'); + } +} diff --git a/packages/combinator/lib/src/combinator/cache.dart b/packages/combinator/lib/src/combinator/cache.dart new file mode 100644 index 0000000..b07b38e --- /dev/null +++ b/packages/combinator/lib/src/combinator/cache.dart @@ -0,0 +1,26 @@ +part of lex.src.combinator; + +class _Cache extends Parser { + final Map> _cache = {}; + final Parser parser; + + _Cache(this.parser); + + @override + ParseResult __parse(ParseArgs args) { + return _cache.putIfAbsent(args.scanner.position, () { + return parser._parse(args.increaseDepth()); + }).change(parser: this); + } + + @override + void stringify(CodeBuffer buffer) { + buffer + ..writeln('cache(${_cache.length}) (') + ..indent(); + parser.stringify(buffer); + buffer + ..outdent() + ..writeln(')'); + } +} diff --git a/packages/combinator/lib/src/combinator/cast.dart b/packages/combinator/lib/src/combinator/cast.dart new file mode 100644 index 0000000..9b531d4 --- /dev/null +++ b/packages/combinator/lib/src/combinator/cast.dart @@ -0,0 +1,63 @@ +part of lex.src.combinator; + +class _Cast extends Parser { + final Parser parser; + + _Cast(this.parser); + + @override + ParseResult __parse(ParseArgs args) { + var result = parser._parse(args.increaseDepth()); + return ParseResult( + args.trampoline, + args.scanner, + this, + result.successful, + result.errors, + span: result.span, + value: result.value as U?, + ); + } + + @override + void stringify(CodeBuffer buffer) { + buffer + ..writeln('cast<$U> (') + ..indent(); + parser.stringify(buffer); + buffer + ..outdent() + ..writeln(')'); + } +} + +class _CastDynamic extends Parser { + final Parser parser; + + _CastDynamic(this.parser); + + @override + ParseResult __parse(ParseArgs args) { + var result = parser._parse(args.increaseDepth()); + return ParseResult( + args.trampoline, + args.scanner, + this, + result.successful, + result.errors, + span: result.span, + value: result.value, + ); + } + + @override + void stringify(CodeBuffer buffer) { + buffer + ..writeln('cast (') + ..indent(); + parser.stringify(buffer); + buffer + ..outdent() + ..writeln(')'); + } +} diff --git a/packages/combinator/lib/src/combinator/chain.dart b/packages/combinator/lib/src/combinator/chain.dart new file mode 100644 index 0000000..101836b --- /dev/null +++ b/packages/combinator/lib/src/combinator/chain.dart @@ -0,0 +1,111 @@ +part of lex.src.combinator; + +/// Expects to parse a sequence of [parsers]. +/// +/// If [failFast] is `true` (default), then the first failure to parse will abort the parse. +ListParser chain(Iterable> parsers, + {bool failFast = true, SyntaxErrorSeverity? severity}) { + return _Chain( + parsers, failFast != false, severity ?? SyntaxErrorSeverity.error); +} + +class _Alt extends Parser { + final Parser parser; + final String? errorMessage; + final SyntaxErrorSeverity severity; + + _Alt(this.parser, this.errorMessage, this.severity); + + @override + ParseResult __parse(ParseArgs args) { + var result = parser._parse(args.increaseDepth()); + return result.successful + ? result + : result.addErrors([ + SyntaxError( + severity, errorMessage, result.span ?? args.scanner.emptySpan), + ]); + } + + @override + void stringify(CodeBuffer buffer) { + parser.stringify(buffer); + } +} + +class _Chain extends ListParser { + final Iterable> parsers; + final bool failFast; + final SyntaxErrorSeverity severity; + + _Chain(this.parsers, this.failFast, this.severity); + + @override + ParseResult> __parse(ParseArgs args) { + var errors = []; + var results = []; + var spans = []; + var successful = true; + + for (var parser in parsers) { + var result = parser._parse(args.increaseDepth()); + + if (!result.successful) { + if (parser is _Alt) errors.addAll(result.errors); + + if (failFast) { + return ParseResult( + args.trampoline, args.scanner, this, false, result.errors); + } + + successful = false; + } + + if (result.value != null) { + results.add(result.value!); + } else { + results.add('NULL' as T); + } + + if (result.span != null) { + spans.add(result.span!); + } + } + + FileSpan? span; + + if (spans.isNotEmpty) { + span = spans.reduce((a, b) => a.expand(b)); + } + + return ParseResult>( + args.trampoline, + args.scanner, + this, + successful, + errors, + span: span, + value: List.unmodifiable(results), + ); + } + + @override + void stringify(CodeBuffer buffer) { + buffer + ..writeln('chain(${parsers.length}) (') + ..indent(); + var i = 1; + + for (var parser in parsers) { + buffer + ..writeln('#${i++}:') + ..indent(); + parser.stringify(buffer); + buffer.outdent(); + } + + buffer + ..outdent() + ..writeln(')'); + } +} diff --git a/packages/combinator/lib/src/combinator/check.dart b/packages/combinator/lib/src/combinator/check.dart new file mode 100644 index 0000000..a6f7bed --- /dev/null +++ b/packages/combinator/lib/src/combinator/check.dart @@ -0,0 +1,42 @@ +part of lex.src.combinator; + +class _Check extends Parser { + final Parser parser; + final Matcher matcher; + final String? errorMessage; + final SyntaxErrorSeverity severity; + + _Check(this.parser, this.matcher, this.errorMessage, this.severity); + + @override + ParseResult __parse(ParseArgs args) { + var matchState = {}; + var result = parser._parse(args.increaseDepth()).change(parser: this); + if (!result.successful) { + return result; + } else if (!matcher.matches(result.value, matchState)) { + return result.change(successful: false).addErrors([ + SyntaxError( + severity, + errorMessage ?? + matcher.describe(StringDescription('Expected ')).toString() + '.', + result.span, + ), + ]); + } else { + return result; + } + } + + @override + void stringify(CodeBuffer buffer) { + var d = matcher.describe(StringDescription()); + buffer + ..writeln('check($d) (') + ..indent(); + parser.stringify(buffer); + buffer + ..outdent() + ..writeln(')'); + } +} diff --git a/packages/combinator/lib/src/combinator/combinator.dart b/packages/combinator/lib/src/combinator/combinator.dart new file mode 100644 index 0000000..717365a --- /dev/null +++ b/packages/combinator/lib/src/combinator/combinator.dart @@ -0,0 +1,394 @@ +library lex.src.combinator; + +import 'dart:collection'; + +import 'package:angel3_code_buffer/angel3_code_buffer.dart'; +import 'package:matcher/matcher.dart'; +import 'package:source_span/source_span.dart'; +import 'package:string_scanner/string_scanner.dart'; +import 'package:tuple/tuple.dart'; +import '../error.dart'; + +part 'any.dart'; + +part 'advance.dart'; + +part 'cache.dart'; + +part 'cast.dart'; + +part 'chain.dart'; + +part 'check.dart'; + +part 'compare.dart'; + +part 'fold_errors.dart'; + +part 'index.dart'; + +part 'longest.dart'; + +part 'map.dart'; + +part 'match.dart'; + +part 'max_depth.dart'; + +part 'negate.dart'; + +part 'opt.dart'; + +part 'recursion.dart'; + +part 'reduce.dart'; + +part 'reference.dart'; + +part 'repeat.dart'; + +part 'safe.dart'; + +part 'to_list.dart'; + +part 'util.dart'; + +part 'value.dart'; + +class ParseArgs { + final Trampoline trampoline; + final SpanScanner scanner; + final int depth; + + ParseArgs(this.trampoline, this.scanner, this.depth); + + ParseArgs increaseDepth() => ParseArgs(trampoline, scanner, depth + 1); +} + +/// A parser combinator, which can parse very complicated grammars in a manageable manner. +abstract class Parser { + ParseResult __parse(ParseArgs args); + + ParseResult _parse(ParseArgs args) { + var pos = args.scanner.position; + + if (args.trampoline.hasMemoized(this, pos)) { + return args.trampoline.getMemoized(this, pos); + } + + if (args.trampoline.isActive(this, pos)) { + return ParseResult(args.trampoline, args.scanner, this, false, []); + } + + args.trampoline.enter(this, pos); + var result = __parse(args); + args.trampoline.memoize(this, pos, result); + args.trampoline.exit(this); + return result; + } + + /// Parses text from a [SpanScanner]. + ParseResult parse(SpanScanner scanner, [int depth = 1]) { + var args = ParseArgs(Trampoline(), scanner, depth); + return _parse(args); + } + + /// Skips forward a certain amount of steps after parsing, if it was successful. + Parser forward(int amount) => _Advance(this, amount); + + /// Moves backward a certain amount of steps after parsing, if it was successful. + Parser back(int amount) => _Advance(this, amount * -1); + + /// Casts this parser to produce [U] objects. + Parser cast() => _Cast(this); + + /// Casts this parser to produce [dynamic] objects. + Parser castDynamic() => _CastDynamic(this); + + /// Runs the given function, which changes the returned [ParseResult] into one relating to a [U] object. + Parser change(ParseResult Function(ParseResult) f) { + return _Change(this, f); + } + + /// Validates the parse result against a [Matcher]. + /// + /// You can provide a custom [errorMessage]. + Parser check(Matcher matcher, + {String? errorMessage, SyntaxErrorSeverity? severity}) => + _Check( + this, matcher, errorMessage, severity ?? SyntaxErrorSeverity.error); + + /// Binds an [errorMessage] to a copy of this parser. + Parser error({String? errorMessage, SyntaxErrorSeverity? severity}) => + _Alt(this, errorMessage, severity ?? SyntaxErrorSeverity.error); + + /// Removes multiple errors that occur in the same spot; this can reduce noise in parser output. + Parser foldErrors({bool Function(SyntaxError a, SyntaxError b)? equal}) { + equal ??= (b, e) => b.span?.start.offset == e.span?.start.offset; + return _FoldErrors(this, equal); + } + + /// Transforms the parse result using a unary function. + Parser map(U Function(ParseResult) f) { + return _Map(this, f); + } + + /// Prevents recursion past a certain [depth], preventing stack overflow errors. + Parser maxDepth(int depth) => _MaxDepth(this, depth); + + Parser operator ~() => negate(); + + /// Ensures this pattern is not matched. + /// + /// You can provide an [errorMessage]. + Parser negate( + {String errorMessage = 'Negate error', + SyntaxErrorSeverity severity = SyntaxErrorSeverity.error}) => + _Negate(this, errorMessage, severity); + + /// Caches the results of parse attempts at various locations within the source text. + /// + /// Use this to prevent excessive recursion. + Parser cache() => _Cache(this); + + Parser operator &(Parser other) => and(other); + + /// Consumes `this` and another parser, but only considers the result of `this` parser. + Parser and(Parser other) => then(other).change((r) { + return ParseResult( + r.trampoline, + r.scanner, + this, + r.successful, + r.errors, + span: r.span, + value: (r.value != null ? r.value![0] : r.value) as T?, + ); + }); + + Parser operator |(Parser other) => or(other); + + /// Shortcut for [or]-ing two parsers. + Parser or(Parser other) => any([this, other]); + + /// Parses this sequence one or more times. + ListParser plus() => times(1, exact: false); + + /// Safely escapes this parser when an error occurs. + /// + /// The generated parser only runs once; repeated uses always exit eagerly. + Parser safe( + {bool backtrack = true, + String errorMessage = 'error', + SyntaxErrorSeverity? severity}) => + _Safe( + this, backtrack, errorMessage, severity ?? SyntaxErrorSeverity.error); + + Parser> separatedByComma() => + separatedBy(match>(',').space()); + + /// Expects to see an infinite amounts of the pattern, separated by the [other] pattern. + /// + /// Use this as a shortcut to parse arrays, parameter lists, etc. + Parser> separatedBy(Parser other) { + var suffix = other.then(this).index(1).cast(); + return then(suffix.star()).map((r) { + var v = r.value; + if (v == null || v.length < 2) { + return []; + } + var preceding = v.isEmpty ? [] : (v[0] == null ? [] : [v[0]]); + var out = List.from(preceding); + if (v[1] != null && v[1] != 'NULL') { + v[1].forEach((element) { + out.add(element as T); + }); + } + return out; + }); + } + + Parser surroundedByCurlyBraces({required T defaultValue}) => opt() + .surroundedBy(match('{').space(), match('}').space()) + .map((r) => r.value ?? defaultValue); + + Parser surroundedBySquareBrackets({required T defaultValue}) => opt() + .surroundedBy(match('[').space(), match(']').space()) + .map((r) => r.value ?? defaultValue); + + /// Expects to see the pattern, surrounded by the others. + /// + /// If no [right] is provided, it expects to see the same pattern on both sides. + /// Use this parse things like parenthesized expressions, arrays, etc. + Parser surroundedBy(Parser left, [Parser? right]) { + return chain([ + left, + this, + right ?? left, + ]).index(1).castDynamic().cast(); + } + + /// Parses `this`, either as-is or wrapped in parentheses. + Parser maybeParenthesized() { + return any([parenthesized(), this]); + } + + /// Parses `this`, wrapped in parentheses. + Parser parenthesized() => + surroundedBy(match('(').space(), match(')').space()); + + /// Consumes any trailing whitespace. + Parser space() => trail(RegExp(r'[ \n\r\t]+')); + + /// Consumes 0 or more instance(s) of this parser. + ListParser star({bool backtrack = true}) => + times(1, exact: false, backtrack: backtrack).opt(); + + /// Shortcut for [chain]-ing two parsers together. + ListParser then(Parser other) => chain([this, other]); + + /// Casts this instance into a [ListParser]. + ListParser toList() => _ToList(this); + + /// Consumes and ignores any trailing occurrences of [pattern]. + Parser trail(Pattern pattern) => + then(match(pattern).opt()).first().cast(); + + /// Expect this pattern a certain number of times. + /// + /// If [exact] is `false` (default: `true`), then the generated parser will accept + /// an infinite amount of occurrences after the specified [count]. + /// + /// You can provide custom error messages for when there are [tooFew] or [tooMany] occurrences. + ListParser times(int count, + {bool exact = true, + String tooFew = 'Too few', + String tooMany = 'Too many', + bool backtrack = true, + SyntaxErrorSeverity? severity}) { + return _Repeat(this, count, exact, tooFew, tooMany, backtrack, + severity ?? SyntaxErrorSeverity.error); + } + + /// Produces an optional copy of this parser. + /// + /// If [backtrack] is `true` (default), then a failed parse will not + /// modify the scanner state. + Parser opt({bool backtrack = true}) => _Opt(this, backtrack); + + /// Sets the value of the [ParseResult]. + Parser value(T Function(ParseResult) f) { + return _Value(this, f); + } + + /// Prints a representation of this parser, ideally without causing a stack overflow. + void stringify(CodeBuffer buffer); +} + +/// A [Parser] that produces [List]s of a type [T]. +abstract class ListParser extends Parser> { + /// Shortcut for calling [index] with `0`. + Parser first() => index(0); + + /// Modifies this parser to only return the value at the given index [i]. + Parser index(int i) => _Index(this, i); + + /// Shortcut for calling [index] with the greatest-possible index. + Parser last() => index(-1); + + /// Modifies this parser to call `List.reduce` on the parsed values. + Parser reduce(T Function(T, T) combine) => _Reduce(this, combine); + + /// Sorts the parsed values, using the given [Comparator]. + ListParser sort(Comparator compare) => _Compare(this, compare); + + @override + ListParser opt({bool backtrack = true}) => _ListOpt(this, backtrack); + + /// Modifies this parser, returning only the values that match a predicate. + Parser> where(bool Function(T) f) => + map>((r) => r.value?.where(f).toList() ?? []); + + /// Condenses a [ListParser] into having a value of the combined span's text. + Parser flatten() => map((r) => r.span?.text ?? ''); +} + +/// Prevents stack overflow in recursive parsers. +class Trampoline { + final Map> _active = {}; + final Map>> _memo = {}; + + bool hasMemoized(Parser parser, int position) { + var list = _memo[parser]; + return list?.any((t) => t.item1 == position) == true; + } + + ParseResult getMemoized(Parser parser, int position) { + return _memo[parser]?.firstWhere((t) => t.item1 == position).item2 + as ParseResult; + } + + void memoize(Parser parser, int position, ParseResult? result) { + if (result != null) { + var list = _memo.putIfAbsent(parser, () => []); + var tuple = Tuple2(position, result); + if (!list.contains(tuple)) list.add(tuple); + } + } + + bool isActive(Parser parser, int position) { + if (!_active.containsKey(parser)) { + return false; + } + var q = _active[parser]!; + if (q.isEmpty) return false; + //return q.contains(position); + return q.first == position; + } + + void enter(Parser parser, int position) { + _active.putIfAbsent(parser, () => Queue()).addFirst(position); + } + + void exit(Parser parser) { + if (_active.containsKey(parser)) _active[parser]?.removeFirst(); + } +} + +/// The result generated by a [Parser]. +class ParseResult { + final Parser parser; + final bool successful; + final Iterable errors; + final FileSpan? span; + final T? value; + final SpanScanner scanner; + final Trampoline trampoline; + + ParseResult( + this.trampoline, this.scanner, this.parser, this.successful, this.errors, + {this.span, this.value}); + + ParseResult change( + {Parser? parser, + bool? successful, + Iterable errors = const [], + FileSpan? span, + T? value}) { + return ParseResult( + trampoline, + scanner, + parser ?? this.parser, + successful ?? this.successful, + errors.isNotEmpty ? errors : this.errors, + span: span ?? this.span, + value: value ?? this.value, + ); + } + + ParseResult addErrors(Iterable errors) { + return change( + errors: List.from(this.errors)..addAll(errors), + ); + } +} diff --git a/packages/combinator/lib/src/combinator/compare.dart b/packages/combinator/lib/src/combinator/compare.dart new file mode 100644 index 0000000..3a5e36d --- /dev/null +++ b/packages/combinator/lib/src/combinator/compare.dart @@ -0,0 +1,38 @@ +part of lex.src.combinator; + +class _Compare extends ListParser { + final ListParser parser; + final Comparator compare; + + _Compare(this.parser, this.compare); + + @override + ParseResult> __parse(ParseArgs args) { + var result = parser._parse(args.increaseDepth()); + if (!result.successful) return result; + + result = result.change( + value: result.value?.isNotEmpty == true ? result.value : []); + result = result.change(value: List.from(result.value!)); + return ParseResult>( + args.trampoline, + args.scanner, + this, + true, + [], + span: result.span, + value: result.value?..sort(compare), + ); + } + + @override + void stringify(CodeBuffer buffer) { + buffer + ..writeln('sort($compare) (') + ..indent(); + parser.stringify(buffer); + buffer + ..outdent() + ..writeln(')'); + } +} diff --git a/packages/combinator/lib/src/combinator/fold_errors.dart b/packages/combinator/lib/src/combinator/fold_errors.dart new file mode 100644 index 0000000..6d15c69 --- /dev/null +++ b/packages/combinator/lib/src/combinator/fold_errors.dart @@ -0,0 +1,29 @@ +part of lex.src.combinator; + +class _FoldErrors extends Parser { + final Parser parser; + final bool Function(SyntaxError, SyntaxError) equal; + + _FoldErrors(this.parser, this.equal); + + @override + ParseResult __parse(ParseArgs args) { + var result = parser._parse(args.increaseDepth()).change(parser: this); + var errors = result.errors.fold>([], (out, e) { + if (!out.any((b) => equal(e, b))) out.add(e); + return out; + }); + return result.change(errors: errors); + } + + @override + void stringify(CodeBuffer buffer) { + buffer + ..writeln('fold errors (') + ..indent(); + parser.stringify(buffer); + buffer + ..outdent() + ..writeln(')'); + } +} diff --git a/packages/combinator/lib/src/combinator/index.dart b/packages/combinator/lib/src/combinator/index.dart new file mode 100644 index 0000000..41d4460 --- /dev/null +++ b/packages/combinator/lib/src/combinator/index.dart @@ -0,0 +1,52 @@ +part of lex.src.combinator; + +class _Index extends Parser { + final ListParser parser; + final int index; + + _Index(this.parser, this.index); + + @override + ParseResult __parse(ParseArgs args) { + var result = parser._parse(args.increaseDepth()); + Object? value; + + if (result.successful) { + var vList = result.value; + if (vList == null) { + throw ArgumentError('ParseResult is null'); + } + if (index == -1) { + value = vList.last; + } else { + if (index < vList.length) { +// print(">>>>Index: $index, Size: ${vList.length}"); +// value = +// index == -1 ? result.value!.last : result.value!.elementAt(index); + value = result.value!.elementAt(index); + } + } + } + + return ParseResult( + args.trampoline, + args.scanner, + this, + result.successful, + result.errors, + span: result.span, + value: value as T?, + ); + } + + @override + void stringify(CodeBuffer buffer) { + buffer + ..writeln('index($index) (') + ..indent(); + parser.stringify(buffer); + buffer + ..outdent() + ..writeln(')'); + } +} diff --git a/packages/combinator/lib/src/combinator/longest.dart b/packages/combinator/lib/src/combinator/longest.dart new file mode 100644 index 0000000..36d9652 --- /dev/null +++ b/packages/combinator/lib/src/combinator/longest.dart @@ -0,0 +1,115 @@ +part of lex.src.combinator; + +/// Matches any one of the given [parsers]. +/// +/// You can provide a custom [errorMessage]. +Parser longest(Iterable> parsers, + {Object? errorMessage, SyntaxErrorSeverity? severity}) { + return _Longest(parsers, errorMessage, severity ?? SyntaxErrorSeverity.error); +} + +class _Longest extends Parser { + final Iterable> parsers; + final Object? errorMessage; + final SyntaxErrorSeverity severity; + + _Longest(this.parsers, this.errorMessage, this.severity); + + @override + ParseResult _parse(ParseArgs args) { + var inactive = parsers + .toList() + .where((p) => !args.trampoline.isActive(p, args.scanner.position)); + + if (inactive.isEmpty) { + return ParseResult(args.trampoline, args.scanner, this, false, []); + } + + var replay = args.scanner.position; + var errors = []; + var results = >[]; + + for (var parser in inactive) { + var result = parser._parse(args.increaseDepth()); + + if (result.successful && result.span != null) { + results.add(result); + } else if (parser is _Alt) errors.addAll(result.errors); + + args.scanner.position = replay; + } + + if (results.isNotEmpty) { + results.sort((a, b) => b.span!.length.compareTo(a.span!.length)); + args.scanner.scan(results.first.span!.text); + return results.first; + } + + if (errorMessage != false) { + errors.add( + SyntaxError( + severity, + errorMessage?.toString() ?? + 'No match found for ${parsers.length} alternative(s)', + args.scanner.emptySpan, + ), + ); + } + + return ParseResult(args.trampoline, args.scanner, this, false, errors); + } + + @override + ParseResult __parse(ParseArgs args) { + var replay = args.scanner.position; + var errors = []; + var results = >[]; + + for (var parser in parsers) { + var result = parser._parse(args.increaseDepth()); + + if (result.successful) { + results.add(result); + } else if (parser is _Alt) errors.addAll(result.errors); + + args.scanner.position = replay; + } + + if (results.isNotEmpty) { + results.sort((a, b) => b.span!.length.compareTo(a.span!.length)); + args.scanner.scan(results.first.span!.text); + return results.first; + } + + errors.add( + SyntaxError( + severity, + errorMessage?.toString() ?? + 'No match found for ${parsers.length} alternative(s)', + args.scanner.emptySpan, + ), + ); + + return ParseResult(args.trampoline, args.scanner, this, false, errors); + } + + @override + void stringify(CodeBuffer buffer) { + buffer + ..writeln('longest(${parsers.length}) (') + ..indent(); + var i = 1; + + for (var parser in parsers) { + buffer + ..writeln('#${i++}:') + ..indent(); + parser.stringify(buffer); + buffer.outdent(); + } + + buffer + ..outdent() + ..writeln(')'); + } +} diff --git a/packages/combinator/lib/src/combinator/map.dart b/packages/combinator/lib/src/combinator/map.dart new file mode 100644 index 0000000..4dfb326 --- /dev/null +++ b/packages/combinator/lib/src/combinator/map.dart @@ -0,0 +1,56 @@ +part of lex.src.combinator; + +class _Map extends Parser { + final Parser parser; + final U Function(ParseResult) f; + + _Map(this.parser, this.f); + + @override + ParseResult __parse(ParseArgs args) { + var result = parser._parse(args.increaseDepth()); + return ParseResult( + args.trampoline, + args.scanner, + this, + result.successful, + result.errors, + span: result.span, + value: result.successful ? f(result) : null, + ); + } + + @override + void stringify(CodeBuffer buffer) { + buffer + ..writeln('map<$U> (') + ..indent(); + parser.stringify(buffer); + buffer + ..outdent() + ..writeln(')'); + } +} + +class _Change extends Parser { + final Parser parser; + final ParseResult Function(ParseResult) f; + + _Change(this.parser, this.f); + + @override + ParseResult __parse(ParseArgs args) { + return f(parser._parse(args.increaseDepth())).change(parser: this); + } + + @override + void stringify(CodeBuffer buffer) { + buffer + ..writeln('change($f) (') + ..indent(); + parser.stringify(buffer); + buffer + ..outdent() + ..writeln(')'); + } +} diff --git a/packages/combinator/lib/src/combinator/match.dart b/packages/combinator/lib/src/combinator/match.dart new file mode 100644 index 0000000..837c306 --- /dev/null +++ b/packages/combinator/lib/src/combinator/match.dart @@ -0,0 +1,41 @@ +part of lex.src.combinator; + +/// Expects to match a given [pattern]. If it is not matched, you can provide a custom [errorMessage]. +Parser match(Pattern pattern, + {String? errorMessage, SyntaxErrorSeverity? severity}) => + _Match(pattern, errorMessage, severity ?? SyntaxErrorSeverity.error); + +class _Match extends Parser { + final Pattern pattern; + final String? errorMessage; + final SyntaxErrorSeverity severity; + + _Match(this.pattern, this.errorMessage, this.severity); + + @override + ParseResult __parse(ParseArgs args) { + var scanner = args.scanner; + if (!scanner.scan(pattern)) { + return ParseResult(args.trampoline, scanner, this, false, [ + SyntaxError( + severity, + errorMessage ?? 'Expected "$pattern".', + scanner.emptySpan, + ), + ]); + } + return ParseResult( + args.trampoline, + scanner, + this, + true, + [], + span: scanner.lastSpan, + ); + } + + @override + void stringify(CodeBuffer buffer) { + buffer.writeln('match($pattern)'); + } +} diff --git a/packages/combinator/lib/src/combinator/max_depth.dart b/packages/combinator/lib/src/combinator/max_depth.dart new file mode 100644 index 0000000..42befcf --- /dev/null +++ b/packages/combinator/lib/src/combinator/max_depth.dart @@ -0,0 +1,28 @@ +part of lex.src.combinator; + +class _MaxDepth extends Parser { + final Parser parser; + final int cap; + + _MaxDepth(this.parser, this.cap); + + @override + ParseResult __parse(ParseArgs args) { + if (args.depth > cap) { + return ParseResult(args.trampoline, args.scanner, this, false, []); + } + + return parser._parse(args.increaseDepth()); + } + + @override + void stringify(CodeBuffer buffer) { + buffer + ..writeln('max depth($cap) (') + ..indent(); + parser.stringify(buffer); + buffer + ..outdent() + ..writeln(')'); + } +} diff --git a/packages/combinator/lib/src/combinator/negate.dart b/packages/combinator/lib/src/combinator/negate.dart new file mode 100644 index 0000000..b507bdc --- /dev/null +++ b/packages/combinator/lib/src/combinator/negate.dart @@ -0,0 +1,51 @@ +part of lex.src.combinator; + +class _Negate extends Parser { + final Parser parser; + final String? errorMessage; + final SyntaxErrorSeverity severity; + + _Negate(this.parser, this.errorMessage, this.severity); + + @override + ParseResult __parse(ParseArgs args) { + var result = parser._parse(args.increaseDepth()).change(parser: this); + + if (!result.successful) { + return ParseResult( + args.trampoline, + args.scanner, + this, + true, + [], + span: result.span ?? args.scanner.lastSpan ?? args.scanner.emptySpan, + value: result.value, + ); + } + + result = result.change(successful: false); + + if (errorMessage != null) { + result = result.addErrors([ + SyntaxError( + severity, + errorMessage, + result.span, + ), + ]); + } + + return result; + } + + @override + void stringify(CodeBuffer buffer) { + buffer + ..writeln('negate (') + ..indent(); + parser.stringify(buffer); + buffer + ..outdent() + ..writeln(')'); + } +} diff --git a/packages/combinator/lib/src/combinator/opt.dart b/packages/combinator/lib/src/combinator/opt.dart new file mode 100644 index 0000000..3f31c99 --- /dev/null +++ b/packages/combinator/lib/src/combinator/opt.dart @@ -0,0 +1,57 @@ +part of lex.src.combinator; + +class _Opt extends Parser { + final Parser parser; + final bool backtrack; + + _Opt(this.parser, this.backtrack); + + @override + ParseResult __parse(ParseArgs args) { + var replay = args.scanner.position; + var result = parser._parse(args.increaseDepth()); + + if (!result.successful) args.scanner.position = replay; + + return result.change(parser: this, successful: true); + } + + @override + void stringify(CodeBuffer buffer) { + buffer + ..writeln('optional (') + ..indent(); + parser.stringify(buffer); + buffer + ..outdent() + ..writeln(')'); + } +} + +class _ListOpt extends ListParser { + final ListParser parser; + final bool backtrack; + + _ListOpt(this.parser, this.backtrack); + + @override + ParseResult> __parse(ParseArgs args) { + var replay = args.scanner.position; + var result = parser._parse(args.increaseDepth()); + + if (!result.successful) args.scanner.position = replay; + + return result.change(parser: this, successful: true); + } + + @override + void stringify(CodeBuffer buffer) { + buffer + ..writeln('optional (') + ..indent(); + parser.stringify(buffer); + buffer + ..outdent() + ..writeln(')'); + } +} diff --git a/packages/combinator/lib/src/combinator/recursion.dart b/packages/combinator/lib/src/combinator/recursion.dart new file mode 100644 index 0000000..7651694 --- /dev/null +++ b/packages/combinator/lib/src/combinator/recursion.dart @@ -0,0 +1,142 @@ +part of lex.src.combinator; + +/* +/// Handles left recursion in a grammar using the Pratt algorithm. +class Recursion { + Iterable> prefix; + Map)> infix; + Map)> postfix; + + Recursion({this.prefix, this.infix, this.postfix}) { + prefix ??= []; + infix ??= {}; + postfix ??= {}; + } + + Parser precedence(int p) => _Precedence(this, p); + + void stringify(CodeBuffer buffer) { + buffer + ..writeln('recursion (') + ..indent() + ..writeln('prefix(${prefix.length}') + ..writeln('infix(${infix.length}') + ..writeln('postfix(${postfix.length}') + ..outdent() + ..writeln(')'); + } +} + +class _Precedence extends Parser { + final Recursion r; + final int precedence; + + _Precedence(this.r, this.precedence); + + @override + ParseResult __parse(ParseArgs args) { + int replay = args.scanner.position; + var errors = []; + var start = args.scanner.state; + var reversedKeys = r.infix.keys.toList().reversed; + + for (var pre in r.prefix) { + var result = pre._parse(args.increaseDepth()), originalResult = result; + + if (!result.successful) { + if (pre is _Alt) errors.addAll(result.errors); + args.scanner.position = replay; + } else { + var left = result.value; + replay = args.scanner.position; + //print('${result.span.text}:\n' + scanner.emptySpan.highlight()); + + while (true) { + bool matched = false; + + //for (int i = 0; i < r.infix.length; i++) { + for (int i = r.infix.length - 1; i >= 0; i--) { + //var fix = r.infix.keys.elementAt(r.infix.length - i - 1); + var fix = reversedKeys.elementAt(i); + + if (i < precedence) continue; + + var result = fix._parse(args.increaseDepth()); + + if (!result.successful) { + if (fix is _Alt) errors.addAll(result.errors); + // If this is the last alternative and it failed, don't continue looping. + //if (true || i + 1 < r.infix.length) + args.scanner.position = replay; + } else { + //print('FOUND $fix when left was $left'); + //print('$i vs $precedence\n${originalResult.span.highlight()}'); + result = r.precedence(i)._parse(args.increaseDepth()); + + if (!result.successful) { + } else { + matched = false; + var old = left; + left = r.infix[fix](left, result.value, result); + print( + '$old $fix ${result.value} = $left\n${result.span.highlight()}'); + break; + } + } + } + + if (!matched) break; + } + + replay = args.scanner.position; + //print('f ${result.span.text}'); + + for (var post in r.postfix.keys) { + var result = pre._parse(args.increaseDepth()); + + if (!result.successful) { + if (post is _Alt) errors.addAll(result.errors); + args.scanner.position = replay; + } else { + left = r.infix[post](left, originalResult.value, result); + } + } + + if (!args.scanner.isDone) { + // If we're not done scanning, then we need some sort of guard to ensure the + // that this exact parser does not run again in the exact position. + } + return ParseResult( + args.trampoline, + args.scanner, + this, + true, + errors, + value: left, + span: args.scanner.spanFrom(start), + ); + } + } + + return ParseResult( + args.trampoline, + args.scanner, + this, + false, + errors, + span: args.scanner.spanFrom(start), + ); + } + + @override + void stringify(CodeBuffer buffer) { + buffer + ..writeln('precedence($precedence) (') + ..indent(); + r.stringify(buffer); + buffer + ..outdent() + ..writeln(')'); + } +} +*/ diff --git a/packages/combinator/lib/src/combinator/reduce.dart b/packages/combinator/lib/src/combinator/reduce.dart new file mode 100644 index 0000000..a8779b4 --- /dev/null +++ b/packages/combinator/lib/src/combinator/reduce.dart @@ -0,0 +1,46 @@ +part of lex.src.combinator; + +class _Reduce extends Parser { + final ListParser parser; + final T Function(T, T) combine; + + _Reduce(this.parser, this.combine); + + @override + ParseResult __parse(ParseArgs args) { + var result = parser._parse(args.increaseDepth()); + + if (!result.successful) { + return ParseResult( + args.trampoline, + args.scanner, + this, + false, + result.errors, + ); + } + + result = result.change( + value: result.value?.isNotEmpty == true ? result.value : []); + return ParseResult( + args.trampoline, + args.scanner, + this, + result.successful, + [], + span: result.span, + value: result.value!.isEmpty ? null : result.value!.reduce(combine), + ); + } + + @override + void stringify(CodeBuffer buffer) { + buffer + ..writeln('reduce($combine) (') + ..indent(); + parser.stringify(buffer); + buffer + ..outdent() + ..writeln(')'); + } +} diff --git a/packages/combinator/lib/src/combinator/reference.dart b/packages/combinator/lib/src/combinator/reference.dart new file mode 100644 index 0000000..5088c4a --- /dev/null +++ b/packages/combinator/lib/src/combinator/reference.dart @@ -0,0 +1,44 @@ +part of lex.src.combinator; + +Reference reference() => Reference._(); + +class Reference extends Parser { + Parser? _parser; + bool printed = false; + + Reference._(); + + set parser(Parser value) { + if (_parser != null) { + throw StateError('There is already a parser assigned to this reference.'); + } + _parser = value; + } + + @override + ParseResult __parse(ParseArgs args) { + if (_parser == null) { + throw StateError('There is no parser assigned to this reference.'); + } + return _parser!._parse(args); + } + + @override + ParseResult _parse(ParseArgs args) { + if (_parser == null) { + throw StateError('There is no parser assigned to this reference.'); + } + return _parser!._parse(args); + } + + @override + void stringify(CodeBuffer buffer) { + if (_parser == null) { + buffer.writeln('(undefined reference <$T>)'); + } else if (!printed) { + _parser!.stringify(buffer); + } + printed = true; + buffer.writeln('(previously printed reference)'); + } +} diff --git a/packages/combinator/lib/src/combinator/repeat.dart b/packages/combinator/lib/src/combinator/repeat.dart new file mode 100644 index 0000000..13dd9f2 --- /dev/null +++ b/packages/combinator/lib/src/combinator/repeat.dart @@ -0,0 +1,89 @@ +part of lex.src.combinator; + +class _Repeat extends ListParser { + final Parser parser; + final int count; + final bool exact, backtrack; + final String tooFew; + final String tooMany; + final SyntaxErrorSeverity severity; + + _Repeat(this.parser, this.count, this.exact, this.tooFew, this.tooMany, + this.backtrack, this.severity); + + @override + ParseResult> __parse(ParseArgs args) { + var errors = []; + var results = []; + var spans = []; + var success = 0; + var replay = args.scanner.position; + ParseResult result; + + do { + result = parser._parse(args.increaseDepth()); + if (result.successful) { + success++; + if (result.value != null) { + results.add(result.value!); + } + replay = args.scanner.position; + } else if (backtrack) args.scanner.position = replay; + + if (result.span != null) { + spans.add(result.span!); + } + } while (result.successful); + + if (success < count) { + errors.addAll(result.errors); + errors.add( + SyntaxError( + severity, + tooFew, + result.span ?? args.scanner.emptySpan, + ), + ); + + if (backtrack) args.scanner.position = replay; + + return ParseResult>( + args.trampoline, args.scanner, this, false, errors); + } else if (success > count && exact) { + if (backtrack) args.scanner.position = replay; + + return ParseResult>(args.trampoline, args.scanner, this, false, [ + SyntaxError( + severity, + tooMany, + result.span ?? args.scanner.emptySpan, + ), + ]); + } + + var span = spans.reduce((a, b) => a.expand(b)); + return ParseResult>( + args.trampoline, + args.scanner, + this, + true, + [], + span: span, + value: results, + ); + } + + @override + void stringify(CodeBuffer buffer) { + var r = StringBuffer('{$count'); + if (!exact) r.write(','); + r.write('}'); + buffer + ..writeln('repeat($r) (') + ..indent(); + parser.stringify(buffer); + buffer + ..outdent() + ..writeln(')'); + } +} diff --git a/packages/combinator/lib/src/combinator/safe.dart b/packages/combinator/lib/src/combinator/safe.dart new file mode 100644 index 0000000..a254229 --- /dev/null +++ b/packages/combinator/lib/src/combinator/safe.dart @@ -0,0 +1,47 @@ +part of lex.src.combinator; + +class _Safe extends Parser { + final Parser parser; + final bool backtrack; + final String errorMessage; + final SyntaxErrorSeverity severity; + bool _triggered = false; + + _Safe(this.parser, this.backtrack, this.errorMessage, this.severity); + + @override + ParseResult __parse(ParseArgs args) { + var replay = args.scanner.position; + + try { + if (_triggered) throw Exception(); + return parser._parse(args.increaseDepth()); + } catch (_) { + _triggered = true; + if (backtrack) args.scanner.position = replay; + var errors = []; + + errors.add( + SyntaxError( + severity, + errorMessage, + args.scanner.lastSpan ?? args.scanner.emptySpan, + ), + ); + + return ParseResult(args.trampoline, args.scanner, this, false, errors); + } + } + + @override + void stringify(CodeBuffer buffer) { + var t = _triggered ? 'triggered' : 'not triggered'; + buffer + ..writeln('safe($t) (') + ..indent(); + parser.stringify(buffer); + buffer + ..outdent() + ..writeln(')'); + } +} diff --git a/packages/combinator/lib/src/combinator/to_list.dart b/packages/combinator/lib/src/combinator/to_list.dart new file mode 100644 index 0000000..320be1f --- /dev/null +++ b/packages/combinator/lib/src/combinator/to_list.dart @@ -0,0 +1,41 @@ +part of lex.src.combinator; + +class _ToList extends ListParser { + final Parser parser; + + _ToList(this.parser); + + @override + ParseResult> __parse(ParseArgs args) { + var result = parser._parse(args.increaseDepth()); + + if (result.value is List) { + return (result as ParseResult>).change(parser: this); + } + + var values = []; + if (result.value != null) { + values.add(result.value!); + } + return ParseResult( + args.trampoline, + args.scanner, + this, + result.successful, + result.errors, + span: result.span, + value: values, + ); + } + + @override + void stringify(CodeBuffer buffer) { + buffer + ..writeln('to list (') + ..indent(); + parser.stringify(buffer); + buffer + ..outdent() + ..writeln(')'); + } +} diff --git a/packages/combinator/lib/src/combinator/util.dart b/packages/combinator/lib/src/combinator/util.dart new file mode 100644 index 0000000..cb5dcdc --- /dev/null +++ b/packages/combinator/lib/src/combinator/util.dart @@ -0,0 +1,57 @@ +part of lex.src.combinator; + +/// A typed parser that parses a sequence of 2 values of different types. +Parser> tuple2(Parser a, Parser b) { + return chain([a, b]).map((r) { + return Tuple2(r.value?[0] as A, r.value?[1] as B); + }); +} + +/// A typed parser that parses a sequence of 3 values of different types. +Parser> tuple3(Parser a, Parser b, Parser c) { + return chain([a, b, c]).map((r) { + return Tuple3(r.value?[0] as A, r.value?[1] as B, r.value?[2] as C); + }); +} + +/// A typed parser that parses a sequence of 4 values of different types. +Parser> tuple4( + Parser a, Parser b, Parser c, Parser d) { + return chain([a, b, c, d]).map((r) { + return Tuple4( + r.value?[0] as A, r.value?[1] as B, r.value?[2] as C, r.value?[3] as D); + }); +} + +/// A typed parser that parses a sequence of 5 values of different types. +Parser> tuple5( + Parser a, Parser b, Parser c, Parser d, Parser e) { + return chain([a, b, c, d, e]).map((r) { + return Tuple5(r.value?[0] as A, r.value?[1] as B, r.value?[2] as C, + r.value?[3] as D, r.value?[4] as E); + }); +} + +/// A typed parser that parses a sequence of 6 values of different types. +Parser> tuple6(Parser a, + Parser b, Parser c, Parser d, Parser e, Parser f) { + return chain([a, b, c, d, e, f]).map((r) { + return Tuple6(r.value?[0] as A, r.value?[1] as B, r.value?[2] as C, + r.value?[3] as D, r.value?[4] as E, r.value?[5] as F); + }); +} + +/// A typed parser that parses a sequence of 7 values of different types. +Parser> tuple7( + Parser a, + Parser b, + Parser c, + Parser d, + Parser e, + Parser f, + Parser g) { + return chain([a, b, c, d, e, f, g]).map((r) { + return Tuple7(r.value?[0] as A, r.value?[1] as B, r.value?[2] as C, + r.value?[3] as D, r.value?[4] as E, r.value?[5] as F, r.value?[6] as G); + }); +} diff --git a/packages/combinator/lib/src/combinator/value.dart b/packages/combinator/lib/src/combinator/value.dart new file mode 100644 index 0000000..b061207 --- /dev/null +++ b/packages/combinator/lib/src/combinator/value.dart @@ -0,0 +1,25 @@ +part of lex.src.combinator; + +class _Value extends Parser { + final Parser parser; + final T Function(ParseResult) f; + + _Value(this.parser, this.f); + + @override + ParseResult __parse(ParseArgs args) { + var result = parser._parse(args.increaseDepth()).change(parser: this); + return result.successful ? result.change(value: f(result)) : result; + } + + @override + void stringify(CodeBuffer buffer) { + buffer + ..writeln('set value($f) (') + ..indent(); + parser.stringify(buffer); + buffer + ..outdent() + ..writeln(')'); + } +} diff --git a/packages/combinator/lib/src/error.dart b/packages/combinator/lib/src/error.dart new file mode 100644 index 0000000..6c7b5f5 --- /dev/null +++ b/packages/combinator/lib/src/error.dart @@ -0,0 +1,23 @@ +import 'package:source_span/source_span.dart'; + +class SyntaxError implements Exception { + final SyntaxErrorSeverity severity; + final String? message; + final FileSpan? span; + String? _toolString; + + SyntaxError(this.severity, this.message, this.span); + + String? get toolString { + if (_toolString != null) return _toolString; + var type = severity == SyntaxErrorSeverity.warning ? 'warning' : 'error'; + return _toolString = '$type: ${span!.start.toolString}: $message'; + } +} + +enum SyntaxErrorSeverity { + warning, + error, + info, + hint, +} diff --git a/packages/combinator/pubspec.yaml b/packages/combinator/pubspec.yaml new file mode 100644 index 0000000..192eee2 --- /dev/null +++ b/packages/combinator/pubspec.yaml @@ -0,0 +1,15 @@ +name: belatuk_combinator +version: 3.0.0 +description: Packrat parser combinators that support static typing, generics, file spans, memoization, and more. +homepage: https://github.com/dart-backend/belatuk-common-utilities/tree/main/packages/combinator +environment: + sdk: '>=2.12.0 <3.0.0' +dependencies: + belatuk_code_buffer: ^3.0.0 + matcher: ^0.12.10 + source_span: ^1.8.1 + string_scanner: ^1.1.0 + tuple: ^2.0.0 +dev_dependencies: + test: ^1.17.4 + lints: ^1.0.0 diff --git a/packages/combinator/test/all.dart b/packages/combinator/test/all.dart new file mode 100644 index 0000000..9022d5f --- /dev/null +++ b/packages/combinator/test/all.dart @@ -0,0 +1,12 @@ +import 'package:test/test.dart'; +import 'list_test.dart' as list; +import 'match_test.dart' as match; +import 'misc_test.dart' as misc; +import 'value_test.dart' as value; + +void main() { + group('list', list.main); + group('match', match.main); + group('value', value.main); + misc.main(); +} diff --git a/packages/combinator/test/common.dart b/packages/combinator/test/common.dart new file mode 100644 index 0000000..9ccc542 --- /dev/null +++ b/packages/combinator/test/common.dart @@ -0,0 +1,3 @@ +import 'package:string_scanner/string_scanner.dart'; + +SpanScanner scan(String text) => SpanScanner(text); diff --git a/packages/combinator/test/list_test.dart b/packages/combinator/test/list_test.dart new file mode 100644 index 0000000..636c322 --- /dev/null +++ b/packages/combinator/test/list_test.dart @@ -0,0 +1,22 @@ +import 'package:angel3_combinator/belatuk_combinator.dart'; +import 'package:test/test.dart'; +import 'common.dart'; + +void main() { + var number = chain([ + match(RegExp(r'[0-9]+')).value((r) => int.parse(r.span!.text)), + match(',').opt(), + ]).first().cast(); + + var numbers = number.plus(); + + test('sort', () { + var parser = numbers.sort((a, b) => a.compareTo(b)); + expect(parser.parse(scan('21,2,3,34,20')).value, [2, 3, 20, 21, 34]); + }); + test('reduce', () { + var parser = numbers.reduce((a, b) => a + b); + expect(parser.parse(scan('21,2,3,34,20')).value, 80); + expect(parser.parse(scan('not numbers')).value, isNull); + }); +} diff --git a/packages/combinator/test/match_test.dart b/packages/combinator/test/match_test.dart new file mode 100644 index 0000000..6fb55e8 --- /dev/null +++ b/packages/combinator/test/match_test.dart @@ -0,0 +1,16 @@ +import 'package:angel3_combinator/belatuk_combinator.dart'; +import 'package:test/test.dart'; +import 'common.dart'; + +void main() { + test('match string', () { + expect(match('hello').parse(scan('hello world')).successful, isTrue); + }); + test('match start only', () { + expect(match('hello').parse(scan('goodbye hello')).successful, isFalse); + }); + + test('fail if no match', () { + expect(match('hello').parse(scan('world')).successful, isFalse); + }); +} diff --git a/packages/combinator/test/misc_test.dart b/packages/combinator/test/misc_test.dart new file mode 100644 index 0000000..7b08e77 --- /dev/null +++ b/packages/combinator/test/misc_test.dart @@ -0,0 +1,66 @@ +import 'package:angel3_combinator/belatuk_combinator.dart'; +import 'package:matcher/matcher.dart'; +import 'package:test/test.dart'; +import 'common.dart'; + +void main() { + test('advance', () { + var scanner = scan('hello world'); + + // Casted -> dynamic just for the sake of coverage. + var parser = match('he').forward(2).castDynamic(); + parser.parse(scanner); + expect(scanner.position, 4); + }); + + test('change', () { + var parser = match('hello').change((r) => r.change(value: 23)); + expect(parser.parse(scan('helloworld')).value, 23); + }); + + test('check', () { + var parser = match(RegExp(r'[A-Za-z]+')) + .value((r) => r.span!.length) + .check(greaterThan(3)); + expect(parser.parse(scan('helloworld')).successful, isTrue); + expect(parser.parse(scan('yo')).successful, isFalse); + }); + + test('map', () { + var parser = match(RegExp(r'[A-Za-z]+')).map((r) => r.span!.length); + expect(parser.parse(scan('hello')).value, 5); + }); + + test('negate', () { + var parser = match('hello').negate(errorMessage: 'world'); + expect(parser.parse(scan('goodbye world')).successful, isTrue); + expect(parser.parse(scan('hello world')).successful, isFalse); + expect(parser.parse(scan('hello world')).errors.first.message, 'world'); + }); + + group('opt', () { + var single = match('hello').opt(backtrack: true); + var list = match('hel').then(match('lo')).opt(); + + test('succeeds if present', () { + expect(single.parse(scan('hello')).successful, isTrue); + expect(list.parse(scan('hello')).successful, isTrue); + }); + + test('succeeds if not present', () { + expect(single.parse(scan('goodbye')).successful, isTrue); + expect(list.parse(scan('goodbye')).successful, isTrue); + }); + + test('backtracks if not present', () { + for (var parser in [single, list]) { + var scanner = scan('goodbye'); + var pos = scanner.position; + parser.parse(scanner); + expect(scanner.position, pos); + } + }); + }); + + test('safe', () {}); +} diff --git a/packages/combinator/test/recursion_test.dart b/packages/combinator/test/recursion_test.dart new file mode 100644 index 0000000..d91cebb --- /dev/null +++ b/packages/combinator/test/recursion_test.dart @@ -0,0 +1,53 @@ +void main() {} + +/* +void main() { + var number = match( RegExp(r'-?[0-9]+(\.[0-9]+)?')) + .map((r) => num.parse(r.span.text)); + + var term = reference(); + + var r = Recursion(); + + r.prefix = [number]; + + r.infix.addAll({ + match('*'): (l, r, _) => l * r, + match('/'): (l, r, _) => l / r, + match('+'): (l, r, _) => l + r, + match('-'): (l, r, _) => l - r, + + + match('-'): (l, r, _) => l - r, + match('+'): (l, r, _) => l + r, + match('/'): (l, r, _) => l / r, + match('*'): (l, r, _) => l * r, + + }); + + term.parser = r.precedence(0); + + num parse(String text) { + var scanner = SpanScanner(text); + var result = term.parse(scanner); + print(result.span.highlight()); + return result.value; + } + + test('prefix', () { + expect(parse('24'), 24); + }); + + test('infix', () { + expect(parse('12/6'), 2); + expect(parse('24+23'), 47); + expect(parse('24-23'), 1); + expect(parse('4*3'), 12); + }); + + test('precedence', () { + expect(parse('2+3*5*2'), 15); + //expect(parse('2+3+5-2*2'), 15); + }); +} +*/ diff --git a/packages/combinator/test/value_test.dart b/packages/combinator/test/value_test.dart new file mode 100644 index 0000000..5bd8645 --- /dev/null +++ b/packages/combinator/test/value_test.dart @@ -0,0 +1,15 @@ +import 'package:angel3_combinator/belatuk_combinator.dart'; +import 'package:test/test.dart'; +import 'common.dart'; + +void main() { + var parser = match('hello').value((r) => 'world'); + + test('sets value', () { + expect(parser.parse(scan('hello world')).value, 'world'); + }); + + test('no value if no match', () { + expect(parser.parse(scan('goodbye world')).value, isNull); + }); +} diff --git a/packages/pub_sub/.travis.yml b/packages/pub_sub/.travis.yml new file mode 100644 index 0000000..de2210c --- /dev/null +++ b/packages/pub_sub/.travis.yml @@ -0,0 +1 @@ +language: dart \ No newline at end of file diff --git a/packages/pub_sub/AUTHORS.md b/packages/pub_sub/AUTHORS.md new file mode 100644 index 0000000..ac95ab5 --- /dev/null +++ b/packages/pub_sub/AUTHORS.md @@ -0,0 +1,12 @@ +Primary Authors +=============== + +* __[Thomas Hii](dukefirehawk.apps@gmail.com)__ + + Thomas is the current maintainer of the code base. He has refactored and migrated the + code base to support NNBD. + +* __[Tobe O](thosakwe@gmail.com)__ + + Tobe has written much of the original code prior to NNBD migration. He has moved on and + is no longer involved with the project. diff --git a/packages/pub_sub/CHANGELOG.md b/packages/pub_sub/CHANGELOG.md new file mode 100644 index 0000000..ce8548b --- /dev/null +++ b/packages/pub_sub/CHANGELOG.md @@ -0,0 +1,36 @@ +# Change Log + +## 4.0.0 + +* Upgraded from `pendantic` to `lints` linter +* Published as `belatuk_combinator` package + +## 3.0.2 + +* Resolved static analysis warnings + +## 3.0.1 + +* Resolved static analysis warnings + +## 3.0.0 + +* Migrated to work with Dart SDK 2.12.x NNBD + +## 2.3.0 + +* Allow `2.x` versions of `stream_channel`. +* Apply `package:pedantic` lints. + +## 2.2.0 + +* Upgrade `uuid`. + +## 2.1.0 + +* Allow for "trusted clients," which are implicitly-registered clients. +This makes using `package:pub_sub` easier, as well making it easier to scale. + +## 2.0.0 + +* Dart 2 updates. diff --git a/packages/pub_sub/LICENSE b/packages/pub_sub/LICENSE new file mode 100644 index 0000000..e37a346 --- /dev/null +++ b/packages/pub_sub/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2021, dukefirehawk.com +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/packages/pub_sub/README.md b/packages/pub_sub/README.md new file mode 100644 index 0000000..3a835df --- /dev/null +++ b/packages/pub_sub/README.md @@ -0,0 +1,230 @@ +# Belatuk Pub Sub + +[![version](https://img.shields.io/badge/pub-v4.0.0-brightgreen)](https://pub.dartlang.org/packages/belatuk_pub_sub) +[![Null Safety](https://img.shields.io/badge/null-safety-brightgreen)](https://dart.dev/null-safety) +[![License](https://img.shields.io/github/license/dart-backend/belatuk-common-utilities)](https://github.com/dart-backend/belatuk-common-utilities/packages/pub_sub/LICENSE) + +Keep application instances in sync with a simple pub/sub API. + +## Installation + +Add `belatuk_pub_sub` as a dependency in your `pubspec.yaml` file: + +```yaml +dependencies: + belatuk_pub_sub: ^4.0.0 +``` + +Then, be sure to run `pub get` in your terminal. + +## Usage + +`belatuk_pub_sub` is your typical pub/sub API. However, `belatuk_pub_sub` enforces authentication of every +request. It is very possible that `belatuk_pub_sub` will run on both servers and in the browser, +or on a platform belatuk_pub_sublike Flutter. Thus, there are provisions available to limit +access. + +**Be careful to not leak any `belatuk_pub_sub` client ID's if operating over a network.** +If you do, you risk malicious users injecting events into your application, which +could ultimately spell *disaster*. + +A `belatuk_pub_sub` server can operate across multiple *adapters*, which take care of interfacing data over different +media. For example, a single server can handle pub/sub between multiple Isolates and TCP Sockets, as well as +WebSockets, simultaneously. + +```dart +import 'package:belatuk_pub_sub/belatuk_pub_sub.dart' as pub_sub; + +main() async { + var server = pub_sub.Server([ + FooAdapter(...), + BarAdapter(...) + ]); + + server.addAdapter( BazAdapter(...)); + + // Call `start` to activate adapters, and begin handling requests. + server.start(); +} +``` + +### Trusted Clients + +You can use `package:belatuk_pub_sub` without explicitly registering +clients, *if and only if* those clients come from trusted sources. + +Clients via `Isolate` are always trusted. + +Clients via `package:json_rpc_2` must be explicitly marked +as trusted (i.e. using an IP whitelist mechanism): + +```dart +JsonRpc2Adapter(..., isTrusted: false); + +// Pass `null` as Client ID when trusted... +pub_sub.IsolateClient(null); +``` + +### Access Control + +The ID's of all *untrusted* clients who will connect to the server must be known at start-up time. +You may not register new clients after the server has started. This is mostly a security consideration; +if it is impossible to register new clients, then malicious users cannot grant themselves additional +privileges within the system. + +```dart +import 'package:belatuk_pub_sub/belatuk_pub_sub.dart' as pub_sub; + +void main() async { + // ... + server.registerClient(const ClientInfo('')); + + // Create a user who can subscribe, but not publish. + server.registerClient(const ClientInfo('', canPublish: false)); + + // Create a user who can publish, but not subscribe. + server.registerClient(const ClientInfo('', canSubscribe: false)); + + // Create a user with no privileges whatsoever. + server.registerClient(const ClientInfo('', canPublish: false, canSubscribe: false)); + + server.start(); +} +``` + +### Isolates + +If you are just running multiple instances of a server, +use `package:belatuk_pub_sub/isolate.dart`. + +You'll need one isolate to be the master. Typically this is the first isolate you create. + +```dart +import 'dart:io'; +import 'dart:isolate'; +import 'package:belatuk_pub_sub/isolate.dart' as pub_sub; +import 'package:belatuk_pub_sub/belatuk_pub_sub.dart' as pub_sub; + +void main() async { + // Easily bring up a server. + var adapter = pub_sub.IsolateAdapter(); + var server = pub_sub.Server([adapter]); + + // You then need to create a client that will connect to the adapter. + // Each isolate in your application should contain a client. + for (int i = 0; i < Platform.numberOfProcessors - 1; i++) { + server.registerClient(pub_sub.ClientInfo('client$i')); + } + + // Start the server. + server.start(); + + // Next, let's start isolates that interact with the server. + // + // Fortunately, we can send SendPorts over Isolates, so this is no hassle. + for (int i = 0; i < Platform.numberOfProcessors - 1; i++) + Isolate.spawn(isolateMain, [i, adapter.receivePort.sendPort]); + + // It's possible that you're running your application in the server isolate as well: + isolateMain([0, adapter.receivePort.sendPort]); +} + +void isolateMain(List args) { + var client = + pub_sub.IsolateClient('client${args[0]}', args[1] as SendPort); + + // The client will connect automatically. In the meantime, we can start subscribing to events. + client.subscribe('user::logged_in').then((sub) { + // The `ClientSubscription` class extends `Stream`. Hooray for asynchrony! + sub.listen((msg) { + print('Logged in: $msg'); + }); + }); +} + +``` + +### JSON RPC 2.0 + +If you are not running on isolates, you need to import +`package:belatuk_pub_sub/json_rpc_2.dart`. This library leverages `package:json_rpc_2` and +`package:stream_channel` to create clients and servers that can hypothetically run on any +medium, i.e. WebSockets, or TCP Sockets. + +Check out `test/json_rpc_2_test.dart` for an example of serving `belatuk_pub_sub` over TCP sockets. + +## Protocol + +`belatuk_pub_sub` is built upon a simple RPC, and this package includes +an implementation that runs via `SendPort`s and `ReceivePort`s, as +well as one that runs on any `StreamChannel`. + +Data sent over the wire looks like the following: + +```typescript +// Sent by a client to initiate an exchange. +interface Request { + // This is an arbitrary string, assigned by your client, but in every case, + // the client uses this to match your requests with asynchronous responses. + request_id: string, + + // The ID of the client to authenticate as. + // + // As you can imagine, this should be kept secret, to prevent breaches. + client_id: string, + + // Required for *every* request. + params: { + // A value to be `publish`ed. + value?: any, + + // The name of an event to `publish`. + event_name?: string, + + // The ID of a subscription to be cancelled. + subscription_id?: string + } +} + +/// Sent by the server in response to a request. +interface Response { + // `true` for success, `false` for failures. + status: boolean, + + // Only appears if `status` is `false`; explains why an operation failed. + error_message?: string, + + // Matches the request_id sent by the client. + request_id: string, + + result?: { + // The number of other clients to whom an event was `publish`ed. + listeners:? number, + + // The ID of a created subscription. + subscription_id?: string + } +} +``` + +When sending via JSON_RPC 2.0, the `params` of a `Request` are simply folded into the object +itself, for simplicity's sake. In this case, a response will be sent as a notification whose +name is the `request_id`. + +In the case of Isolate clients/servers, events will be simply sent as Lists: + +```dart +['', value] +``` + +Clients can send the following (3) methods: + +* `subscribe` (`event_name`:string): Subscribe to an event. +* `unsubscribe` (`subscription_id`:string): Unsubscribe from an event you previously subscribed to. +* `publish` (`event_name`:string, `value`:any): Publish an event to all other clients who are subscribed. + +The client and server in `package:belatuk_pub_sub/isolate.dart` must make extra +provisions to keep track of client ID's. Since `SendPort`s and `ReceivePort`s +do not have any sort of guaranteed-unique ID's, new clients must send their +`SendPort` to the server before sending any requests. The server then responds +with an `id` that must be used to identify a `SendPort` to send a response to. diff --git a/packages/pub_sub/analysis_options.yaml b/packages/pub_sub/analysis_options.yaml new file mode 100644 index 0000000..ea2c9e9 --- /dev/null +++ b/packages/pub_sub/analysis_options.yaml @@ -0,0 +1 @@ +include: package:lints/recommended.yaml \ No newline at end of file diff --git a/packages/pub_sub/example/main.dart b/packages/pub_sub/example/main.dart new file mode 100644 index 0000000..42c7727 --- /dev/null +++ b/packages/pub_sub/example/main.dart @@ -0,0 +1,44 @@ +import 'dart:io'; +import 'dart:isolate'; +import 'package:belatuk_pub_sub/isolate.dart' as pub_sub; +import 'package:belatuk_pub_sub/belatuk_pub_sub.dart' as pub_sub; + +void main() async { + // Easily bring up a server. + var adapter = pub_sub.IsolateAdapter(); + var server = pub_sub.Server([adapter]); + + // You then need to create a client that will connect to the adapter. + // Every untrusted client in your application should be pre-registered. + // + // In the case of Isolates, however, those are always implicitly trusted. + for (var i = 0; i < Platform.numberOfProcessors - 1; i++) { + server.registerClient(pub_sub.ClientInfo('client$i')); + } + + // Start the server. + server.start(); + + // Next, let's start isolates that interact with the server. + // + // Fortunately, we can send SendPorts over Isolates, so this is no hassle. + for (var i = 0; i < Platform.numberOfProcessors - 1; i++) { + await Isolate.spawn(isolateMain, [i, adapter.receivePort.sendPort]); + } + + // It's possible that you're running your application in the server isolate as well: + isolateMain([0, adapter.receivePort.sendPort]); +} + +void isolateMain(List args) { + // Isolates are always trusted, so technically we don't need to pass a client iD. + var client = pub_sub.IsolateClient('client${args[0]}', args[1] as SendPort); + + // The client will connect automatically. In the meantime, we can start subscribing to events. + client.subscribe('user::logged_in').then((sub) { + // The `ClientSubscription` class extends `Stream`. Hooray for asynchrony! + sub.listen((msg) { + print('Logged in: $msg'); + }); + }); +} diff --git a/packages/pub_sub/lib/belatuk_pub_sub.dart b/packages/pub_sub/lib/belatuk_pub_sub.dart new file mode 100644 index 0000000..8450547 --- /dev/null +++ b/packages/pub_sub/lib/belatuk_pub_sub.dart @@ -0,0 +1 @@ +export 'src/protocol/protocol.dart'; diff --git a/packages/pub_sub/lib/isolate.dart b/packages/pub_sub/lib/isolate.dart new file mode 100644 index 0000000..0fcf44b --- /dev/null +++ b/packages/pub_sub/lib/isolate.dart @@ -0,0 +1,2 @@ +export 'src/isolate/client.dart'; +export 'src/isolate/server.dart'; diff --git a/packages/pub_sub/lib/json_rpc_2.dart b/packages/pub_sub/lib/json_rpc_2.dart new file mode 100644 index 0000000..41bd3b0 --- /dev/null +++ b/packages/pub_sub/lib/json_rpc_2.dart @@ -0,0 +1,2 @@ +export 'src/json_rpc/client.dart'; +export 'src/json_rpc/server.dart'; diff --git a/packages/pub_sub/lib/src/isolate/client.dart b/packages/pub_sub/lib/src/isolate/client.dart new file mode 100644 index 0000000..4fab48e --- /dev/null +++ b/packages/pub_sub/lib/src/isolate/client.dart @@ -0,0 +1,186 @@ +import 'dart:async'; +import 'dart:collection'; +import 'dart:isolate'; +import 'package:uuid/uuid.dart'; +import '../../belatuk_pub_sub.dart'; + +/// A [Client] implementation that communicates via [SendPort]s and [ReceivePort]s. +class IsolateClient extends Client { + final Queue> _onConnect = Queue>(); + final Map> _requests = {}; + final List<_IsolateClientSubscription> _subscriptions = []; + final Uuid _uuid = Uuid(); + + String? _id; + + /// The ID of the client we are authenticating as. + /// + /// May be `null`, if and only if we are marked as a trusted source on + /// the server side. + String? get clientId => _clientId; + String? _clientId; + + /// A server's [SendPort] that messages should be sent to. + final SendPort serverSendPort; + + /// A [ReceivePort] that receives messages from the server. + final ReceivePort receivePort = ReceivePort(); + + IsolateClient(String? clientId, this.serverSendPort) { + _clientId = clientId; + receivePort.listen((data) { + if (data is Map && data['request_id'] is String) { + var requestId = data['request_id'] as String?; + var c = _requests.remove(requestId); + + if (c != null && !c.isCompleted) { + if (data['status'] is! bool) { + c.completeError( + FormatException('The server sent an invalid response.')); + } else if (!(data['status'] as bool)) { + c.completeError(PubSubException(data['error_message']?.toString() ?? + 'The server sent a failure response, but did not provide an error message.')); + } else if (data['result'] is! Map) { + c.completeError(FormatException( + 'The server sent a success response, but did not include a result.')); + } else { + c.complete(data['result'] as Map?); + } + } + } else if (data is Map && data['id'] is String && _id == null) { + _id = data['id'] as String?; + + for (var c in _onConnect) { + if (!c.isCompleted) c.complete(_id); + } + + _onConnect.clear(); + } else if (data is List && data.length == 2 && data[0] is String) { + var eventName = data[0] as String; + var event = data[1]; + for (var s in _subscriptions.where((s) => s.eventName == eventName)) { + if (!s._stream.isClosed) s._stream.add(event); + } + } + }); + serverSendPort.send(receivePort.sendPort); + } + + Future _whenConnected(FutureOr Function() callback) { + if (_id != null) { + return Future.sync(callback); + } else { + var c = Completer(); + _onConnect.add(c); + return c.future.then((_) => callback()); + } + } + + @override + Future publish(String eventName, value) { + return _whenConnected(() { + var c = Completer(); + var requestId = _uuid.v4(); + _requests[requestId] = c; + serverSendPort.send({ + 'id': _id, + 'request_id': requestId, + 'method': 'publish', + 'params': { + 'client_id': clientId, + 'event_name': eventName, + 'value': value + } + }); + return c.future.then((result) { + _clientId = result['client_id'] as String?; + }); + }); + } + + @override + Future subscribe(String eventName) { + return _whenConnected(() { + var c = Completer(); + var requestId = _uuid.v4(); + _requests[requestId] = c; + serverSendPort.send({ + 'id': _id, + 'request_id': requestId, + 'method': 'subscribe', + 'params': {'client_id': clientId, 'event_name': eventName} + }); + return c.future.then((result) { + _clientId = result['client_id'] as String?; + var s = _IsolateClientSubscription( + eventName, result['subscription_id'] as String?, this); + _subscriptions.add(s); + return s; + }); + }); + } + + @override + Future close() { + receivePort.close(); + + for (var c in _onConnect) { + if (!c.isCompleted) { + c.completeError(StateError( + 'The client was closed before the server ever accepted the connection.')); + } + } + + for (var c in _requests.values) { + if (!c.isCompleted) { + c.completeError(StateError( + 'The client was closed before the server responded to this request.')); + } + } + + for (var s in _subscriptions) { + s._close(); + } + + _requests.clear(); + return Future.value(); + } +} + +class _IsolateClientSubscription extends ClientSubscription { + final StreamController _stream = StreamController(); + final String? eventName, id; + final IsolateClient client; + + _IsolateClientSubscription(this.eventName, this.id, this.client); + + void _close() { + if (!_stream.isClosed) _stream.close(); + } + + @override + StreamSubscription listen(void Function(dynamic event)? onData, + {Function? onError, void Function()? onDone, bool? cancelOnError}) { + return _stream.stream.listen(onData, + onError: onError, onDone: onDone, cancelOnError: cancelOnError); + } + + @override + Future unsubscribe() { + return client._whenConnected(() { + var c = Completer(); + var requestId = client._uuid.v4(); + client._requests[requestId] = c; + client.serverSendPort.send({ + 'id': client._id, + 'request_id': requestId, + 'method': 'unsubscribe', + 'params': {'client_id': client.clientId, 'subscription_id': id} + }); + + return c.future.then((_) { + _close(); + }); + }); + } +} diff --git a/packages/pub_sub/lib/src/isolate/server.dart b/packages/pub_sub/lib/src/isolate/server.dart new file mode 100644 index 0000000..bd9fadf --- /dev/null +++ b/packages/pub_sub/lib/src/isolate/server.dart @@ -0,0 +1,253 @@ +import 'dart:async'; +import 'dart:isolate'; +import 'package:uuid/uuid.dart'; +import '../../belatuk_pub_sub.dart'; + +/// A [Adapter] implementation that communicates via [SendPort]s and [ReceivePort]s. +class IsolateAdapter extends Adapter { + final Map _clients = {}; + final StreamController _onPublish = + StreamController(); + final StreamController _onSubscribe = + StreamController(); + final StreamController _onUnsubscribe = + StreamController(); + final Uuid _uuid = Uuid(); + + /// A [ReceivePort] on which to listen for incoming data. + final ReceivePort receivePort = ReceivePort(); + + @override + Stream get onPublish => _onPublish.stream; + + @override + Stream get onSubscribe => _onSubscribe.stream; + + @override + Stream get onUnsubscribe => _onUnsubscribe.stream; + + @override + Future close() { + receivePort.close(); + _clients.clear(); + _onPublish.close(); + _onSubscribe.close(); + _onUnsubscribe.close(); + return Future.value(); + } + + @override + void start() { + receivePort.listen((data) { + if (data is SendPort) { + var id = _uuid.v4(); + _clients[id] = data; + data.send({'status': true, 'id': id}); + } else if (data is Map && + data['id'] is String && + data['request_id'] is String && + data['method'] is String && + data['params'] is Map) { + var id = data['id'] as String?, + requestId = data['request_id'] as String?, + method = data['method'] as String?; + var params = data['params'] as Map?; + var sp = _clients[id!]; + + if (sp == null) { + // There's nobody to respond to, so don't send anything to anyone. Oops. + } else if (method == 'publish') { + if (_isValidClientId(params!['client_id']) && + params['event_name'] is String && + params.containsKey('value')) { + var clientId = params['client_id'] as String?, + eventName = params['event_name'] as String?; + var value = params['value']; + var rq = _IsolatePublishRequestImpl( + requestId, clientId, eventName, value, sp); + _onPublish.add(rq); + } else { + sp.send({ + 'status': false, + 'request_id': requestId, + 'error_message': 'Expected client_id, event_name, and value.' + }); + } + } else if (method == 'subscribe') { + if (_isValidClientId(params!['client_id']) && + params['event_name'] is String) { + var clientId = params['client_id'] as String?, + eventName = params['event_name'] as String?; + var rq = _IsolateSubscriptionRequestImpl( + clientId, eventName, sp, requestId, _uuid); + _onSubscribe.add(rq); + } else { + sp.send({ + 'status': false, + 'request_id': requestId, + 'error_message': 'Expected client_id, and event_name.' + }); + } + } else if (method == 'unsubscribe') { + if (_isValidClientId(params!['client_id']) && + params['subscription_id'] is String) { + var clientId = params['client_id'] as String?, + subscriptionId = params['subscription_id'] as String?; + var rq = _IsolateUnsubscriptionRequestImpl( + clientId, subscriptionId, sp, requestId); + _onUnsubscribe.add(rq); + } else { + sp.send({ + 'status': false, + 'request_id': requestId, + 'error_message': 'Expected client_id, and subscription_id.' + }); + } + } else { + sp.send({ + 'status': false, + 'request_id': requestId, + 'error_message': + 'Unrecognized method "$method". Or, you omitted id, request_id, method, or params.' + }); + } + } + }); + } + + bool _isValidClientId(id) => id == null || id is String; + + @override + bool isTrustedPublishRequest(PublishRequest request) { + // Isolate clients are considered trusted, because they are + // running in the same process as the central server. + return true; + } + + @override + bool isTrustedSubscriptionRequest(SubscriptionRequest request) { + return true; + } +} + +class _IsolatePublishRequestImpl extends PublishRequest { + @override + final String? clientId; + + @override + final String? eventName; + + @override + final dynamic value; + + final SendPort sendPort; + + final String? requestId; + + _IsolatePublishRequestImpl( + this.requestId, this.clientId, this.eventName, this.value, this.sendPort); + + @override + void accept(PublishResponse response) { + sendPort.send({ + 'status': true, + 'request_id': requestId, + 'result': { + 'listeners': response.listeners, + 'client_id': response.clientId + } + }); + } + + @override + void reject(String errorMessage) { + sendPort.send({ + 'status': false, + 'request_id': requestId, + 'error_message': errorMessage + }); + } +} + +class _IsolateSubscriptionRequestImpl extends SubscriptionRequest { + @override + final String? clientId; + + @override + final String? eventName; + + final SendPort sendPort; + + final String? requestId; + + final Uuid _uuid; + + _IsolateSubscriptionRequestImpl( + this.clientId, this.eventName, this.sendPort, this.requestId, this._uuid); + + @override + void reject(String errorMessage) { + sendPort.send({ + 'status': false, + 'request_id': requestId, + 'error_message': errorMessage + }); + } + + @override + FutureOr accept(String? clientId) { + var id = _uuid.v4(); + sendPort.send({ + 'status': true, + 'request_id': requestId, + 'result': {'subscription_id': id, 'client_id': clientId} + }); + return _IsolateSubscriptionImpl(clientId, id, eventName, sendPort); + } +} + +class _IsolateSubscriptionImpl extends Subscription { + @override + final String? clientId, id; + + final String? eventName; + + final SendPort sendPort; + + _IsolateSubscriptionImpl( + this.clientId, this.id, this.eventName, this.sendPort); + + @override + void dispatch(event) { + sendPort.send([eventName, event]); + } +} + +class _IsolateUnsubscriptionRequestImpl extends UnsubscriptionRequest { + @override + final String? clientId; + + @override + final String? subscriptionId; + + final SendPort sendPort; + + final String? requestId; + + _IsolateUnsubscriptionRequestImpl( + this.clientId, this.subscriptionId, this.sendPort, this.requestId); + + @override + void reject(String errorMessage) { + sendPort.send({ + 'status': false, + 'request_id': requestId, + 'error_message': errorMessage + }); + } + + @override + void accept() { + sendPort.send({'status': true, 'request_id': requestId, 'result': {}}); + } +} diff --git a/packages/pub_sub/lib/src/json_rpc/client.dart b/packages/pub_sub/lib/src/json_rpc/client.dart new file mode 100644 index 0000000..d1efb83 --- /dev/null +++ b/packages/pub_sub/lib/src/json_rpc/client.dart @@ -0,0 +1,145 @@ +import 'dart:async'; +import 'package:stream_channel/stream_channel.dart'; +import 'package:json_rpc_2/json_rpc_2.dart' as json_rpc_2; +import 'package:uuid/uuid.dart'; +import '../../belatuk_pub_sub.dart'; + +/// A [Client] implementation that communicates via JSON RPC 2.0. +class JsonRpc2Client extends Client { + final Map> _requests = {}; + final List<_JsonRpc2ClientSubscription> _subscriptions = []; + final Uuid _uuid = Uuid(); + + json_rpc_2.Peer? _peer; + + /// The ID of the client we are authenticating as. + /// + /// May be `null`, if and only if we are marked as a trusted source on + /// the server side. + String? get clientId => _clientId; + String? _clientId; + + JsonRpc2Client(String? clientId, StreamChannel channel) { + _clientId = clientId; + _peer = json_rpc_2.Peer(channel); + + _peer!.registerMethod('event', (json_rpc_2.Parameters params) { + String? eventName = params['event_name'].asString; + var event = params['value'].value; + for (var s in _subscriptions.where((s) => s.eventName == eventName)) { + if (!s._stream.isClosed) s._stream.add(event); + } + }); + + _peer!.registerFallback((json_rpc_2.Parameters params) { + var c = _requests.remove(params.method); + + if (c == null) { + throw json_rpc_2.RpcException.methodNotFound(params.method); + } else { + var data = params.asMap; + + if (data['status'] is! bool) { + c.completeError( + FormatException('The server sent an invalid response.')); + } else if (!(data['status'] as bool)) { + c.completeError(PubSubException(data['error_message']?.toString() ?? + 'The server sent a failure response, but did not provide an error message.')); + } else { + c.complete(data); + } + } + }); + + _peer!.listen(); + } + + @override + Future publish(String eventName, value) { + var c = Completer(); + var requestId = _uuid.v4(); + _requests[requestId] = c; + _peer!.sendNotification('publish', { + 'request_id': requestId, + 'client_id': clientId, + 'event_name': eventName, + 'value': value + }); + return c.future.then((data) { + _clientId = data['result']['client_id'] as String?; + }); + } + + @override + Future subscribe(String eventName) { + var c = Completer(); + var requestId = _uuid.v4(); + _requests[requestId] = c; + _peer!.sendNotification('subscribe', { + 'request_id': requestId, + 'client_id': clientId, + 'event_name': eventName + }); + return c.future.then((result) { + _clientId = result['client_id'] as String?; + var s = _JsonRpc2ClientSubscription( + eventName, result['subscription_id'] as String?, this); + _subscriptions.add(s); + return s; + }); + } + + @override + Future close() { + if (_peer?.isClosed != true) _peer!.close(); + + for (var c in _requests.values) { + if (!c.isCompleted) { + c.completeError(StateError( + 'The client was closed before the server responded to this request.')); + } + } + + for (var s in _subscriptions) { + s._close(); + } + + _requests.clear(); + return Future.value(); + } +} + +class _JsonRpc2ClientSubscription extends ClientSubscription { + final StreamController _stream = StreamController(); + final String? eventName, id; + final JsonRpc2Client client; + + _JsonRpc2ClientSubscription(this.eventName, this.id, this.client); + + void _close() { + if (!_stream.isClosed) _stream.close(); + } + + @override + StreamSubscription listen(void Function(dynamic event)? onData, + {Function? onError, void Function()? onDone, bool? cancelOnError}) { + return _stream.stream.listen(onData, + onError: onError, onDone: onDone, cancelOnError: cancelOnError); + } + + @override + Future unsubscribe() { + var c = Completer(); + var requestId = client._uuid.v4(); + client._requests[requestId] = c; + client._peer!.sendNotification('unsubscribe', { + 'request_id': requestId, + 'client_id': client.clientId, + 'subscription_id': id + }); + + return c.future.then((_) { + _close(); + }); + } +} diff --git a/packages/pub_sub/lib/src/json_rpc/server.dart b/packages/pub_sub/lib/src/json_rpc/server.dart new file mode 100644 index 0000000..8d4c5d0 --- /dev/null +++ b/packages/pub_sub/lib/src/json_rpc/server.dart @@ -0,0 +1,220 @@ +import 'dart:async'; +import 'package:stream_channel/stream_channel.dart'; +import 'package:json_rpc_2/json_rpc_2.dart' as json_rpc_2; +import 'package:uuid/uuid.dart'; +import '../../belatuk_pub_sub.dart'; + +/// A [Adapter] implementation that communicates via JSON RPC 2.0. +class JsonRpc2Adapter extends Adapter { + final StreamController _onPublish = + StreamController(); + final StreamController _onSubscribe = + StreamController(); + final StreamController _onUnsubscribe = + StreamController(); + + final List _peers = []; + final Uuid _uuid = Uuid(); + + json_rpc_2.Peer? _peer; + + /// A [Stream] of incoming clients, who can both send and receive string data. + final Stream> clientStream; + + /// If `true`, clients can connect through this endpoint, *without* providing a client ID. + /// + /// This can be a security vulnerability if you don't know what you're doing. + /// If you *must* use this over the Internet, use an IP whitelist. + final bool isTrusted; + + JsonRpc2Adapter(this.clientStream, {this.isTrusted = false}); + + @override + Stream get onPublish => _onPublish.stream; + + @override + Stream get onSubscribe => _onSubscribe.stream; + + @override + Stream get onUnsubscribe => _onUnsubscribe.stream; + + @override + Future close() { + if (_peer?.isClosed != true) _peer?.close(); + + Future.wait(_peers.where((s) => !s.isClosed).map((s) => s.close())) + .then((_) => _peers.clear()); + return Future.value(); + } + + String? _getClientId(json_rpc_2.Parameters params) { + try { + return params['client_id'].asString; + } catch (_) { + return null; + } + } + + @override + void start() { + clientStream.listen((client) { + var peer = _peer = json_rpc_2.Peer(client); + + peer.registerMethod('publish', (json_rpc_2.Parameters params) async { + var requestId = params['request_id'].asString; + var clientId = _getClientId(params); + var eventName = params['event_name'].asString; + var value = params['value'].value; + var rq = _JsonRpc2PublishRequestImpl( + requestId, clientId, eventName, value, peer); + _onPublish.add(rq); + }); + + peer.registerMethod('subscribe', (json_rpc_2.Parameters params) async { + var requestId = params['request_id'].asString; + var clientId = _getClientId(params); + var eventName = params['event_name'].asString; + var rq = _JsonRpc2SubscriptionRequestImpl( + clientId, eventName, requestId, peer, _uuid); + _onSubscribe.add(rq); + }); + + peer.registerMethod('unsubscribe', (json_rpc_2.Parameters params) async { + var requestId = params['request_id'].asString; + var clientId = _getClientId(params); + var subscriptionId = params['subscription_id'].asString; + var rq = _JsonRpc2UnsubscriptionRequestImpl( + clientId, subscriptionId, peer, requestId); + _onUnsubscribe.add(rq); + }); + + peer.listen(); + }); + } + + @override + bool isTrustedPublishRequest(PublishRequest request) { + return isTrusted; + } + + @override + bool isTrustedSubscriptionRequest(SubscriptionRequest request) { + return isTrusted; + } +} + +class _JsonRpc2PublishRequestImpl extends PublishRequest { + final String requestId; + + @override + final String? clientId, eventName; + + @override + final dynamic value; + + final json_rpc_2.Peer peer; + + _JsonRpc2PublishRequestImpl( + this.requestId, this.clientId, this.eventName, this.value, this.peer); + + @override + void accept(PublishResponse response) { + peer.sendNotification(requestId, { + 'status': true, + 'request_id': requestId, + 'result': { + 'listeners': response.listeners, + 'client_id': response.clientId + } + }); + } + + @override + void reject(String errorMessage) { + peer.sendNotification(requestId, { + 'status': false, + 'request_id': requestId, + 'error_message': errorMessage + }); + } +} + +class _JsonRpc2SubscriptionRequestImpl extends SubscriptionRequest { + @override + final String? clientId, eventName; + + final String requestId; + + final json_rpc_2.Peer peer; + + final Uuid _uuid; + + _JsonRpc2SubscriptionRequestImpl( + this.clientId, this.eventName, this.requestId, this.peer, this._uuid); + + @override + FutureOr accept(String? clientId) { + var id = _uuid.v4(); + peer.sendNotification(requestId, { + 'status': true, + 'request_id': requestId, + 'subscription_id': id, + 'client_id': clientId + }); + return _JsonRpc2SubscriptionImpl(clientId, id, eventName, peer); + } + + @override + void reject(String errorMessage) { + peer.sendNotification(requestId, { + 'status': false, + 'request_id': requestId, + 'error_message': errorMessage + }); + } +} + +class _JsonRpc2SubscriptionImpl extends Subscription { + @override + final String? clientId, id; + + final String? eventName; + + final json_rpc_2.Peer peer; + + _JsonRpc2SubscriptionImpl(this.clientId, this.id, this.eventName, this.peer); + + @override + void dispatch(event) { + peer.sendNotification('event', {'event_name': eventName, 'value': event}); + } +} + +class _JsonRpc2UnsubscriptionRequestImpl extends UnsubscriptionRequest { + @override + final String? clientId; + + @override + final String subscriptionId; + + final json_rpc_2.Peer peer; + + final String requestId; + + _JsonRpc2UnsubscriptionRequestImpl( + this.clientId, this.subscriptionId, this.peer, this.requestId); + + @override + void accept() { + peer.sendNotification(requestId, {'status': true, 'result': {}}); + } + + @override + void reject(String errorMessage) { + peer.sendNotification(requestId, { + 'status': false, + 'request_id': requestId, + 'error_message': errorMessage + }); + } +} diff --git a/packages/pub_sub/lib/src/protocol/client/client.dart b/packages/pub_sub/lib/src/protocol/client/client.dart new file mode 100644 index 0000000..4a32716 --- /dev/null +++ b/packages/pub_sub/lib/src/protocol/client/client.dart @@ -0,0 +1,30 @@ +import 'dart:async'; + +/// Queries a `pub_sub` server. +abstract class Client { + /// Publishes an event to the server. + Future publish(String eventName, value); + + /// Request a [ClientSubscription] to the desired [eventName] from the server. + Future subscribe(String eventName); + + /// Disposes of this client. + Future close(); +} + +/// A client-side implementation of a subscription, which acts as a [Stream], and can be cancelled easily. +abstract class ClientSubscription extends Stream { + /// Stops listening for new events, and instructs the server to cancel the subscription. + Future unsubscribe(); +} + +/// Thrown as the result of an invalid request, or an attempt to perform an action without the correct privileges. +class PubSubException implements Exception { + /// The error message sent by the server. + final String message; + + const PubSubException(this.message); + + @override + String toString() => '`pub_sub` exception: $message'; +} diff --git a/packages/pub_sub/lib/src/protocol/client/sync_client.dart b/packages/pub_sub/lib/src/protocol/client/sync_client.dart new file mode 100644 index 0000000..93a3257 --- /dev/null +++ b/packages/pub_sub/lib/src/protocol/client/sync_client.dart @@ -0,0 +1 @@ +export 'client.dart'; diff --git a/packages/pub_sub/lib/src/protocol/protocol.dart b/packages/pub_sub/lib/src/protocol/protocol.dart new file mode 100644 index 0000000..9bf74c6 --- /dev/null +++ b/packages/pub_sub/lib/src/protocol/protocol.dart @@ -0,0 +1,2 @@ +export 'client/sync_client.dart'; +export 'server/sync_server.dart'; diff --git a/packages/pub_sub/lib/src/protocol/server/adapter.dart b/packages/pub_sub/lib/src/protocol/server/adapter.dart new file mode 100644 index 0000000..e129b4a --- /dev/null +++ b/packages/pub_sub/lib/src/protocol/server/adapter.dart @@ -0,0 +1,29 @@ +import 'dart:async'; +import 'publish.dart'; +import 'subscription.dart'; + +/// Adapts an abstract medium to serve the `pub_sub` RPC protocol. +abstract class Adapter { + /// Determines if a given [request] comes from a trusted source. + /// + /// If so, the request does not have to provide a pre-established ID, + /// and instead will be assigned one. + bool isTrustedPublishRequest(PublishRequest request); + + bool isTrustedSubscriptionRequest(SubscriptionRequest request); + + /// Fires an event whenever a client tries to publish data. + Stream get onPublish; + + /// Fires whenever a client tries to subscribe to an event. + Stream get onSubscribe; + + /// Fires whenever a client cancels a subscription. + Stream get onUnsubscribe; + + /// Disposes of this adapter. + Future close(); + + /// Start listening for incoming clients. + void start(); +} diff --git a/packages/pub_sub/lib/src/protocol/server/client.dart b/packages/pub_sub/lib/src/protocol/server/client.dart new file mode 100644 index 0000000..976e3fc --- /dev/null +++ b/packages/pub_sub/lib/src/protocol/server/client.dart @@ -0,0 +1,14 @@ +/// Represents information about a client that will be accessing +/// this `angel_sync` server. +class ClientInfo { + /// A unique identifier for this client. + final String id; + + /// If `true` (default), then the client is allowed to publish events. + final bool canPublish; + + /// If `true` (default), then the client can subscribe to events. + final bool canSubscribe; + + const ClientInfo(this.id, {this.canPublish = true, this.canSubscribe = true}); +} diff --git a/packages/pub_sub/lib/src/protocol/server/publish.dart b/packages/pub_sub/lib/src/protocol/server/publish.dart new file mode 100644 index 0000000..b6cbf95 --- /dev/null +++ b/packages/pub_sub/lib/src/protocol/server/publish.dart @@ -0,0 +1,28 @@ +/// Represents a request to publish information to other clients. +abstract class PublishRequest { + /// The ID of the client sending this request. + String? get clientId; + + /// The name of the event to be sent. + String? get eventName; + + /// The value to be published as an event. + dynamic get value; + + /// Accept the request, with a response. + void accept(PublishResponse response); + + /// Deny the request with an error message. + void reject(String errorMessage); +} + +/// A response to a publish request. Informs the caller of how much clients received the event. +class PublishResponse { + /// The number of unique listeners to whom this event was propogated. + final int listeners; + + /// The client ID returned the server. Significant in cases where an ad-hoc client was registered. + final String? clientId; + + const PublishResponse(this.listeners, this.clientId); +} diff --git a/packages/pub_sub/lib/src/protocol/server/server.dart b/packages/pub_sub/lib/src/protocol/server/server.dart new file mode 100644 index 0000000..b0cc08d --- /dev/null +++ b/packages/pub_sub/lib/src/protocol/server/server.dart @@ -0,0 +1,160 @@ +import 'dart:async'; +import 'dart:math'; +import 'adapter.dart'; +import 'client.dart'; +import 'package:collection/collection.dart' show IterableExtension; +import 'publish.dart'; +import 'subscription.dart'; + +/// A server that implements the `pub_sub` protocol. +/// +/// It can work using multiple [Adapter]s, to simultaneously +/// serve local and remote clients alike. +class Server { + final List _adapters = []; + final List _clients = []; + final _rnd = Random.secure(); + final Map> _subscriptions = {}; + bool _started = false; + int _adHocIds = 0; + + /// Initialize a server, optionally with a number of [adapters]. + Server([Iterable adapters = const []]) { + _adapters.addAll(adapters); + } + + /// Adds a new [Adapter] to adapt incoming clients from a new interface. + void addAdapter(Adapter adapter) { + if (_started) { + throw StateError( + 'You cannot add new adapters after the server has started listening.'); + } else { + _adapters.add(adapter); + } + } + + /// Registers a new client with the server. + void registerClient(ClientInfo client) { + if (_started) { + throw StateError( + 'You cannot register new clients after the server has started listening.'); + } else { + _clients.add(client); + } + } + + /// Disposes of this server, and closes all of its adapters. + Future close() { + Future.wait(_adapters.map((a) => a.close())); + _adapters.clear(); + _clients.clear(); + _subscriptions.clear(); + return Future.value(); + } + + String _newClientId() { + // Create an unpredictable-enough ID. The harder it is for an attacker to guess, the better. + var id = + 'pub_sub::adhoc_client${_rnd.nextDouble()}::${_adHocIds++}:${DateTime.now().millisecondsSinceEpoch * _rnd.nextDouble()}'; + + // This client is coming from a trusted source, and can therefore both publish and subscribe. + _clients.add(ClientInfo(id)); + return id; + } + + void start() { + if (_adapters.isEmpty) { + throw StateError( + 'Cannot start a SyncServer that has no adapters attached.'); + } else if (_started) { + throw StateError('A SyncServer may only be started once.'); + } + + _started = true; + + for (var adapter in _adapters) { + adapter.start(); + } + + for (var adapter in _adapters) { + // Handle publishes + adapter.onPublish.listen((rq) { + ClientInfo? client; + String? clientId; + + if (rq.clientId?.isNotEmpty == true || + adapter.isTrustedPublishRequest(rq)) { + clientId = + rq.clientId?.isNotEmpty == true ? rq.clientId : _newClientId(); + client = _clients.firstWhereOrNull((c) => c.id == clientId); + } + + if (client == null) { + rq.reject('Unrecognized client ID "${clientId ?? ''}".'); + } else if (!client.canPublish) { + rq.reject('You are not allowed to publish events.'); + } else { + var listeners = _subscriptions[rq.eventName] + ?.where((s) => s.clientId != clientId) ?? + []; + + if (listeners.isEmpty) { + rq.accept(PublishResponse(0, clientId)); + } else { + for (var listener in listeners) { + listener.dispatch(rq.value); + } + + rq.accept(PublishResponse(listeners.length, clientId)); + } + } + }); + + // Listen for incoming subscriptions + adapter.onSubscribe.listen((rq) async { + ClientInfo? client; + String? clientId; + + if (rq.clientId?.isNotEmpty == true || + adapter.isTrustedSubscriptionRequest(rq)) { + clientId = + rq.clientId?.isNotEmpty == true ? rq.clientId : _newClientId(); + client = _clients.firstWhereOrNull((c) => c.id == clientId); + } + + if (client == null) { + rq.reject('Unrecognized client ID "${clientId ?? ''}".'); + } else if (!client.canSubscribe) { + rq.reject('You are not allowed to subscribe to events.'); + } else { + var sub = await rq.accept(clientId); + var list = _subscriptions.putIfAbsent(rq.eventName, () => []); + list.add(sub); + } + }); + + // Unregister subscriptions on unsubscribe + adapter.onUnsubscribe.listen((rq) { + Subscription? toRemove; + late List sourceList; + + for (var list in _subscriptions.values) { + toRemove = list.firstWhereOrNull((s) => s.id == rq.subscriptionId); + if (toRemove != null) { + sourceList = list; + break; + } + } + + if (toRemove == null) { + rq.reject('The specified subscription does not exist.'); + } else if (toRemove.clientId != rq.clientId) { + rq.reject('That is not your subscription to cancel.'); + } else { + sourceList.remove(toRemove); + rq.accept(); + } + }); + } + } +} diff --git a/packages/pub_sub/lib/src/protocol/server/subscription.dart b/packages/pub_sub/lib/src/protocol/server/subscription.dart new file mode 100644 index 0000000..9f5db23 --- /dev/null +++ b/packages/pub_sub/lib/src/protocol/server/subscription.dart @@ -0,0 +1,47 @@ +import 'dart:async'; + +/// Represents a request to subscribe to an event. +abstract class SubscriptionRequest { + /// The ID of the client requesting to subscribe. + String? get clientId; + + /// The name of the event the client wants to subscribe to. + String? get eventName; + + /// Accept the request, and grant the client access to subscribe to the event. + /// + /// Includes the client's ID, which is necessary for ad-hoc clients. + FutureOr accept(String? clientId); + + /// Deny the request with an error message. + void reject(String errorMessage); +} + +/// Represents a request to unsubscribe to an event. +abstract class UnsubscriptionRequest { + /// The ID of the client requesting to unsubscribe. + String? get clientId; + + /// The name of the event the client wants to unsubscribe from. + String? get subscriptionId; + + /// Accept the request. + FutureOr accept(); + + /// Deny the request with an error message. + void reject(String errorMessage); +} + +/// Represents a client's subscription to an event. +/// +/// Also provides a means to fire an event. +abstract class Subscription { + /// A unique identifier for this subscription. + String? get id; + + /// The ID of the client who requested this subscription. + String? get clientId; + + /// Alerts a client of an event. + void dispatch(event); +} diff --git a/packages/pub_sub/lib/src/protocol/server/sync_server.dart b/packages/pub_sub/lib/src/protocol/server/sync_server.dart new file mode 100644 index 0000000..5777b27 --- /dev/null +++ b/packages/pub_sub/lib/src/protocol/server/sync_server.dart @@ -0,0 +1,5 @@ +export 'adapter.dart'; +export 'client.dart'; +export 'publish.dart'; +export 'server.dart'; +export 'subscription.dart'; diff --git a/packages/pub_sub/pubspec.yaml b/packages/pub_sub/pubspec.yaml new file mode 100644 index 0000000..963f3f2 --- /dev/null +++ b/packages/pub_sub/pubspec.yaml @@ -0,0 +1,14 @@ +name: belatuk_pub_sub +version: 4.0.0 +description: Keep application instances in sync with a simple pub/sub API. +homepage: https://github.com/dart-backend/belatuk-common-utilities/tree/main/packages/pub_sub +environment: + sdk: '>=2.12.0 <3.0.0' +dependencies: + json_rpc_2: ^3.0.0 + stream_channel: ^2.1.0 + uuid: ^3.0.4 + collection: ^1.15.0 +dev_dependencies: + lints: ^1.0.0 + test: ^1.17.4 diff --git a/packages/pub_sub/test/isolate_test.dart b/packages/pub_sub/test/isolate_test.dart new file mode 100644 index 0000000..566bdf1 --- /dev/null +++ b/packages/pub_sub/test/isolate_test.dart @@ -0,0 +1,122 @@ +import 'dart:async'; +import 'package:belatuk_pub_sub/belatuk_pub_sub.dart'; +import 'package:belatuk_pub_sub/isolate.dart'; +import 'package:test/test.dart'; + +void main() { + late Server server; + late Client client1, client2, client3; + late IsolateClient trustedClient; + late IsolateAdapter adapter; + + setUp(() async { + adapter = IsolateAdapter(); + client1 = + IsolateClient('isolate_test::secret', adapter.receivePort.sendPort); + client2 = + IsolateClient('isolate_test::secret2', adapter.receivePort.sendPort); + client3 = + IsolateClient('isolate_test::secret3', adapter.receivePort.sendPort); + trustedClient = IsolateClient(null, adapter.receivePort.sendPort); + + server = Server([adapter]) + ..registerClient(const ClientInfo('isolate_test::secret')) + ..registerClient(const ClientInfo('isolate_test::secret2')) + ..registerClient(const ClientInfo('isolate_test::secret3')) + ..registerClient( + const ClientInfo('isolate_test::no_publish', canPublish: false)) + ..registerClient( + const ClientInfo('isolate_test::no_subscribe', canSubscribe: false)) + ..start(); + + var sub = await client3.subscribe('foo'); + sub.listen((data) { + print('Client3 caught foo: $data'); + }); + }); + + tearDown(() { + Future.wait([ + server.close(), + client1.close(), + client2.close(), + client3.close(), + trustedClient.close() + ]); + }); + + group('trusted', () { + test('can publish', () async { + await trustedClient.publish('hey', 'bye'); + expect(trustedClient.clientId, isNotNull); + }); + test('can sub/unsub', () async { + String? clientId; + await trustedClient.publish('heyaaa', 'byeaa'); + expect(clientId = trustedClient.clientId, isNotNull); + + var sub = await trustedClient.subscribe('yeppp'); + expect(trustedClient.clientId, clientId); + + await sub.unsubscribe(); + expect(trustedClient.clientId, clientId); + }); + }); + + test('subscribers receive published events', () async { + var sub = await client2.subscribe('foo'); + await client1.publish('foo', 'bar'); + expect(await sub.first, 'bar'); + }); + + test('subscribers are not sent their own events', () async { + var sub = await client1.subscribe('foo'); + await client1.publish('foo', + ''); + await sub.unsubscribe(); + expect(await sub.isEmpty, isTrue); + }); + + test('can unsubscribe', () async { + var sub = await client2.subscribe('foo'); + await client1.publish('foo', 'bar'); + await sub.unsubscribe(); + await client1.publish('foo', ''); + expect(await sub.length, 1); + }); + + group('isolate_server', () { + test('reject unknown client id', () async { + try { + var client = IsolateClient( + 'isolate_test::invalid', adapter.receivePort.sendPort); + await client.publish('foo', 'bar'); + throw 'Invalid client ID\'s should throw an error, but they do not.'; + } on PubSubException catch (e) { + print('Expected exception was thrown: ${e.message}'); + } + }); + + test('reject unprivileged publish', () async { + try { + var client = IsolateClient( + 'isolate_test::no_publish', adapter.receivePort.sendPort); + await client.publish('foo', 'bar'); + throw 'Unprivileged publishes should throw an error, but they do not.'; + } on PubSubException catch (e) { + print('Expected exception was thrown: ${e.message}'); + } + }); + + test('reject unprivileged subscribe', () async { + try { + var client = IsolateClient( + 'isolate_test::no_subscribe', adapter.receivePort.sendPort); + await client.subscribe('foo'); + throw 'Unprivileged subscribes should throw an error, but they do not.'; + } on PubSubException catch (e) { + print('Expected exception was thrown: ${e.message}'); + } + }); + }); +} diff --git a/packages/pub_sub/test/json_rpc_2_test.dart b/packages/pub_sub/test/json_rpc_2_test.dart new file mode 100644 index 0000000..2f6932b --- /dev/null +++ b/packages/pub_sub/test/json_rpc_2_test.dart @@ -0,0 +1,187 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; +import 'package:belatuk_pub_sub/belatuk_pub_sub.dart'; +import 'package:belatuk_pub_sub/json_rpc_2.dart'; +import 'package:stream_channel/stream_channel.dart'; +import 'package:test/test.dart'; + +void main() { + late ServerSocket serverSocket; + late Server server; + late Client client1, client2, client3; + late JsonRpc2Client trustedClient; + JsonRpc2Adapter adapter; + + setUp(() async { + serverSocket = await ServerSocket.bind(InternetAddress.loopbackIPv4, 0); + + adapter = JsonRpc2Adapter( + serverSocket.map>(streamSocket), + isTrusted: true); + + var socket1 = + await Socket.connect(InternetAddress.loopbackIPv4, serverSocket.port); + var socket2 = + await Socket.connect(InternetAddress.loopbackIPv4, serverSocket.port); + var socket3 = + await Socket.connect(InternetAddress.loopbackIPv4, serverSocket.port); + var socket4 = + await Socket.connect(InternetAddress.loopbackIPv4, serverSocket.port); + + client1 = JsonRpc2Client('json_rpc_2_test::secret', streamSocket(socket1)); + client2 = JsonRpc2Client('json_rpc_2_test::secret2', streamSocket(socket2)); + client3 = JsonRpc2Client('json_rpc_2_test::secret3', streamSocket(socket3)); + trustedClient = JsonRpc2Client(null, streamSocket(socket4)); + + server = Server([adapter]) + ..registerClient(const ClientInfo('json_rpc_2_test::secret')) + ..registerClient(const ClientInfo('json_rpc_2_test::secret2')) + ..registerClient(const ClientInfo('json_rpc_2_test::secret3')) + ..registerClient( + const ClientInfo('json_rpc_2_test::no_publish', canPublish: false)) + ..registerClient(const ClientInfo('json_rpc_2_test::no_subscribe', + canSubscribe: false)) + ..start(); + + var sub = await client3.subscribe('foo'); + sub.listen((data) { + print('Client3 caught foo: $data'); + }); + }); + + tearDown(() { + Future.wait( + [server.close(), client1.close(), client2.close(), client3.close()]); + }); + + group('trusted', () { + test('can publish', () async { + await trustedClient.publish('hey', 'bye'); + expect(trustedClient.clientId, isNotNull); + }); + test('can sub/unsub', () async { + String? clientId; + await trustedClient.publish('heyaaa', 'byeaa'); + expect(clientId = trustedClient.clientId, isNotNull); + + var sub = await trustedClient.subscribe('yeppp'); + expect(trustedClient.clientId, clientId); + + await sub.unsubscribe(); + expect(trustedClient.clientId, clientId); + }); + }); + + test('subscribers receive published events', () async { + var sub = await client2.subscribe('foo'); + await client1.publish('foo', 'bar'); + expect(await sub.first, 'bar'); + }); + + test('subscribers are not sent their own events', () async { + var sub = await client1.subscribe('foo'); + await client1.publish('foo', + ''); + await sub.unsubscribe(); + expect(await sub.isEmpty, isTrue); + }); + + test('can unsubscribe', () async { + var sub = await client2.subscribe('foo'); + await client1.publish('foo', 'bar'); + await sub.unsubscribe(); + await client1.publish('foo', ''); + expect(await sub.length, 1); + }); + + group('json_rpc_2_server', () { + test('reject unknown client id', () async { + try { + var sock = await Socket.connect( + InternetAddress.loopbackIPv4, serverSocket.port); + var client = + JsonRpc2Client('json_rpc_2_test::invalid', streamSocket(sock)); + await client.publish('foo', 'bar'); + throw 'Invalid client ID\'s should throw an error, but they do not.'; + } on PubSubException catch (e) { + print('Expected exception was thrown: ${e.message}'); + } + }); + + test('reject unprivileged publish', () async { + try { + var sock = await Socket.connect( + InternetAddress.loopbackIPv4, serverSocket.port); + var client = + JsonRpc2Client('json_rpc_2_test::no_publish', streamSocket(sock)); + await client.publish('foo', 'bar'); + throw 'Unprivileged publishes should throw an error, but they do not.'; + } on PubSubException catch (e) { + print('Expected exception was thrown: ${e.message}'); + } + }); + + test('reject unprivileged subscribe', () async { + try { + var sock = await Socket.connect( + InternetAddress.loopbackIPv4, serverSocket.port); + var client = + JsonRpc2Client('json_rpc_2_test::no_subscribe', streamSocket(sock)); + await client.subscribe('foo'); + throw 'Unprivileged subscribes should throw an error, but they do not.'; + } on PubSubException catch (e) { + print('Expected exception was thrown: ${e.message}'); + } + }); + }); +} + +StreamChannel streamSocket(Socket socket) { + var channel = _SocketStreamChannel(socket); + return channel + .cast>() + .transform(StreamChannelTransformer.fromCodec(utf8)); +} + +class _SocketStreamChannel extends StreamChannelMixin> { + _SocketSink? _sink; + final Socket socket; + + _SocketStreamChannel(this.socket); + + @override + StreamSink> get sink => _sink ??= _SocketSink(socket); + + @override + Stream> get stream => socket; +} + +class _SocketSink extends StreamSink> { + final Socket socket; + + _SocketSink(this.socket); + + @override + void add(List event) { + socket.add(event); + } + + @override + void addError(Object error, [StackTrace? stackTrace]) { + Zone.current.errorCallback(error, stackTrace); + } + + @override + Future addStream(Stream> stream) { + return socket.addStream(stream); + } + + @override + Future close() { + return socket.close(); + } + + @override + Future get done => socket.done; +}