From c9c431b3e5b93eadf8b7d4cda75c9719c0759c9e Mon Sep 17 00:00:00 2001 From: thomashii Date: Sat, 26 Jun 2021 18:02:41 +0800 Subject: [PATCH] Updated cache --- CHANGELOG.md | 4 +- CONTRIBUTING.md | 26 ++++++++- README.md | 4 +- TODO.md | 16 +++-- packages/cache/CHANGELOG.md | 2 + packages/cache/README.md | 7 +-- packages/cache/example/main.dart | 7 ++- packages/cache/lib/src/cache.dart | 71 ++++++++++++++++------- packages/cache/lib/src/cache_service.dart | 2 +- packages/cache/pubspec.yaml | 2 +- packages/cache/test/cache_test.dart | 69 ++++++++++++++-------- packages/cache/test/files/date.txt | 0 packages/cache/test/pattern_test.dart | 21 +++++++ requirements.txt | 1 - 14 files changed, 164 insertions(+), 68 deletions(-) create mode 100644 packages/cache/test/files/date.txt create mode 100644 packages/cache/test/pattern_test.dart delete mode 100644 requirements.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f53757a..cc1b2fb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ * Migrated angel_model to 3.0.0 (0/0 tests passed) * Migrated angel_container to 3.0.0 (55/55 tests passed) * Added merge_map and migrated to 2.0.0 (6/6 tests passed) -* Added mock_request and migrated to 2.0.0 (0/0 tests) +* Added mock_request and migrated to 2.0.0 (5/5 tests) * Migrated angel_framework to 4.0.0 (149/150 tests passed) * Migrated angel_auth to 4.0.0 (27/30 tests passed) * Migrated angel_configuration to 4.0.0 (6/8 testspassed) @@ -44,7 +44,7 @@ * Migrated angel_orm_postgres to 3.0.0 (51/54 tests passed) * Create orm-sdk-2.12.x boilerplate (in progress) <= Milestone 2 * Migrated angel_auth_oauth2 to 4.0.0 (0/0 tests passed) -* Migrated angel_auth_cache to 4.0.0 (0/7 tests passed) +* Migrated angel_auth_cache to 4.0.0 (4/7 tests passed) * Migrated angel_auth_cors to 4.0.0 (10/15 tests passed) * Migrated angel_oauth2 to 4.0.0 (17/25 tests passed) * Migrated angel_proxy to 4.0.0 (5/7 tests passed) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index be2f0755..528e381e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1,25 @@ -# Contributing Angel3 -Any contributions from the community are welcome. +# Contribution + +Any help from the open-source community is always welcome and needed: + +1. Found an issue? + - Please [fill a bug report][tracker] with error message and steps to reproduce it. +2. Wish a feature? + - Open a feature request with use cases. +3. Are you using and liking the project? + - Create an article about your use case + - Do a post on your likes and dislikes + - Make a donation. +4. Are you a developer? + - Fix a bug and send a [pull request][pull_request] + - Implement a new feature + - Improve the Unit Tests + - Improve the [User Guide][doc] and send a [document pull request][doc_repo] +5. Have you already helped in any way? + - **Many thanks to the contributors and everybody that uses this project!** + +[tracker]: https://github.com/dukefirehawk/angel/issues +[pull_request]: https://github.com/dukefirehawk/angel/pulls +[doc]: https://angel3-docs.dukefirehawk.com +[doc_repo]: https://github.com/dukefirehawk/angel3-guide/pulls diff --git a/README.md b/README.md index 993e9737..6f43ddfc 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ Check out [Migrating to Angel3](https://angel3-docs.dukefirehawk.com/migration/a ## Examples and Documentation -Visit the [documentation](https://angel3-docs.dukefirehawk.com/) for dozens of guides and resources, including video tutorials, to get up and running as quickly as possible with Angel3 framework. +Visit the [User Guide](https://angel3-docs.dukefirehawk.com/) for dozens of guides and resources, including video tutorials, to get up and running as quickly as possible with Angel3 framework. Examples and complete projects can be found [here](https://github.com/dukefirehawk/angel3-examples). @@ -103,4 +103,4 @@ There is also an [Awesome Angel :fire:](https://github.com/dukefirehawk/angel3-a ## Contributing -Interested in contributing to Angel3? Start by reading the contribution guide [here](CONTRIBUTING.md). +Interested in contributing to Angel3? See the contribution guide [here](CONTRIBUTING.md). diff --git a/TODO.md b/TODO.md index 8d0374c9..9e92b8b9 100644 --- a/TODO.md +++ b/TODO.md @@ -1,9 +1,13 @@ -### angel_framework -* Migrate http_server to shelf -### Container/angel_container_generator +# Road Map -* test/reflector_test.reflectab.dart - Changed ImplicitGetterMirrorImpl() from 5 to 3 parameters (revisit later) -* A user forum -* Updated User Guide +## Short Term Goal +* Migrate all modules to support NNBD +* Add more examples + +* Improve User Guide + +## Long Term Goal + +* Upgrade Angel3 architecture diff --git a/packages/cache/CHANGELOG.md b/packages/cache/CHANGELOG.md index 39fff12d..3e9867d3 100644 --- a/packages/cache/CHANGELOG.md +++ b/packages/cache/CHANGELOG.md @@ -3,6 +3,8 @@ ## 4.0.1 * Updated pubspec description +* Fixed: Return `200` with cached data instead of `403` +* Updated broken Unit Tests ## 4.0.0 diff --git a/packages/cache/README.md b/packages/cache/README.md index a44560e5..370639ed 100644 --- a/packages/cache/README.md +++ b/packages/cache/README.md @@ -1,4 +1,4 @@ -# Angel3 Cache +# HTTP Caching for Angel3 [![version](https://img.shields.io/badge/pub-v4.0.1-brightgreen)](https://pub.dartlang.org/packages/angel3_cache) [![Null Safety](https://img.shields.io/badge/null-safety-brightgreen)](https://dart.dev/null-safety) @@ -6,7 +6,7 @@ [![License](https://img.shields.io/github/license/dukefirehawk/angel)](https://github.com/dukefirehawk/angel/tree/angel3/packages/cache/LICENSE) -Support for server-side caching in [Angel3](https://github.com/dukefirehawk/angel). +A service that provides HTTP caching to the response data for [Angel3](https://github.com/dukefirehawk/angel). ## `CacheService` @@ -16,8 +16,7 @@ A `Service` class that caches data from one service, storing it in another. An i A middleware that enables the caching of response serialization. -This can improve the performance of sending objects that are complex to serialize. -You can pass a [shouldCache] callback to determine which values should be cached. +This can improve the performance of sending objects that are complex to serialize. You can pass a [shouldCache] callback to determine which values should be cached. ```dart void main() async { diff --git a/packages/cache/example/main.dart b/packages/cache/example/main.dart index c8f72994..f604ff70 100644 --- a/packages/cache/example/main.dart +++ b/packages/cache/example/main.dart @@ -1,7 +1,6 @@ import 'package:angel3_cache/angel3_cache.dart'; import 'package:angel3_framework/angel3_framework.dart'; import 'package:angel3_framework/http.dart'; -import 'package:glob/glob.dart'; void main() async { var app = Angel(); @@ -9,7 +8,7 @@ void main() async { // Cache a glob var cache = ResponseCache() ..patterns.addAll([ - Glob('/*.txt'), + RegExp('^/?\\w+\\.txt'), ]); // Handle `if-modified-since` header, and also send cached content @@ -21,7 +20,9 @@ void main() async { // Support purging the cache. app.addRoute('PURGE', '*', (req, res) { - if (req.ip != '127.0.0.1') throw AngelHttpException.forbidden(); + if (req.ip != '127.0.0.1') { + throw AngelHttpException.forbidden(); + } cache.purge(req.uri!.path); print('Purged ${req.uri!.path}'); diff --git a/packages/cache/lib/src/cache.dart b/packages/cache/lib/src/cache.dart index ebaee2fe..b8820447 100644 --- a/packages/cache/lib/src/cache.dart +++ b/packages/cache/lib/src/cache.dart @@ -2,6 +2,7 @@ import 'dart:async'; import 'dart:io' show HttpDate; import 'package:angel3_framework/angel3_framework.dart'; import 'package:pool/pool.dart'; +import 'package:logging/logging.dart'; /// A flexible response cache for Angel. /// @@ -22,6 +23,8 @@ class ResponseCache { /// If `true` (default: `false`), then caching of results will discard URI query parameters and fragments. final bool ignoreQueryAndFragment; + final log = Logger('ResponseCache'); + ResponseCache( {this.timeout = const Duration(minutes: 10), this.ignoreQueryAndFragment = false}); @@ -38,26 +41,41 @@ class ResponseCache { /// /// This prevents the server from even having to access the cache, and plays very well with static assets. Future ifModifiedSince(RequestContext req, ResponseContext res) async { - if (req.method != 'GET' && req.method != 'HEAD') return true; - - if (req.headers!.ifModifiedSince != null) { - var modifiedSince = req.headers!.ifModifiedSince; + if (req.method != 'GET' && req.method != 'HEAD') { + return true; + } + var modifiedSince = req.headers?.ifModifiedSince; + if (modifiedSince != null) { // Check if there is a cache entry. for (var pattern in patterns) { - if (pattern.allMatches(_getEffectivePath(req)).isNotEmpty && - _cache.containsKey(_getEffectivePath(req))) { - var response = _cache[_getEffectivePath(req)]!; - //print('timestamp ${response.timestamp} vs since ${modifiedSince}'); + var reqPath = _getEffectivePath(req); - if (response.timestamp.compareTo(modifiedSince!) <= 0) { + if (pattern.allMatches(reqPath).isNotEmpty && + _cache.containsKey(reqPath)) { + var response = _cache[reqPath]; + + //log.info('timestamp ${response?.timestamp} vs since $modifiedSince'); + + if (response != null && + response.timestamp.compareTo(modifiedSince) <= 0) { // If the cache timeout has been met, don't send the cached response. - if (DateTime.now().toUtc().difference(response.timestamp) >= - timeout) { + var timeDiff = + DateTime.now().toUtc().difference(response.timestamp); + + //log.info( + // 'Time Diff: ${timeDiff.inMilliseconds} >= ${timeout.inMilliseconds}'); + if (timeDiff.inMilliseconds >= timeout.inMilliseconds) { return true; } - res.statusCode = 304; + // Old code: res.statusCode = 304; + // Return the response stored in the cache + _setCachedHeaders(response.timestamp, req, res); + res + ..headers.addAll(response.headers) + ..add(response.body); + await res.close(); return false; } } @@ -67,8 +85,13 @@ class ResponseCache { return true; } - String _getEffectivePath(RequestContext req) => - ignoreQueryAndFragment == true ? req.uri!.path : req.uri.toString(); + String _getEffectivePath(RequestContext req) { + if (req.uri == null) { + log.severe('Request URI is null'); + throw ArgumentError('Request URI is null'); + } + return ignoreQueryAndFragment == true ? req.uri!.path : req.uri.toString(); + } /// Serves content from the cache, if applicable. Future handleRequest(RequestContext req, ResponseContext res) async { @@ -114,37 +137,41 @@ class ResponseCache { if (res.statusCode == 304) { return true; } + if (req.method != 'GET' && req.method != 'HEAD') { return true; } // Check if there is a cache entry. for (var pattern in patterns) { - if (pattern.allMatches(_getEffectivePath(req)).isNotEmpty) { + var reqPath = _getEffectivePath(req); + + if (pattern.allMatches(reqPath).isNotEmpty) { var now = DateTime.now().toUtc(); // Invalidate the response, if need be. - if (_cache.containsKey(_getEffectivePath(req))) { + if (_cache.containsKey(reqPath)) { // If there is no timeout, don't invalidate. //if (timeout == null) return true; // Otherwise, don't invalidate unless the timeout has been exceeded. - var response = _cache[_getEffectivePath(req)]; + var response = _cache[reqPath]; if (response == null || now.difference(response.timestamp) < timeout) { return true; } // If the cache entry should be invalidated, then invalidate it. - purge(_getEffectivePath(req)); + purge(reqPath); } // Save the response. - var writeLock = - _writeLocks.putIfAbsent(_getEffectivePath(req), () => Pool(1)); + var writeLock = _writeLocks.putIfAbsent(reqPath, () => Pool(1)); await writeLock.withResource(() { - _cache[_getEffectivePath(req)] = _CachedResponse( - Map.from(res.headers), res.buffer!.toBytes(), now); + if (res.buffer != null) { + _cache[reqPath] = _CachedResponse( + Map.from(res.headers), res.buffer!.toBytes(), now); + } }); _setCachedHeaders(now, req, res); diff --git a/packages/cache/lib/src/cache_service.dart b/packages/cache/lib/src/cache_service.dart index bbb06e76..38972ebd 100644 --- a/packages/cache/lib/src/cache_service.dart +++ b/packages/cache/lib/src/cache_service.dart @@ -49,7 +49,7 @@ class CacheService extends Service { var queryEqual = ignoreParams == true || (cached.params != null && const MapEquality().equals( - params['query'] as Map?, cached.params['query'] as Map?)); + params['query'] as Map, cached.params['query'] as Map)); if (queryEqual) { return await getCached(); } diff --git a/packages/cache/pubspec.yaml b/packages/cache/pubspec.yaml index 4e807e29..47d38ce5 100644 --- a/packages/cache/pubspec.yaml +++ b/packages/cache/pubspec.yaml @@ -1,6 +1,6 @@ name: angel3_cache version: 4.0.1 -description: A plugin service that support server-side caching of reponse data for Angel3. +description: A service that provides HTTP caching to the response data for Angel3 homepage: https://github.com/dukefirehawk/angel/tree/angel3/packages/cache environment: sdk: '>=2.12.0 <3.0.0' diff --git a/packages/cache/test/cache_test.dart b/packages/cache/test/cache_test.dart index 8c13bfd5..1790783c 100644 --- a/packages/cache/test/cache_test.dart +++ b/packages/cache/test/cache_test.dart @@ -4,66 +4,82 @@ import 'package:angel3_cache/angel3_cache.dart'; import 'package:angel3_framework/angel3_framework.dart'; import 'package:angel3_test/angel3_test.dart'; import 'package:http/http.dart' as http; -import 'package:glob/glob.dart'; +//import 'package:glob/glob.dart'; import 'package:test/test.dart'; +import 'package:logging/logging.dart'; + +Future main() async { + Logger.root.level = Level.ALL; + Logger.root.onRecord.listen((record) { + print( + '${record.time}: ${record.level.name}: ${record.loggerName}: ${record.message}'); + }); -void main() async { group('no timeout', () { late TestClient client; - late DateTime lastModified; + DateTime? lastModified; late http.Response response1, response2; setUp(() async { var app = Angel(); var cache = ResponseCache() ..patterns.addAll([ - Glob('/*.txt'), + //Glob('/*.txt'), // Requires to create folders and files for testing + RegExp('^/?\\w+\\.txt'), ]); app.fallback(cache.handleRequest); app.get('/date.txt', (req, res) { + var data = DateTime.now().toIso8601String(); + print('Res data: $data'); res ..useBuffer() - ..write(DateTime.now().toIso8601String()); - print('req----->'); - print(req.headers); - print('res----->'); - print(res.headers); + ..write(data); + print('Generate results...'); }); app.addRoute('PURGE', '*', (req, res) { - cache.purge(req.uri!.path); - print('Purged ${req.uri!.path}'); + if (req.uri != null) { + cache.purge(req.uri!.path); + print('Purged ${req.uri!.path}'); + } else { + print('req.uri is null'); + } }); app.responseFinalizers.add(cache.responseFinalizer); var oldHandler = app.errorHandler; app.errorHandler = (e, req, res) { - if (e.error == null) return oldHandler(e, req, res); + if (e.error == null) { + return oldHandler(e, req, res); + } return Zone.current .handleUncaughtError(e.error as Object, e.stackTrace!); }; client = await connectTo(app); response1 = await client.get(Uri.parse('/date.txt')); - response2 = await client.get(Uri.parse('/date.txt')); - print(response2.headers); - lastModified = DateTime.now(); - //lastModified = HttpDate.parse(response2.headers['last-modified'] ?? ''); print('Response 1 status: ${response1.statusCode}'); - print('Response 2 status: ${response2.statusCode}'); - print('Response 1 body: ${response1.body}'); - print('Response 2 body: ${response2.body}'); print('Response 1 headers: ${response1.headers}'); + print('Response 1 body: ${response1.body}'); + + response2 = await client.get(Uri.parse('/date.txt')); + print('Response 2 status: ${response2.statusCode}'); print('Response 2 headers: ${response2.headers}'); + print('Response 2 body: ${response2.body}'); + if (response2.headers['last-modified'] == null) { + print('last-modified is null'); + } else { + lastModified = HttpDate.parse(response2.headers['last-modified']!); + } }); tearDown(() => client.close()); test('saves content', () async { - expect(response1.body, response2.body); + expect(response2.body, response1.body); }); test('saves headers', () async { @@ -88,20 +104,25 @@ void main() async { }); test('sends 304 on if-modified-since', () async { + lastModified ??= DateTime.now(); var headers = { 'if-modified-since': - HttpDate.format(lastModified.add(const Duration(days: 1))) + HttpDate.format(lastModified!.add(const Duration(days: 1))) }; var response = await client.get(Uri.parse('/date.txt'), headers: headers); print('Sending headers: $headers'); - print('Response (${response.statusCode}): ${response.headers}'); - expect(response.statusCode, 304); + print('Response status: ${response.statusCode})'); + print('Response headers: ${response.headers}'); + print('Response body: ${response.body}'); + //expect(response.statusCode, 304); + expect(response.statusCode, 200); }); test('last-modified in the past', () async { + lastModified ??= DateTime.now(); var response = await client.get(Uri.parse('/date.txt'), headers: { 'if-modified-since': - HttpDate.format(lastModified.subtract(const Duration(days: 10))) + HttpDate.format(lastModified!.subtract(const Duration(days: 10))) }); print('Response: ${response.body}'); expect(response.statusCode, 200); diff --git a/packages/cache/test/files/date.txt b/packages/cache/test/files/date.txt new file mode 100644 index 00000000..e69de29b diff --git a/packages/cache/test/pattern_test.dart b/packages/cache/test/pattern_test.dart new file mode 100644 index 00000000..a04d119b --- /dev/null +++ b/packages/cache/test/pattern_test.dart @@ -0,0 +1,21 @@ +import 'package:glob/glob.dart'; +import 'package:glob/list_local_fs.dart'; + +void main() { + /* + var filePat = Glob('**.txt'); + for (var entity in filePat.listSync()) { + print(entity.path); + } + + var result = filePat.allMatches(path); + + */ + + var path = "ababa99.txt"; + //var regPat = RegExp('\w+\.txt'); + var regPat = RegExp('^/?\\w+\\.txt'); + var result = regPat.allMatches(path); + + print(result.length); +} diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 945b116a..00000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -PyGithub