From cb25da3077a40aa79bd80dbe2620c411ad6391bd Mon Sep 17 00:00:00 2001 From: Thomas Hii Date: Fri, 19 Jul 2024 07:43:29 +0800 Subject: [PATCH 1/3] Updated Angel3 migration --- doc/deployment/docker/README.md | 4 +- packages/orm/angel_migration/CHANGELOG.md | 4 ++ packages/orm/angel_migration/README.md | 12 ++--- packages/orm/angel_migration/pubspec.yaml | 4 +- .../orm/angel_migration_runner/CHANGELOG.md | 7 +++ packages/orm/angel_migration_runner/README.md | 4 +- .../lib/src/mariadb/runner.dart | 4 +- .../lib/src/mysql/runner.dart | 6 ++- .../orm/angel_migration_runner/pubspec.yaml | 5 +- .../test/mariadb_test.dart | 31 +++++++++++ .../test/models/todo.dart | 53 +++++++++++++++++++ .../test/mysql_test.dart | 34 ++++++++++++ .../angel_migration_runner/test/pg_test.dart | 32 +++++++++++ 13 files changed, 178 insertions(+), 22 deletions(-) create mode 100644 packages/orm/angel_migration_runner/test/mariadb_test.dart create mode 100644 packages/orm/angel_migration_runner/test/models/todo.dart create mode 100644 packages/orm/angel_migration_runner/test/mysql_test.dart create mode 100644 packages/orm/angel_migration_runner/test/pg_test.dart diff --git a/doc/deployment/docker/README.md b/doc/deployment/docker/README.md index 8ee0e05a..c363f1be 100644 --- a/doc/deployment/docker/README.md +++ b/doc/deployment/docker/README.md @@ -1,6 +1,6 @@ -# Runing Ancillary Docker Services +# Docker Services -The required ancillary services required by the framework can be run using the compose files provided in this folder. +The required applications by the framework can be run using the docker compose files provided in this folder. ## PostreSQL diff --git a/packages/orm/angel_migration/CHANGELOG.md b/packages/orm/angel_migration/CHANGELOG.md index d1375bab..44977d14 100755 --- a/packages/orm/angel_migration/CHANGELOG.md +++ b/packages/orm/angel_migration/CHANGELOG.md @@ -1,5 +1,9 @@ # Change Log +## 8.2.1 + +* Updated README + ## 8.2.0 * Require Dart >= 3.3 diff --git a/packages/orm/angel_migration/README.md b/packages/orm/angel_migration/README.md index 232a633e..7b31ebb5 100755 --- a/packages/orm/angel_migration/README.md +++ b/packages/orm/angel_migration/README.md @@ -5,15 +5,9 @@ [![Discord](https://img.shields.io/discord/1060322353214660698)](https://discord.gg/3X6bxTUdCM) [![License](https://img.shields.io/github/license/dart-backend/angel)](https://github.com/dart-backend/angel/tree/master/packages/orm/angel_migration/LICENSE) -A basic database migration framework built for Angel3 ORM. +This package contains the abstract classes for implementing database migration in Angel3 framework. It is designed to work with Angel3 ORM. Please refer to the implementation in the [ORM Migration Runner]() package for more details. -## Supported database - -* PostgreSQL version 10 or later -* MariaDB 10.2.x or later -* MySQL 8.x or later - -## Features +## Supported Features * Create tables based on ORM models * Drop tables based on ORM models @@ -21,4 +15,4 @@ A basic database migration framework built for Angel3 ORM. ## Limitation -* Alter table/fields based on updated ORM models not supported +* Alter table/fields based on updated ORM models is not supported diff --git a/packages/orm/angel_migration/pubspec.yaml b/packages/orm/angel_migration/pubspec.yaml index d54b9e49..73c9ee85 100755 --- a/packages/orm/angel_migration/pubspec.yaml +++ b/packages/orm/angel_migration/pubspec.yaml @@ -1,6 +1,6 @@ name: angel3_migration -version: 8.2.0 -description: Database migration runtime for Angel3 ORM. Use this package to define schemas. +version: 8.2.1 +description: The abstract classes for implementing database migration in Angel3 framework. Designed to work with Angel3 ORM. homepage: https://angel3-framework.web.app/ repository: https://github.com/dart-backend/angel/tree/master/packages/orm/angel_migration environment: diff --git a/packages/orm/angel_migration_runner/CHANGELOG.md b/packages/orm/angel_migration_runner/CHANGELOG.md index 2b8a736a..08f41f70 100755 --- a/packages/orm/angel_migration_runner/CHANGELOG.md +++ b/packages/orm/angel_migration_runner/CHANGELOG.md @@ -1,5 +1,12 @@ # Change Log +## 8.2.1 + +* Updated README +* Added test cases for `PostgreSQL` +* Added test cases for `MySQL` +* Added test cases for `MariaDB` + ## 8.2.0 * Require Dart >= 3.3 diff --git a/packages/orm/angel_migration_runner/README.md b/packages/orm/angel_migration_runner/README.md index 628fb764..8c7fe159 100755 --- a/packages/orm/angel_migration_runner/README.md +++ b/packages/orm/angel_migration_runner/README.md @@ -5,9 +5,7 @@ [![Discord](https://img.shields.io/discord/1060322353214660698)](https://discord.gg/3X6bxTUdCM) [![License](https://img.shields.io/github/license/dart-backend/angel)](https://github.com/dart-backend/angel/tree/master/packages/orm/angel_migration_runner/LICENSE) -Database migration runner for Angel3 ORM. - -Supported database: +This package contains the implementation of the database migration for the following databases. It is designed to work with Angel3 ORM. * PostgreSQL 10.x or greater * MariaDB 10.2.x or greater diff --git a/packages/orm/angel_migration_runner/lib/src/mariadb/runner.dart b/packages/orm/angel_migration_runner/lib/src/mariadb/runner.dart index 24473657..73ad3710 100644 --- a/packages/orm/angel_migration_runner/lib/src/mariadb/runner.dart +++ b/packages/orm/angel_migration_runner/lib/src/mariadb/runner.dart @@ -17,8 +17,8 @@ class MariaDbMigrationRunner implements MigrationRunner { MariaDbMigrationRunner(this.connection, {Iterable migrations = const [], bool connected = false}) { - if (migrations.isNotEmpty == true) migrations.forEach(addMigration); - _connected = connected == true; + if (migrations.isNotEmpty) migrations.forEach(addMigration); + _connected = connected; } @override diff --git a/packages/orm/angel_migration_runner/lib/src/mysql/runner.dart b/packages/orm/angel_migration_runner/lib/src/mysql/runner.dart index 561ace41..29d606ab 100644 --- a/packages/orm/angel_migration_runner/lib/src/mysql/runner.dart +++ b/packages/orm/angel_migration_runner/lib/src/mysql/runner.dart @@ -17,8 +17,10 @@ class MySqlMigrationRunner implements MigrationRunner { MySqlMigrationRunner(this.connection, {Iterable migrations = const [], bool connected = false}) { - if (migrations.isNotEmpty == true) migrations.forEach(addMigration); - _connected = connected == true; + if (migrations.isNotEmpty) { + migrations.forEach(addMigration); + } + _connected = connected; } @override diff --git a/packages/orm/angel_migration_runner/pubspec.yaml b/packages/orm/angel_migration_runner/pubspec.yaml index 8c649fb1..a54efa61 100755 --- a/packages/orm/angel_migration_runner/pubspec.yaml +++ b/packages/orm/angel_migration_runner/pubspec.yaml @@ -1,6 +1,6 @@ name: angel3_migration_runner -version: 8.2.0 -description: Command-line based database migration runner for Angel3's ORM. +version: 8.2.1 +description: The implementation of database migration for Angel3 framework. Designed to work with Angel3 ORM. homepage: https://angel3-framework.web.app/ repository: https://github.com/dart-backend/angel/tree/master/packages/orm/angel_migration_runner environment: @@ -16,6 +16,7 @@ dependencies: logging: ^1.2.0 dev_dependencies: lints: ^4.0.0 + test: ^1.25.0 # dependency_overrides: # angel3_orm: # path: ../angel_orm diff --git a/packages/orm/angel_migration_runner/test/mariadb_test.dart b/packages/orm/angel_migration_runner/test/mariadb_test.dart new file mode 100644 index 00000000..6b9bbc47 --- /dev/null +++ b/packages/orm/angel_migration_runner/test/mariadb_test.dart @@ -0,0 +1,31 @@ +import 'dart:io'; + +import 'package:mysql1/mysql1.dart'; +import 'package:test/test.dart'; + +void main() async { + late MySqlConnection conn; + + setUp(() async { + var host = Platform.environment['MYSQL_HOST'] ?? 'localhost'; + var database = Platform.environment['MYSQL_DB'] ?? 'orm_test'; + var username = Platform.environment['MYSQL_USERNAME'] ?? 'test'; + var password = Platform.environment['MYSQL_PASSWORD'] ?? 'test123'; + + var settings = ConnectionSettings( + host: host, + port: 3306, + db: database, + user: username, + password: password); + conn = await MySqlConnection.connect(settings); + }); + + group('MariaDB', () { + test('migrate tables', () async {}); + }); + + tearDown(() async { + await conn.close(); + }); +} diff --git a/packages/orm/angel_migration_runner/test/models/todo.dart b/packages/orm/angel_migration_runner/test/models/todo.dart new file mode 100644 index 00000000..42304920 --- /dev/null +++ b/packages/orm/angel_migration_runner/test/models/todo.dart @@ -0,0 +1,53 @@ +import 'package:angel3_migration/angel3_migration.dart'; +import 'package:angel3_orm/angel3_orm.dart'; + +class UserMigration implements Migration { + @override + void up(Schema schema) { + schema.create('users', (table) { + table + ..serial('id').primaryKey() + ..varChar('username', length: 32).unique() + ..varChar('password') + ..boolean('account_confirmed').defaultsTo(false); + }); + } + + @override + void down(Schema schema) { + schema.drop('users'); + } +} + +class TodoMigration implements Migration { + @override + void up(Schema schema) { + schema.create('todos', (table) { + table + ..serial('id').primaryKey() + ..integer('user_id').references('users', 'id').onDeleteCascade() + ..varChar('text') + ..boolean('completed').defaultsTo(false); + }); + } + + @override + void down(Schema schema) { + schema.drop('todos'); + } +} + +class ItemMigration extends Migration { + @override + void up(Schema schema) { + schema.create('items', (table) { + table + ..serial('id').primaryKey() + ..varChar('name', length: 64) + ..timeStamp('created_at').defaultsTo(currentTimestamp); + }); + } + + @override + void down(Schema schema) => schema.drop('items'); +} diff --git a/packages/orm/angel_migration_runner/test/mysql_test.dart b/packages/orm/angel_migration_runner/test/mysql_test.dart new file mode 100644 index 00000000..a30c9fd2 --- /dev/null +++ b/packages/orm/angel_migration_runner/test/mysql_test.dart @@ -0,0 +1,34 @@ +import 'dart:io'; + +import 'package:mysql_client/mysql_client.dart'; +import 'package:test/test.dart'; + +void main() async { + late MySQLConnection conn; + + setUp(() async { + var host = Platform.environment['MYSQL_HOST'] ?? 'localhost'; + var database = Platform.environment['MYSQL_DB'] ?? 'orm_test'; + var username = Platform.environment['MYSQL_USERNAME'] ?? 'test'; + var password = Platform.environment['MYSQL_PASSWORD'] ?? 'test123'; + var secure = !('false' == Platform.environment['MYSQL_SECURE']); + + print("$host $database $username $password $secure"); + + conn = await MySQLConnection.createConnection( + databaseName: database, + port: 3306, + host: host, + userName: username, + password: password, + secure: secure); + }); + + group('Mysql', () { + test('migrate tables', () async {}); + }); + + tearDown(() async { + await conn.close(); + }); +} diff --git a/packages/orm/angel_migration_runner/test/pg_test.dart b/packages/orm/angel_migration_runner/test/pg_test.dart new file mode 100644 index 00000000..8d0db4b0 --- /dev/null +++ b/packages/orm/angel_migration_runner/test/pg_test.dart @@ -0,0 +1,32 @@ +import 'dart:io'; + +import 'package:postgres/postgres.dart'; +import 'package:test/test.dart'; + +void main() async { + late Connection conn; + + setUp(() async { + var host = Platform.environment['POSTGRES_HOST'] ?? 'localhost'; + var database = Platform.environment['POSTGRES_DB'] ?? 'orm_test'; + var username = Platform.environment['POSTGRES_USERNAME'] ?? 'test'; + var password = Platform.environment['POSTGRES_PASSWORD'] ?? 'test123'; + + print("$host $database $username $password"); + + conn = await Connection.open(Endpoint( + host: host, + port: 5432, + database: database, + username: username, + password: password)); + }); + + group('PostgreSQL', () { + test('migrate tables', () async {}); + }); + + tearDown(() async { + await conn.close(); + }); +} From 7e42a7e31757208c8b3cf3ef7054d43502aa8b24 Mon Sep 17 00:00:00 2001 From: Thomas Hii Date: Sat, 20 Jul 2024 10:23:45 +0800 Subject: [PATCH 2/3] Updated angel3 migration --- doc/deployment/docker/README.md | 38 +++++++++++++--- doc/deployment/docker/docker-compose-pg.yml | 2 +- .../orm/angel_migration_runner/CHANGELOG.md | 1 + .../angel_migration_runner/example/main.dart | 43 +++++++++++++----- .../lib/src/mariadb/runner.dart | 1 + .../lib/src/mariadb/schema.dart | 1 + .../lib/src/mysql/runner.dart | 1 + .../lib/src/mysql/schema.dart | 1 + .../lib/src/postgres/runner.dart | 24 ++++++---- .../lib/src/postgres/schema.dart | 1 + .../test/mariadb_test.dart | 22 ++++++++- .../test/mysql_test.dart | 36 ++++++++++++--- .../angel_migration_runner/test/pg_test.dart | 45 +++++++++++++++---- .../orm/angel_orm_postgres/test/common.dart | 4 +- 14 files changed, 176 insertions(+), 44 deletions(-) diff --git a/doc/deployment/docker/README.md b/doc/deployment/docker/README.md index c363f1be..9b9b201d 100644 --- a/doc/deployment/docker/README.md +++ b/doc/deployment/docker/README.md @@ -30,12 +30,12 @@ The required applications by the framework can be run using the docker compose f psql --username postgres ``` -### Create database, user and access +### Create PostgreSQL database, user and grant access - ```psql - postgres=# create database orm_test; - postgres=# create user test with encrypted password 'test123'; - postgres=# grant all privileges on database orm_test to test; + ```sql + create database orm_test; + create user test with encrypted password 'test123'; + grant all privileges on database orm_test to test; ``` ## MariaDB @@ -59,6 +59,20 @@ The required applications by the framework can be run using the docker compose f docker logs maria-mariadb-1 -f ``` +### Create MariaDB database, user and grant access + + ```sql + create database orm_test; + + -- Granting localhost access only + create user 'test'@'localhost' identified by 'test123'; + grant all privileges on orm_test.* to 'test'@'localhost'; + + -- Granting localhost and remote access + create user 'test'@'%' identified by 'test123'; + grant all privileges on orm_test.* to 'test'@'%'; + ``` + ## MySQL ### Starting the MySQL container @@ -80,6 +94,20 @@ The required applications by the framework can be run using the docker compose f docker logs mysql-mysql-1 -f ``` +### Create MySQL database, user and grant access + + ```sql + create database orm_test; + + -- Granting localhost access only + create user 'test'@'localhost' identified by 'test123'; + grant all privileges on orm_test.* to 'test'@'localhost'; + + -- Granting localhost and remote access + create user 'test'@'%' identified by 'test123'; + grant all privileges on orm_test.* to 'test'@'%'; + ``` + ## MongoDB ### Starting the MongoDB container diff --git a/doc/deployment/docker/docker-compose-pg.yml b/doc/deployment/docker/docker-compose-pg.yml index b0e21a6b..e89b67ae 100644 --- a/doc/deployment/docker/docker-compose-pg.yml +++ b/doc/deployment/docker/docker-compose-pg.yml @@ -10,7 +10,7 @@ services: volumes: - "db:/var/lib/postgresql/data" networks: - - webnet + - appnet pgadmin4: image: dpage/pgadmin4:latest diff --git a/packages/orm/angel_migration_runner/CHANGELOG.md b/packages/orm/angel_migration_runner/CHANGELOG.md index 08f41f70..b0da4f72 100755 --- a/packages/orm/angel_migration_runner/CHANGELOG.md +++ b/packages/orm/angel_migration_runner/CHANGELOG.md @@ -3,6 +3,7 @@ ## 8.2.1 * Updated README +* Updated examples * Added test cases for `PostgreSQL` * Added test cases for `MySQL` * Added test cases for `MariaDB` diff --git a/packages/orm/angel_migration_runner/example/main.dart b/packages/orm/angel_migration_runner/example/main.dart index d0ec7fe8..61ff0cd7 100755 --- a/packages/orm/angel_migration_runner/example/main.dart +++ b/packages/orm/angel_migration_runner/example/main.dart @@ -2,13 +2,23 @@ import 'dart:io'; import 'package:angel3_migration/angel3_migration.dart'; import 'package:angel3_migration_runner/angel3_migration_runner.dart'; +import 'package:angel3_migration_runner/mysql.dart'; import 'package:angel3_migration_runner/postgres.dart'; import 'package:angel3_orm/angel3_orm.dart'; +import 'package:mysql_client/mysql_client.dart'; import 'package:postgres/postgres.dart'; import 'todo.dart'; void main(List args) async { + // Run migration on PostgreSQL database + postgresqlMigration(args); + + // Run migration on MySQL database + mysqlMigration(args); +} + +void postgresqlMigration(List args) async { var host = Platform.environment['DB_HOST'] ?? 'localhost'; var database = Platform.environment['DB_NAME'] ?? 'demo'; var username = Platform.environment['DB_USERNAME'] ?? 'demouser'; @@ -16,14 +26,16 @@ void main(List args) async { print("$host $database $username $password"); - Connection conn = await Connection.open(Endpoint( - host: host, - port: 5432, - database: database, - username: username, - password: password)); + Connection conn = await Connection.open( + Endpoint( + host: host, + port: 5432, + database: database, + username: username, + password: password), + settings: ConnectionSettings(sslMode: SslMode.disable)); - var postgresqlMigrationRunner = PostgresMigrationRunner( + var runner = PostgresMigrationRunner( conn, migrations: [ UserMigration(), @@ -32,17 +44,25 @@ void main(List args) async { ], ); - /* + runMigrations(runner, args); +} + +void mysqlMigration(List args) async { + var host = Platform.environment['MYSQL_HOST'] ?? 'localhost'; + var database = Platform.environment['MYSQL_DB'] ?? 'orm_test'; + var username = Platform.environment['MYSQL_USERNAME'] ?? 'test'; + var password = Platform.environment['MYSQL_PASSWORD'] ?? 'test123'; + var mySQLConn = await MySQLConnection.createConnection( host: host, port: 3306, databaseName: database, userName: username, password: password, - secure: false); + secure: true); // ignore: unused_local_variable - var mysqlMigrationRunner = MySqlMigrationRunner( + var runner = MySqlMigrationRunner( mySQLConn, migrations: [ UserMigration(), @@ -50,9 +70,8 @@ void main(List args) async { FooMigration(), ], ); - */ - runMigrations(postgresqlMigrationRunner, args); + runMigrations(runner, args); } class FooMigration extends Migration { diff --git a/packages/orm/angel_migration_runner/lib/src/mariadb/runner.dart b/packages/orm/angel_migration_runner/lib/src/mariadb/runner.dart index 73ad3710..4a4623a1 100644 --- a/packages/orm/angel_migration_runner/lib/src/mariadb/runner.dart +++ b/packages/orm/angel_migration_runner/lib/src/mariadb/runner.dart @@ -7,6 +7,7 @@ import '../runner.dart'; import '../util.dart'; import 'schema.dart'; +/// A MariaDB database migration runner. class MariaDbMigrationRunner implements MigrationRunner { final _log = Logger('MariaDbMigrationRunner'); diff --git a/packages/orm/angel_migration_runner/lib/src/mariadb/schema.dart b/packages/orm/angel_migration_runner/lib/src/mariadb/schema.dart index 90a04d4f..0fccfbe7 100644 --- a/packages/orm/angel_migration_runner/lib/src/mariadb/schema.dart +++ b/packages/orm/angel_migration_runner/lib/src/mariadb/schema.dart @@ -5,6 +5,7 @@ import 'package:mysql1/mysql1.dart'; import 'table.dart'; +/// A MariaDB database schema generator class MariaDbSchema extends Schema { final _log = Logger('MariaDbSchema'); diff --git a/packages/orm/angel_migration_runner/lib/src/mysql/runner.dart b/packages/orm/angel_migration_runner/lib/src/mysql/runner.dart index 29d606ab..ad50fe56 100644 --- a/packages/orm/angel_migration_runner/lib/src/mysql/runner.dart +++ b/packages/orm/angel_migration_runner/lib/src/mysql/runner.dart @@ -7,6 +7,7 @@ import '../runner.dart'; import '../util.dart'; import 'schema.dart'; +/// A MySQL database migration runner. class MySqlMigrationRunner implements MigrationRunner { final _log = Logger('MysqlMigrationRunner'); diff --git a/packages/orm/angel_migration_runner/lib/src/mysql/schema.dart b/packages/orm/angel_migration_runner/lib/src/mysql/schema.dart index 20ab4ad0..8aff3a19 100644 --- a/packages/orm/angel_migration_runner/lib/src/mysql/schema.dart +++ b/packages/orm/angel_migration_runner/lib/src/mysql/schema.dart @@ -5,6 +5,7 @@ import 'package:mysql_client/mysql_client.dart'; import 'table.dart'; +/// A MySQL database schema generator class MySqlSchema extends Schema { final _log = Logger('MysqlSchema'); diff --git a/packages/orm/angel_migration_runner/lib/src/postgres/runner.dart b/packages/orm/angel_migration_runner/lib/src/postgres/runner.dart index 8f1b33bc..58590110 100755 --- a/packages/orm/angel_migration_runner/lib/src/postgres/runner.dart +++ b/packages/orm/angel_migration_runner/lib/src/postgres/runner.dart @@ -7,6 +7,7 @@ import '../runner.dart'; import '../util.dart'; import 'schema.dart'; +/// A PostgreSQL database migration runner class PostgresMigrationRunner implements MigrationRunner { final _log = Logger('PostgresMigrationRunner'); @@ -17,8 +18,10 @@ class PostgresMigrationRunner implements MigrationRunner { PostgresMigrationRunner(this.connection, {Iterable migrations = const [], bool connected = false}) { - if (migrations.isNotEmpty == true) migrations.forEach(addMigration); - _connected = connected == true; + if (migrations.isNotEmpty) { + migrations.forEach(addMigration); + } + _connected = connected; } @override @@ -26,17 +29,16 @@ class PostgresMigrationRunner implements MigrationRunner { _migrationQueue.addLast(migration); } - Future _init() async { + Future _init() async { while (_migrationQueue.isNotEmpty) { var migration = _migrationQueue.removeFirst(); var path = await absoluteSourcePath(migration.runtimeType); migrations.putIfAbsent(path.replaceAll('\\', '\\\\'), () => migration); } + _connected = connection.isOpen; if (!_connected) { - //await connection.open(); - //Connection.open(_endpoint!, settings: _settings); - _connected = true; + throw Exception("PostgreSQL connection is not open"); } await connection.execute(''' @@ -47,9 +49,10 @@ class PostgresMigrationRunner implements MigrationRunner { PRIMARY KEY(id) ); ''').then((result) { - _log.info('Check and create "migrations" table'); + _log.info('Created "migrations" table'); }).catchError((e) { _log.severe('Failed to create "migrations" table.'); + throw e; }); } @@ -73,8 +76,8 @@ class PostgresMigrationRunner implements MigrationRunner { var migration = migrations[k]!; var schema = PostgresSchema(); migration.up(schema); - _log.info('Added "$k" into "migrations" table.'); - await schema.run(connection).then((_) { + + var result = await schema.run(connection).then((_) { return connection.runTx((ctx) async { var result = await ctx.execute( "INSERT INTO MIGRATIONS (batch, path) VALUES ($batch, '$k')"); @@ -85,6 +88,9 @@ class PostgresMigrationRunner implements MigrationRunner { _log.severe('Failed to insert into "migrations" table.'); return -1; }); + if (result > 0) { + _log.info('Inserted "$k" into "migrations" table.'); + } } } else { _log.warning('Nothing to add into "migrations" table.'); diff --git a/packages/orm/angel_migration_runner/lib/src/postgres/schema.dart b/packages/orm/angel_migration_runner/lib/src/postgres/schema.dart index d287a983..279b6055 100755 --- a/packages/orm/angel_migration_runner/lib/src/postgres/schema.dart +++ b/packages/orm/angel_migration_runner/lib/src/postgres/schema.dart @@ -4,6 +4,7 @@ import 'package:postgres/postgres.dart'; import 'package:logging/logging.dart'; import 'table.dart'; +/// A PostgreSQL database schema generator class PostgresSchema extends Schema { final _log = Logger('PostgresSchema'); diff --git a/packages/orm/angel_migration_runner/test/mariadb_test.dart b/packages/orm/angel_migration_runner/test/mariadb_test.dart index 6b9bbc47..088affdf 100644 --- a/packages/orm/angel_migration_runner/test/mariadb_test.dart +++ b/packages/orm/angel_migration_runner/test/mariadb_test.dart @@ -1,12 +1,19 @@ import 'dart:io'; +import 'package:angel3_migration_runner/angel3_migration_runner.dart'; +import 'package:angel3_migration_runner/mariadb.dart'; import 'package:mysql1/mysql1.dart'; import 'package:test/test.dart'; +import 'models/todo.dart'; + void main() async { late MySqlConnection conn; + late MigrationRunner runner; setUp(() async { + print("Setup..."); + var host = Platform.environment['MYSQL_HOST'] ?? 'localhost'; var database = Platform.environment['MYSQL_DB'] ?? 'orm_test'; var username = Platform.environment['MYSQL_USERNAME'] ?? 'test'; @@ -19,13 +26,26 @@ void main() async { user: username, password: password); conn = await MySqlConnection.connect(settings); + + runner = MariaDbMigrationRunner( + conn, + migrations: [ + UserMigration(), + TodoMigration(), + ItemMigration(), + ], + ); }); group('MariaDB', () { - test('migrate tables', () async {}); + test('migrate tables', () async { + print("Test migration up"); + runner.up(); + }); }); tearDown(() async { + print("Teardown..."); await conn.close(); }); } diff --git a/packages/orm/angel_migration_runner/test/mysql_test.dart b/packages/orm/angel_migration_runner/test/mysql_test.dart index a30c9fd2..8d2e49f7 100644 --- a/packages/orm/angel_migration_runner/test/mysql_test.dart +++ b/packages/orm/angel_migration_runner/test/mysql_test.dart @@ -1,19 +1,26 @@ import 'dart:io'; +import 'package:angel3_migration_runner/angel3_migration_runner.dart'; +import 'package:angel3_migration_runner/mysql.dart'; import 'package:mysql_client/mysql_client.dart'; import 'package:test/test.dart'; +import 'models/todo.dart'; + void main() async { late MySQLConnection conn; + late MigrationRunner runner; setUp(() async { + print("Setup..."); + var host = Platform.environment['MYSQL_HOST'] ?? 'localhost'; var database = Platform.environment['MYSQL_DB'] ?? 'orm_test'; var username = Platform.environment['MYSQL_USERNAME'] ?? 'test'; var password = Platform.environment['MYSQL_PASSWORD'] ?? 'test123'; - var secure = !('false' == Platform.environment['MYSQL_SECURE']); + //var secure = !('false' == Platform.environment['MYSQL_SECURE']); - print("$host $database $username $password $secure"); + print("$host $database $username $password"); conn = await MySQLConnection.createConnection( databaseName: database, @@ -21,14 +28,31 @@ void main() async { host: host, userName: username, password: password, - secure: secure); + secure: true); + + await conn.connect(); + + runner = MySqlMigrationRunner( + conn, + migrations: [ + UserMigration(), + TodoMigration(), + ItemMigration(), + ], + ); }); - group('Mysql', () { - test('migrate tables', () async {}); + group('Mysql migrate tables', () { + test('up', () async { + print("Test migration up"); + await runner.up(); + }); }); tearDown(() async { - await conn.close(); + print("Teardown..."); + if (conn.connected) { + await conn.close(); + } }); } diff --git a/packages/orm/angel_migration_runner/test/pg_test.dart b/packages/orm/angel_migration_runner/test/pg_test.dart index 8d0db4b0..25c3f962 100644 --- a/packages/orm/angel_migration_runner/test/pg_test.dart +++ b/packages/orm/angel_migration_runner/test/pg_test.dart @@ -1,32 +1,59 @@ import 'dart:io'; +import 'package:angel3_migration_runner/angel3_migration_runner.dart'; +import 'package:angel3_migration_runner/postgres.dart'; import 'package:postgres/postgres.dart'; import 'package:test/test.dart'; +import 'models/todo.dart'; + void main() async { late Connection conn; + late MigrationRunner runner; setUp(() async { + print("Setup..."); + var host = Platform.environment['POSTGRES_HOST'] ?? 'localhost'; var database = Platform.environment['POSTGRES_DB'] ?? 'orm_test'; var username = Platform.environment['POSTGRES_USERNAME'] ?? 'test'; var password = Platform.environment['POSTGRES_PASSWORD'] ?? 'test123'; - print("$host $database $username $password"); + //print("$host $database $username $password"); - conn = await Connection.open(Endpoint( - host: host, - port: 5432, - database: database, - username: username, - password: password)); + conn = await Connection.open( + Endpoint( + host: host, + port: 5432, + database: database, + username: username, + password: password), + settings: ConnectionSettings(sslMode: SslMode.disable)); + + runner = PostgresMigrationRunner( + conn, + migrations: [ + UserMigration(), + TodoMigration(), + ItemMigration(), + ], + ); }); - group('PostgreSQL', () { - test('migrate tables', () async {}); + group('PostgreSQL migrate tables', () { + test('up', () async { + print("Test migration up"); + await runner.up(); + }); + + test('reset', () async { + print("Test migration reset"); + await runner.reset(); + }); }); tearDown(() async { + print("Teardown..."); await conn.close(); }); } diff --git a/packages/orm/angel_orm_postgres/test/common.dart b/packages/orm/angel_orm_postgres/test/common.dart index 6464b73f..0b04bcd7 100644 --- a/packages/orm/angel_orm_postgres/test/common.dart +++ b/packages/orm/angel_orm_postgres/test/common.dart @@ -89,7 +89,9 @@ Future connectToPostgresPool( ) ], settings: PoolSettings( - maxConnectionAge: Duration(hours: 1), maxConnectionCount: 5)); + maxConnectionAge: Duration(hours: 1), + maxConnectionCount: 5, + sslMode: SslMode.disable)); // Run sql to create the tables in a transaction await dbPool.runTx((conn) async { From 5ed443ae33d99d81958c86ebda6f80cb103ddeb8 Mon Sep 17 00:00:00 2001 From: Thomas Hii Date: Sat, 20 Jul 2024 15:37:25 +0800 Subject: [PATCH 3/3] Fixed MySQL migration --- packages/orm/angel_migration/README.md | 2 +- .../orm/angel_migration_runner/CHANGELOG.md | 7 +-- .../angel_migration_runner/lib/src/cli.dart | 4 +- .../lib/src/mariadb/runner.dart | 2 +- .../lib/src/mariadb/table.dart | 8 ++- .../lib/src/mysql/runner.dart | 27 ++++++---- .../lib/src/mysql/schema.dart | 2 + .../lib/src/mysql/table.dart | 17 +++++- .../lib/src/postgres/runner.dart | 2 +- .../test/mariadb_test.dart | 2 +- .../test/models/mysql_todo.dart | 53 +++++++++++++++++++ .../test/models/{todo.dart => pg_todo.dart} | 0 .../test/mysql_test.dart | 7 ++- .../angel_migration_runner/test/pg_test.dart | 2 +- 14 files changed, 113 insertions(+), 22 deletions(-) create mode 100644 packages/orm/angel_migration_runner/test/models/mysql_todo.dart rename packages/orm/angel_migration_runner/test/models/{todo.dart => pg_todo.dart} (100%) diff --git a/packages/orm/angel_migration/README.md b/packages/orm/angel_migration/README.md index 7b31ebb5..3ce74bd9 100755 --- a/packages/orm/angel_migration/README.md +++ b/packages/orm/angel_migration/README.md @@ -11,7 +11,7 @@ This package contains the abstract classes for implementing database migration i * Create tables based on ORM models * Drop tables based on ORM models -* Add new tables based ORM models +* Add new tables based on ORM models ## Limitation diff --git a/packages/orm/angel_migration_runner/CHANGELOG.md b/packages/orm/angel_migration_runner/CHANGELOG.md index b0da4f72..0ddc92d3 100755 --- a/packages/orm/angel_migration_runner/CHANGELOG.md +++ b/packages/orm/angel_migration_runner/CHANGELOG.md @@ -4,9 +4,10 @@ * Updated README * Updated examples -* Added test cases for `PostgreSQL` -* Added test cases for `MySQL` -* Added test cases for `MariaDB` +* Updated `PostgresMigrationRunner` error handling +* Fixed `MySqlMigrationRunner` migration issues +* Added test cases for `PostgreSQL`, `MySQL` and `MariaDB` +* Added auto increment integer primary key suppport to MySQL and MariaDB ## 8.2.0 diff --git a/packages/orm/angel_migration_runner/lib/src/cli.dart b/packages/orm/angel_migration_runner/lib/src/cli.dart index 37cce4c6..b3c5abe7 100755 --- a/packages/orm/angel_migration_runner/lib/src/cli.dart +++ b/packages/orm/angel_migration_runner/lib/src/cli.dart @@ -50,6 +50,7 @@ class _RefreshCommand extends Command { @override String get name => 'refresh'; + @override String get description => 'Resets the database, and then re-runs all migrations.'; @@ -67,8 +68,9 @@ class _RollbackCommand extends Command { @override String get name => 'rollback'; + @override - String get description => 'Undoes the last batch of migrations.'; + String get description => 'Undo the last batch of migrations.'; final MigrationRunner migrationRunner; diff --git a/packages/orm/angel_migration_runner/lib/src/mariadb/runner.dart b/packages/orm/angel_migration_runner/lib/src/mariadb/runner.dart index 4a4623a1..a4a344eb 100644 --- a/packages/orm/angel_migration_runner/lib/src/mariadb/runner.dart +++ b/packages/orm/angel_migration_runner/lib/src/mariadb/runner.dart @@ -40,7 +40,7 @@ class MariaDbMigrationRunner implements MigrationRunner { await connection.query(''' CREATE TABLE IF NOT EXISTS migrations ( - id serial, + id integer NOT NULL AUTO_INCREMENT, batch integer, path varchar(255), PRIMARY KEY(id) diff --git a/packages/orm/angel_migration_runner/lib/src/mariadb/table.dart b/packages/orm/angel_migration_runner/lib/src/mariadb/table.dart index 748505a7..405e8273 100644 --- a/packages/orm/angel_migration_runner/lib/src/mariadb/table.dart +++ b/packages/orm/angel_migration_runner/lib/src/mariadb/table.dart @@ -48,6 +48,12 @@ abstract class MariaDbGenerator { buf.write(' UNIQUE'); } else if (column.indexType == IndexType.primaryKey) { buf.write(' PRIMARY KEY'); + + // For int based primary key, apply NOT NULL + // and AUTO_INCREMENT + if (column.type == ColumnType.int) { + buf.write(' NOT NULL AUTO_INCREMENT'); + } } for (var ref in column.externalReferences) { @@ -105,7 +111,7 @@ class MariaDbTable extends Table { if (indexBuf.isNotEmpty) { for (var i = 0; i < indexBuf.length; i++) { - buf.write(',\n${indexBuf[$1]}'); + buf.write(',\n${indexBuf[i]}'); } } } diff --git a/packages/orm/angel_migration_runner/lib/src/mysql/runner.dart b/packages/orm/angel_migration_runner/lib/src/mysql/runner.dart index ad50fe56..ab2020c2 100644 --- a/packages/orm/angel_migration_runner/lib/src/mysql/runner.dart +++ b/packages/orm/angel_migration_runner/lib/src/mysql/runner.dart @@ -42,15 +42,18 @@ class MySqlMigrationRunner implements MigrationRunner { await connection.execute(''' CREATE TABLE IF NOT EXISTS migrations ( - id serial, + id integer NOT NULL AUTO_INCREMENT, batch integer, - path varchar(255), + path varchar(500), PRIMARY KEY(id) ); ''').then((result) { + //print(result); _log.info('Check and create "migrations" table'); }).catchError((e) { + //print(e); _log.severe('Failed to create "migrations" table.'); + throw e; }); } @@ -59,8 +62,9 @@ class MySqlMigrationRunner implements MigrationRunner { await _init(); var result = await connection.execute('SELECT path from migrations;'); var existing = []; - if (result.rows.isNotEmpty) { - existing = result.rows.first.assoc().values.cast().toList(); + for (var item in result.rows) { + var rec = item.assoc().values.first ?? ""; + existing.add(rec.replaceAll("\\", "\\\\")); } var toRun = []; @@ -113,8 +117,9 @@ class MySqlMigrationRunner implements MigrationRunner { result = await connection .execute('SELECT path from migrations WHERE batch = $curBatch;'); var existing = []; - if (result.rows.isNotEmpty) { - existing = result.rows.first.assoc().values.cast().toList(); + for (var item in result.rows) { + var rec = item.assoc().values.first ?? ""; + existing.add(rec.replaceAll("\\", "\\\\")); } var toRun = []; @@ -143,11 +148,15 @@ class MySqlMigrationRunner implements MigrationRunner { await _init(); var result = await connection .execute('SELECT path from migrations ORDER BY batch DESC;'); + + // "mysql_client" driver will auto convert path containing "\\" to "\". + // So need to revert "\" back to "\\" for the migration logic to work var existing = []; - if (result.rows.isNotEmpty) { - var firstRow = result.rows.first; - existing = firstRow.assoc().values.cast().toList(); + for (var item in result.rows) { + var rec = item.assoc().values.first ?? ""; + existing.add(rec.replaceAll("\\", "\\\\")); } + var toRun = existing.where(migrations.containsKey).toList(); if (toRun.isNotEmpty) { diff --git a/packages/orm/angel_migration_runner/lib/src/mysql/schema.dart b/packages/orm/angel_migration_runner/lib/src/mysql/schema.dart index 8aff3a19..0182e6f9 100644 --- a/packages/orm/angel_migration_runner/lib/src/mysql/schema.dart +++ b/packages/orm/angel_migration_runner/lib/src/mysql/schema.dart @@ -22,11 +22,13 @@ class MySqlSchema extends Schema { await connection.transactional((ctx) async { var sql = compile(); var result = await ctx.execute(sql).catchError((e) { + print(e); _log.severe('Failed to run query: [ $sql ]', e); throw Exception(e); }); affectedRows = result.affectedRows.toInt(); }).catchError((e) { + print(e); _log.severe('Failed to run query in a transaction', e); }); diff --git a/packages/orm/angel_migration_runner/lib/src/mysql/table.dart b/packages/orm/angel_migration_runner/lib/src/mysql/table.dart index 407f7358..dd2520db 100644 --- a/packages/orm/angel_migration_runner/lib/src/mysql/table.dart +++ b/packages/orm/angel_migration_runner/lib/src/mysql/table.dart @@ -10,6 +10,7 @@ abstract class MySqlGenerator { if (column.type == ColumnType.timeStamp) { str = ColumnType.dateTime.name; } + if (column.type.hasLength) { return '$str(${column.length})'; } else { @@ -20,7 +21,11 @@ abstract class MySqlGenerator { static String compileColumn(MigrationColumn column) { var buf = StringBuffer(columnType(column)); - if (column.isNullable == false) buf.write(' NOT NULL'); + if (!column.isNullable) { + buf.write(' NOT NULL'); + } + + // Default value if (column.defaultValue != null) { String s; var value = column.defaultValue; @@ -47,6 +52,12 @@ abstract class MySqlGenerator { buf.write(' UNIQUE'); } else if (column.indexType == IndexType.primaryKey) { buf.write(' PRIMARY KEY'); + + // For int based primary key, apply NOT NULL + // and AUTO_INCREMENT + if (column.type == ColumnType.int) { + buf.write(' NOT NULL AUTO_INCREMENT'); + } } for (var ref in column.externalReferences) { @@ -104,9 +115,11 @@ class MysqlTable extends Table { if (indexBuf.isNotEmpty) { for (var i = 0; i < indexBuf.length; i++) { - buf.write(',\n${indexBuf[$1]}'); + buf.write(',\n${indexBuf[i]}'); } } + + print(buf); } } diff --git a/packages/orm/angel_migration_runner/lib/src/postgres/runner.dart b/packages/orm/angel_migration_runner/lib/src/postgres/runner.dart index 58590110..05a6344e 100755 --- a/packages/orm/angel_migration_runner/lib/src/postgres/runner.dart +++ b/packages/orm/angel_migration_runner/lib/src/postgres/runner.dart @@ -45,7 +45,7 @@ class PostgresMigrationRunner implements MigrationRunner { CREATE TABLE IF NOT EXISTS "migrations" ( id serial, batch integer, - path varchar, + path varchar(500), PRIMARY KEY(id) ); ''').then((result) { diff --git a/packages/orm/angel_migration_runner/test/mariadb_test.dart b/packages/orm/angel_migration_runner/test/mariadb_test.dart index 088affdf..19edc8dd 100644 --- a/packages/orm/angel_migration_runner/test/mariadb_test.dart +++ b/packages/orm/angel_migration_runner/test/mariadb_test.dart @@ -5,7 +5,7 @@ import 'package:angel3_migration_runner/mariadb.dart'; import 'package:mysql1/mysql1.dart'; import 'package:test/test.dart'; -import 'models/todo.dart'; +import 'models/mysql_todo.dart'; void main() async { late MySqlConnection conn; diff --git a/packages/orm/angel_migration_runner/test/models/mysql_todo.dart b/packages/orm/angel_migration_runner/test/models/mysql_todo.dart new file mode 100644 index 00000000..178aef4b --- /dev/null +++ b/packages/orm/angel_migration_runner/test/models/mysql_todo.dart @@ -0,0 +1,53 @@ +import 'package:angel3_migration/angel3_migration.dart'; +import 'package:angel3_orm/angel3_orm.dart'; + +class UserMigration implements Migration { + @override + void up(Schema schema) { + schema.create('users', (table) { + table + ..integer('id').primaryKey() + ..varChar('username', length: 32).unique() + ..varChar('password') + ..boolean('account_confirmed').defaultsTo(false); + }); + } + + @override + void down(Schema schema) { + schema.drop('users'); + } +} + +class TodoMigration implements Migration { + @override + void up(Schema schema) { + schema.create('todos', (table) { + table + ..integer('id').primaryKey() + ..integer('user_id').references('users', 'id').onDeleteCascade() + ..varChar('text') + ..boolean('completed').defaultsTo(false); + }); + } + + @override + void down(Schema schema) { + schema.drop('todos'); + } +} + +class ItemMigration extends Migration { + @override + void up(Schema schema) { + schema.create('items', (table) { + table + ..integer('id').primaryKey() + ..varChar('name', length: 64) + ..timeStamp('created_at').defaultsTo(currentTimestamp); + }); + } + + @override + void down(Schema schema) => schema.drop('items'); +} diff --git a/packages/orm/angel_migration_runner/test/models/todo.dart b/packages/orm/angel_migration_runner/test/models/pg_todo.dart similarity index 100% rename from packages/orm/angel_migration_runner/test/models/todo.dart rename to packages/orm/angel_migration_runner/test/models/pg_todo.dart diff --git a/packages/orm/angel_migration_runner/test/mysql_test.dart b/packages/orm/angel_migration_runner/test/mysql_test.dart index 8d2e49f7..ff062397 100644 --- a/packages/orm/angel_migration_runner/test/mysql_test.dart +++ b/packages/orm/angel_migration_runner/test/mysql_test.dart @@ -5,7 +5,7 @@ import 'package:angel3_migration_runner/mysql.dart'; import 'package:mysql_client/mysql_client.dart'; import 'package:test/test.dart'; -import 'models/todo.dart'; +import 'models/mysql_todo.dart'; void main() async { late MySQLConnection conn; @@ -47,6 +47,11 @@ void main() async { print("Test migration up"); await runner.up(); }); + + test('reset', () async { + print("Test migration reset"); + await runner.reset(); + }); }); tearDown(() async { diff --git a/packages/orm/angel_migration_runner/test/pg_test.dart b/packages/orm/angel_migration_runner/test/pg_test.dart index 25c3f962..8075bcf3 100644 --- a/packages/orm/angel_migration_runner/test/pg_test.dart +++ b/packages/orm/angel_migration_runner/test/pg_test.dart @@ -5,7 +5,7 @@ import 'package:angel3_migration_runner/postgres.dart'; import 'package:postgres/postgres.dart'; import 'package:test/test.dart'; -import 'models/todo.dart'; +import 'models/pg_todo.dart'; void main() async { late Connection conn;