Merge pull request #144 from dukefirehawk/bugfix/mysql

Bugfix/mysql
This commit is contained in:
Thomas 2024-07-20 15:58:45 +08:00 committed by GitHub
commit 19a7f8316d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
24 changed files with 445 additions and 66 deletions

View file

@ -1,6 +1,6 @@
# Runing Ancillary Docker Services
# Docker Services
The required ancillary services required by the framework can be run using the compose files provided in this folder.
The required applications by the framework can be run using the docker compose files provided in this folder.
## PostreSQL
@ -30,12 +30,12 @@ The required ancillary services required by the framework can be run using the c
psql --username postgres
```
### Create database, user and access
### Create PostgreSQL database, user and grant access
```psql
postgres=# create database orm_test;
postgres=# create user test with encrypted password 'test123';
postgres=# grant all privileges on database orm_test to test;
```sql
create database orm_test;
create user test with encrypted password 'test123';
grant all privileges on database orm_test to test;
```
## MariaDB
@ -59,6 +59,20 @@ The required ancillary services required by the framework can be run using the c
docker logs maria-mariadb-1 -f
```
### Create MariaDB database, user and grant access
```sql
create database orm_test;
-- Granting localhost access only
create user 'test'@'localhost' identified by 'test123';
grant all privileges on orm_test.* to 'test'@'localhost';
-- Granting localhost and remote access
create user 'test'@'%' identified by 'test123';
grant all privileges on orm_test.* to 'test'@'%';
```
## MySQL
### Starting the MySQL container
@ -80,6 +94,20 @@ The required ancillary services required by the framework can be run using the c
docker logs mysql-mysql-1 -f
```
### Create MySQL database, user and grant access
```sql
create database orm_test;
-- Granting localhost access only
create user 'test'@'localhost' identified by 'test123';
grant all privileges on orm_test.* to 'test'@'localhost';
-- Granting localhost and remote access
create user 'test'@'%' identified by 'test123';
grant all privileges on orm_test.* to 'test'@'%';
```
## MongoDB
### Starting the MongoDB container

View file

@ -10,7 +10,7 @@ services:
volumes:
- "db:/var/lib/postgresql/data"
networks:
- webnet
- appnet
pgadmin4:
image: dpage/pgadmin4:latest

View file

@ -1,5 +1,9 @@
# Change Log
## 8.2.1
* Updated README
## 8.2.0
* Require Dart >= 3.3

View file

@ -5,20 +5,14 @@
[![Discord](https://img.shields.io/discord/1060322353214660698)](https://discord.gg/3X6bxTUdCM)
[![License](https://img.shields.io/github/license/dart-backend/angel)](https://github.com/dart-backend/angel/tree/master/packages/orm/angel_migration/LICENSE)
A basic database migration framework built for Angel3 ORM.
This package contains the abstract classes for implementing database migration in Angel3 framework. It is designed to work with Angel3 ORM. Please refer to the implementation in the [ORM Migration Runner](<https://pub.dev/packages/angel3_migration_runner>) package for more details.
## Supported database
* PostgreSQL version 10 or later
* MariaDB 10.2.x or later
* MySQL 8.x or later
## Features
## Supported Features
* Create tables based on ORM models
* Drop tables based on ORM models
* Add new tables based ORM models
* Add new tables based on ORM models
## Limitation
* Alter table/fields based on updated ORM models not supported
* Alter table/fields based on updated ORM models is not supported

View file

@ -1,6 +1,6 @@
name: angel3_migration
version: 8.2.0
description: Database migration runtime for Angel3 ORM. Use this package to define schemas.
version: 8.2.1
description: The abstract classes for implementing database migration in Angel3 framework. Designed to work with Angel3 ORM.
homepage: https://angel3-framework.web.app/
repository: https://github.com/dart-backend/angel/tree/master/packages/orm/angel_migration
environment:

View file

@ -1,5 +1,14 @@
# Change Log
## 8.2.1
* Updated README
* Updated examples
* Updated `PostgresMigrationRunner` error handling
* Fixed `MySqlMigrationRunner` migration issues
* Added test cases for `PostgreSQL`, `MySQL` and `MariaDB`
* Added auto increment integer primary key suppport to MySQL and MariaDB
## 8.2.0
* Require Dart >= 3.3

View file

@ -5,9 +5,7 @@
[![Discord](https://img.shields.io/discord/1060322353214660698)](https://discord.gg/3X6bxTUdCM)
[![License](https://img.shields.io/github/license/dart-backend/angel)](https://github.com/dart-backend/angel/tree/master/packages/orm/angel_migration_runner/LICENSE)
Database migration runner for Angel3 ORM.
Supported database:
This package contains the implementation of the database migration for the following databases. It is designed to work with Angel3 ORM.
* PostgreSQL 10.x or greater
* MariaDB 10.2.x or greater

View file

@ -2,13 +2,23 @@ import 'dart:io';
import 'package:angel3_migration/angel3_migration.dart';
import 'package:angel3_migration_runner/angel3_migration_runner.dart';
import 'package:angel3_migration_runner/mysql.dart';
import 'package:angel3_migration_runner/postgres.dart';
import 'package:angel3_orm/angel3_orm.dart';
import 'package:mysql_client/mysql_client.dart';
import 'package:postgres/postgres.dart';
import 'todo.dart';
void main(List<String> args) async {
// Run migration on PostgreSQL database
postgresqlMigration(args);
// Run migration on MySQL database
mysqlMigration(args);
}
void postgresqlMigration(List<String> args) async {
var host = Platform.environment['DB_HOST'] ?? 'localhost';
var database = Platform.environment['DB_NAME'] ?? 'demo';
var username = Platform.environment['DB_USERNAME'] ?? 'demouser';
@ -16,14 +26,16 @@ void main(List<String> args) async {
print("$host $database $username $password");
Connection conn = await Connection.open(Endpoint(
host: host,
port: 5432,
database: database,
username: username,
password: password));
Connection conn = await Connection.open(
Endpoint(
host: host,
port: 5432,
database: database,
username: username,
password: password),
settings: ConnectionSettings(sslMode: SslMode.disable));
var postgresqlMigrationRunner = PostgresMigrationRunner(
var runner = PostgresMigrationRunner(
conn,
migrations: [
UserMigration(),
@ -32,17 +44,25 @@ void main(List<String> args) async {
],
);
/*
runMigrations(runner, args);
}
void mysqlMigration(List<String> args) async {
var host = Platform.environment['MYSQL_HOST'] ?? 'localhost';
var database = Platform.environment['MYSQL_DB'] ?? 'orm_test';
var username = Platform.environment['MYSQL_USERNAME'] ?? 'test';
var password = Platform.environment['MYSQL_PASSWORD'] ?? 'test123';
var mySQLConn = await MySQLConnection.createConnection(
host: host,
port: 3306,
databaseName: database,
userName: username,
password: password,
secure: false);
secure: true);
// ignore: unused_local_variable
var mysqlMigrationRunner = MySqlMigrationRunner(
var runner = MySqlMigrationRunner(
mySQLConn,
migrations: [
UserMigration(),
@ -50,9 +70,8 @@ void main(List<String> args) async {
FooMigration(),
],
);
*/
runMigrations(postgresqlMigrationRunner, args);
runMigrations(runner, args);
}
class FooMigration extends Migration {

View file

@ -50,6 +50,7 @@ class _RefreshCommand extends Command {
@override
String get name => 'refresh';
@override
String get description =>
'Resets the database, and then re-runs all migrations.';
@ -67,8 +68,9 @@ class _RollbackCommand extends Command {
@override
String get name => 'rollback';
@override
String get description => 'Undoes the last batch of migrations.';
String get description => 'Undo the last batch of migrations.';
final MigrationRunner migrationRunner;

View file

@ -7,6 +7,7 @@ import '../runner.dart';
import '../util.dart';
import 'schema.dart';
/// A MariaDB database migration runner.
class MariaDbMigrationRunner implements MigrationRunner {
final _log = Logger('MariaDbMigrationRunner');
@ -17,8 +18,8 @@ class MariaDbMigrationRunner implements MigrationRunner {
MariaDbMigrationRunner(this.connection,
{Iterable<Migration> migrations = const [], bool connected = false}) {
if (migrations.isNotEmpty == true) migrations.forEach(addMigration);
_connected = connected == true;
if (migrations.isNotEmpty) migrations.forEach(addMigration);
_connected = connected;
}
@override
@ -39,7 +40,7 @@ class MariaDbMigrationRunner implements MigrationRunner {
await connection.query('''
CREATE TABLE IF NOT EXISTS migrations (
id serial,
id integer NOT NULL AUTO_INCREMENT,
batch integer,
path varchar(255),
PRIMARY KEY(id)

View file

@ -5,6 +5,7 @@ import 'package:mysql1/mysql1.dart';
import 'table.dart';
/// A MariaDB database schema generator
class MariaDbSchema extends Schema {
final _log = Logger('MariaDbSchema');

View file

@ -48,6 +48,12 @@ abstract class MariaDbGenerator {
buf.write(' UNIQUE');
} else if (column.indexType == IndexType.primaryKey) {
buf.write(' PRIMARY KEY');
// For int based primary key, apply NOT NULL
// and AUTO_INCREMENT
if (column.type == ColumnType.int) {
buf.write(' NOT NULL AUTO_INCREMENT');
}
}
for (var ref in column.externalReferences) {
@ -105,7 +111,7 @@ class MariaDbTable extends Table {
if (indexBuf.isNotEmpty) {
for (var i = 0; i < indexBuf.length; i++) {
buf.write(',\n${indexBuf[$1]}');
buf.write(',\n${indexBuf[i]}');
}
}
}

View file

@ -7,6 +7,7 @@ import '../runner.dart';
import '../util.dart';
import 'schema.dart';
/// A MySQL database migration runner.
class MySqlMigrationRunner implements MigrationRunner {
final _log = Logger('MysqlMigrationRunner');
@ -17,8 +18,10 @@ class MySqlMigrationRunner implements MigrationRunner {
MySqlMigrationRunner(this.connection,
{Iterable<Migration> migrations = const [], bool connected = false}) {
if (migrations.isNotEmpty == true) migrations.forEach(addMigration);
_connected = connected == true;
if (migrations.isNotEmpty) {
migrations.forEach(addMigration);
}
_connected = connected;
}
@override
@ -39,15 +42,18 @@ class MySqlMigrationRunner implements MigrationRunner {
await connection.execute('''
CREATE TABLE IF NOT EXISTS migrations (
id serial,
id integer NOT NULL AUTO_INCREMENT,
batch integer,
path varchar(255),
path varchar(500),
PRIMARY KEY(id)
);
''').then((result) {
//print(result);
_log.info('Check and create "migrations" table');
}).catchError((e) {
//print(e);
_log.severe('Failed to create "migrations" table.');
throw e;
});
}
@ -56,8 +62,9 @@ class MySqlMigrationRunner implements MigrationRunner {
await _init();
var result = await connection.execute('SELECT path from migrations;');
var existing = <String>[];
if (result.rows.isNotEmpty) {
existing = result.rows.first.assoc().values.cast<String>().toList();
for (var item in result.rows) {
var rec = item.assoc().values.first ?? "";
existing.add(rec.replaceAll("\\", "\\\\"));
}
var toRun = <String>[];
@ -110,8 +117,9 @@ class MySqlMigrationRunner implements MigrationRunner {
result = await connection
.execute('SELECT path from migrations WHERE batch = $curBatch;');
var existing = <String>[];
if (result.rows.isNotEmpty) {
existing = result.rows.first.assoc().values.cast<String>().toList();
for (var item in result.rows) {
var rec = item.assoc().values.first ?? "";
existing.add(rec.replaceAll("\\", "\\\\"));
}
var toRun = <String>[];
@ -140,11 +148,15 @@ class MySqlMigrationRunner implements MigrationRunner {
await _init();
var result = await connection
.execute('SELECT path from migrations ORDER BY batch DESC;');
// "mysql_client" driver will auto convert path containing "\\" to "\".
// So need to revert "\" back to "\\" for the migration logic to work
var existing = <String>[];
if (result.rows.isNotEmpty) {
var firstRow = result.rows.first;
existing = firstRow.assoc().values.cast<String>().toList();
for (var item in result.rows) {
var rec = item.assoc().values.first ?? "";
existing.add(rec.replaceAll("\\", "\\\\"));
}
var toRun = existing.where(migrations.containsKey).toList();
if (toRun.isNotEmpty) {

View file

@ -5,6 +5,7 @@ import 'package:mysql_client/mysql_client.dart';
import 'table.dart';
/// A MySQL database schema generator
class MySqlSchema extends Schema {
final _log = Logger('MysqlSchema');
@ -21,11 +22,13 @@ class MySqlSchema extends Schema {
await connection.transactional((ctx) async {
var sql = compile();
var result = await ctx.execute(sql).catchError((e) {
print(e);
_log.severe('Failed to run query: [ $sql ]', e);
throw Exception(e);
});
affectedRows = result.affectedRows.toInt();
}).catchError((e) {
print(e);
_log.severe('Failed to run query in a transaction', e);
});

View file

@ -10,6 +10,7 @@ abstract class MySqlGenerator {
if (column.type == ColumnType.timeStamp) {
str = ColumnType.dateTime.name;
}
if (column.type.hasLength) {
return '$str(${column.length})';
} else {
@ -20,7 +21,11 @@ abstract class MySqlGenerator {
static String compileColumn(MigrationColumn column) {
var buf = StringBuffer(columnType(column));
if (column.isNullable == false) buf.write(' NOT NULL');
if (!column.isNullable) {
buf.write(' NOT NULL');
}
// Default value
if (column.defaultValue != null) {
String s;
var value = column.defaultValue;
@ -47,6 +52,12 @@ abstract class MySqlGenerator {
buf.write(' UNIQUE');
} else if (column.indexType == IndexType.primaryKey) {
buf.write(' PRIMARY KEY');
// For int based primary key, apply NOT NULL
// and AUTO_INCREMENT
if (column.type == ColumnType.int) {
buf.write(' NOT NULL AUTO_INCREMENT');
}
}
for (var ref in column.externalReferences) {
@ -104,9 +115,11 @@ class MysqlTable extends Table {
if (indexBuf.isNotEmpty) {
for (var i = 0; i < indexBuf.length; i++) {
buf.write(',\n${indexBuf[$1]}');
buf.write(',\n${indexBuf[i]}');
}
}
print(buf);
}
}

View file

@ -7,6 +7,7 @@ import '../runner.dart';
import '../util.dart';
import 'schema.dart';
/// A PostgreSQL database migration runner
class PostgresMigrationRunner implements MigrationRunner {
final _log = Logger('PostgresMigrationRunner');
@ -17,8 +18,10 @@ class PostgresMigrationRunner implements MigrationRunner {
PostgresMigrationRunner(this.connection,
{Iterable<Migration> migrations = const [], bool connected = false}) {
if (migrations.isNotEmpty == true) migrations.forEach(addMigration);
_connected = connected == true;
if (migrations.isNotEmpty) {
migrations.forEach(addMigration);
}
_connected = connected;
}
@override
@ -26,30 +29,30 @@ class PostgresMigrationRunner implements MigrationRunner {
_migrationQueue.addLast(migration);
}
Future _init() async {
Future<void> _init() async {
while (_migrationQueue.isNotEmpty) {
var migration = _migrationQueue.removeFirst();
var path = await absoluteSourcePath(migration.runtimeType);
migrations.putIfAbsent(path.replaceAll('\\', '\\\\'), () => migration);
}
_connected = connection.isOpen;
if (!_connected) {
//await connection.open();
//Connection.open(_endpoint!, settings: _settings);
_connected = true;
throw Exception("PostgreSQL connection is not open");
}
await connection.execute('''
CREATE TABLE IF NOT EXISTS "migrations" (
id serial,
batch integer,
path varchar,
path varchar(500),
PRIMARY KEY(id)
);
''').then((result) {
_log.info('Check and create "migrations" table');
_log.info('Created "migrations" table');
}).catchError((e) {
_log.severe('Failed to create "migrations" table.');
throw e;
});
}
@ -73,8 +76,8 @@ class PostgresMigrationRunner implements MigrationRunner {
var migration = migrations[k]!;
var schema = PostgresSchema();
migration.up(schema);
_log.info('Added "$k" into "migrations" table.');
await schema.run(connection).then((_) {
var result = await schema.run(connection).then((_) {
return connection.runTx((ctx) async {
var result = await ctx.execute(
"INSERT INTO MIGRATIONS (batch, path) VALUES ($batch, '$k')");
@ -85,6 +88,9 @@ class PostgresMigrationRunner implements MigrationRunner {
_log.severe('Failed to insert into "migrations" table.');
return -1;
});
if (result > 0) {
_log.info('Inserted "$k" into "migrations" table.');
}
}
} else {
_log.warning('Nothing to add into "migrations" table.');

View file

@ -4,6 +4,7 @@ import 'package:postgres/postgres.dart';
import 'package:logging/logging.dart';
import 'table.dart';
/// A PostgreSQL database schema generator
class PostgresSchema extends Schema {
final _log = Logger('PostgresSchema');

View file

@ -1,6 +1,6 @@
name: angel3_migration_runner
version: 8.2.0
description: Command-line based database migration runner for Angel3's ORM.
version: 8.2.1
description: The implementation of database migration for Angel3 framework. Designed to work with Angel3 ORM.
homepage: https://angel3-framework.web.app/
repository: https://github.com/dart-backend/angel/tree/master/packages/orm/angel_migration_runner
environment:
@ -16,6 +16,7 @@ dependencies:
logging: ^1.2.0
dev_dependencies:
lints: ^4.0.0
test: ^1.25.0
# dependency_overrides:
# angel3_orm:
# path: ../angel_orm

View file

@ -0,0 +1,51 @@
import 'dart:io';
import 'package:angel3_migration_runner/angel3_migration_runner.dart';
import 'package:angel3_migration_runner/mariadb.dart';
import 'package:mysql1/mysql1.dart';
import 'package:test/test.dart';
import 'models/mysql_todo.dart';
void main() async {
late MySqlConnection conn;
late MigrationRunner runner;
setUp(() async {
print("Setup...");
var host = Platform.environment['MYSQL_HOST'] ?? 'localhost';
var database = Platform.environment['MYSQL_DB'] ?? 'orm_test';
var username = Platform.environment['MYSQL_USERNAME'] ?? 'test';
var password = Platform.environment['MYSQL_PASSWORD'] ?? 'test123';
var settings = ConnectionSettings(
host: host,
port: 3306,
db: database,
user: username,
password: password);
conn = await MySqlConnection.connect(settings);
runner = MariaDbMigrationRunner(
conn,
migrations: [
UserMigration(),
TodoMigration(),
ItemMigration(),
],
);
});
group('MariaDB', () {
test('migrate tables', () async {
print("Test migration up");
runner.up();
});
});
tearDown(() async {
print("Teardown...");
await conn.close();
});
}

View file

@ -0,0 +1,53 @@
import 'package:angel3_migration/angel3_migration.dart';
import 'package:angel3_orm/angel3_orm.dart';
class UserMigration implements Migration {
@override
void up(Schema schema) {
schema.create('users', (table) {
table
..integer('id').primaryKey()
..varChar('username', length: 32).unique()
..varChar('password')
..boolean('account_confirmed').defaultsTo(false);
});
}
@override
void down(Schema schema) {
schema.drop('users');
}
}
class TodoMigration implements Migration {
@override
void up(Schema schema) {
schema.create('todos', (table) {
table
..integer('id').primaryKey()
..integer('user_id').references('users', 'id').onDeleteCascade()
..varChar('text')
..boolean('completed').defaultsTo(false);
});
}
@override
void down(Schema schema) {
schema.drop('todos');
}
}
class ItemMigration extends Migration {
@override
void up(Schema schema) {
schema.create('items', (table) {
table
..integer('id').primaryKey()
..varChar('name', length: 64)
..timeStamp('created_at').defaultsTo(currentTimestamp);
});
}
@override
void down(Schema schema) => schema.drop('items');
}

View file

@ -0,0 +1,53 @@
import 'package:angel3_migration/angel3_migration.dart';
import 'package:angel3_orm/angel3_orm.dart';
class UserMigration implements Migration {
@override
void up(Schema schema) {
schema.create('users', (table) {
table
..serial('id').primaryKey()
..varChar('username', length: 32).unique()
..varChar('password')
..boolean('account_confirmed').defaultsTo(false);
});
}
@override
void down(Schema schema) {
schema.drop('users');
}
}
class TodoMigration implements Migration {
@override
void up(Schema schema) {
schema.create('todos', (table) {
table
..serial('id').primaryKey()
..integer('user_id').references('users', 'id').onDeleteCascade()
..varChar('text')
..boolean('completed').defaultsTo(false);
});
}
@override
void down(Schema schema) {
schema.drop('todos');
}
}
class ItemMigration extends Migration {
@override
void up(Schema schema) {
schema.create('items', (table) {
table
..serial('id').primaryKey()
..varChar('name', length: 64)
..timeStamp('created_at').defaultsTo(currentTimestamp);
});
}
@override
void down(Schema schema) => schema.drop('items');
}

View file

@ -0,0 +1,63 @@
import 'dart:io';
import 'package:angel3_migration_runner/angel3_migration_runner.dart';
import 'package:angel3_migration_runner/mysql.dart';
import 'package:mysql_client/mysql_client.dart';
import 'package:test/test.dart';
import 'models/mysql_todo.dart';
void main() async {
late MySQLConnection conn;
late MigrationRunner runner;
setUp(() async {
print("Setup...");
var host = Platform.environment['MYSQL_HOST'] ?? 'localhost';
var database = Platform.environment['MYSQL_DB'] ?? 'orm_test';
var username = Platform.environment['MYSQL_USERNAME'] ?? 'test';
var password = Platform.environment['MYSQL_PASSWORD'] ?? 'test123';
//var secure = !('false' == Platform.environment['MYSQL_SECURE']);
print("$host $database $username $password");
conn = await MySQLConnection.createConnection(
databaseName: database,
port: 3306,
host: host,
userName: username,
password: password,
secure: true);
await conn.connect();
runner = MySqlMigrationRunner(
conn,
migrations: [
UserMigration(),
TodoMigration(),
ItemMigration(),
],
);
});
group('Mysql migrate tables', () {
test('up', () async {
print("Test migration up");
await runner.up();
});
test('reset', () async {
print("Test migration reset");
await runner.reset();
});
});
tearDown(() async {
print("Teardown...");
if (conn.connected) {
await conn.close();
}
});
}

View file

@ -0,0 +1,59 @@
import 'dart:io';
import 'package:angel3_migration_runner/angel3_migration_runner.dart';
import 'package:angel3_migration_runner/postgres.dart';
import 'package:postgres/postgres.dart';
import 'package:test/test.dart';
import 'models/pg_todo.dart';
void main() async {
late Connection conn;
late MigrationRunner runner;
setUp(() async {
print("Setup...");
var host = Platform.environment['POSTGRES_HOST'] ?? 'localhost';
var database = Platform.environment['POSTGRES_DB'] ?? 'orm_test';
var username = Platform.environment['POSTGRES_USERNAME'] ?? 'test';
var password = Platform.environment['POSTGRES_PASSWORD'] ?? 'test123';
//print("$host $database $username $password");
conn = await Connection.open(
Endpoint(
host: host,
port: 5432,
database: database,
username: username,
password: password),
settings: ConnectionSettings(sslMode: SslMode.disable));
runner = PostgresMigrationRunner(
conn,
migrations: [
UserMigration(),
TodoMigration(),
ItemMigration(),
],
);
});
group('PostgreSQL migrate tables', () {
test('up', () async {
print("Test migration up");
await runner.up();
});
test('reset', () async {
print("Test migration reset");
await runner.reset();
});
});
tearDown(() async {
print("Teardown...");
await conn.close();
});
}

View file

@ -89,7 +89,9 @@ Future<PostgreSqlPoolExecutor> connectToPostgresPool(
)
],
settings: PoolSettings(
maxConnectionAge: Duration(hours: 1), maxConnectionCount: 5));
maxConnectionAge: Duration(hours: 1),
maxConnectionCount: 5,
sslMode: SslMode.disable));
// Run sql to create the tables in a transaction
await dbPool.runTx((conn) async {