From 2b984d45b28e165efab58b78246e6c5c98e6f23c Mon Sep 17 00:00:00 2001 From: Doug Lauder Date: Fri, 14 Oct 2022 18:39:25 -0400 Subject: [PATCH] Fix collation and charset after migrations (every run) (#4002) * fix collation and charset after migrations (every run) --- mattermost-plugin/server/manifest.go | 3 +- .../store/sqlstore/data_migrations.go | 124 ++++++++++++++++++ .../store/sqlstore/data_migrations_test.go | 37 ++++++ .../services/store/sqlstore/helpers_test.go | 4 + server/services/store/sqlstore/migrate.go | 13 +- 5 files changed, 177 insertions(+), 4 deletions(-) diff --git a/mattermost-plugin/server/manifest.go b/mattermost-plugin/server/manifest.go index 481a57c9f..24a40fda5 100644 --- a/mattermost-plugin/server/manifest.go +++ b/mattermost-plugin/server/manifest.go @@ -45,8 +45,7 @@ const manifestStr = ` "type": "bool", "help_text": "This allows board editors to share boards that can be accessed by anyone with the link.", "placeholder": "", - "default": false, - "hosting": "" + "default": false } ] } diff --git a/server/services/store/sqlstore/data_migrations.go b/server/services/store/sqlstore/data_migrations.go index 558cddfeb..aa0787da3 100644 --- a/server/services/store/sqlstore/data_migrations.go +++ b/server/services/store/sqlstore/data_migrations.go @@ -3,9 +3,11 @@ package sqlstore import ( "context" "fmt" + "os" "strconv" sq "github.com/Masterminds/squirrel" + "github.com/wiggin77/merror" "github.com/mattermost/focalboard/server/model" "github.com/mattermost/focalboard/server/utils" @@ -645,3 +647,125 @@ func (s *SQLStore) getDeletedMembershipBoards(tx sq.BaseRunner) ([]*model.Board, return boards, err } + +func (s *SQLStore) RunFixCollationsAndCharsetsMigration() error { + // This is for MySQL only + if s.dbType != model.MysqlDBType { + return nil + } + + // get collation and charSet setting that Channels is using. + // when unit testing, no channels tables exist so just set to a default. + var collation string + var charSet string + var err error + if os.Getenv("FOCALBOARD_UNIT_TESTING") == "1" { + collation = "utf8mb4_general_ci" + charSet = "utf8mb4" + } else { + collation, charSet, err = s.getCollationAndCharset() + if err != nil { + return err + } + } + + // get all FocalBoard tables + tableNames, err := s.getFocalBoardTableNames() + if err != nil { + return err + } + + merr := merror.New() + + // alter each table; this is idempotent + for _, name := range tableNames { + sql := fmt.Sprintf("ALTER TABLE %s CONVERT TO CHARACTER SET '%s' COLLATE '%s'", name, charSet, collation) + result, err := s.db.Exec(sql) + if err != nil { + merr.Append(err) + continue + } + num, err := result.RowsAffected() + if err != nil { + merr.Append(err) + } + if num > 0 { + s.logger.Debug("table collation and/or charSet fixed", + mlog.String("table_name", name), + ) + } + } + return merr.ErrorOrNil() +} + +func (s *SQLStore) getFocalBoardTableNames() ([]string, error) { + if s.dbType != model.MysqlDBType { + return nil, newErrInvalidDBType("getFocalBoardTableNames requires MySQL") + } + + query := s.getQueryBuilder(s.db). + Select("table_name"). + From("information_schema.tables"). + Where(sq.Like{"table_name": s.tablePrefix + "%"}). + Where("table_schema=(SELECT DATABASE())") + + rows, err := query.Query() + if err != nil { + return nil, fmt.Errorf("error fetching FocalBoard table names: %w", err) + } + defer rows.Close() + + names := make([]string, 0) + + for rows.Next() { + var tableName string + + err := rows.Scan(&tableName) + if err != nil { + return nil, fmt.Errorf("cannot scan result while fetching table names: %w", err) + } + + names = append(names, tableName) + } + + return names, nil +} + +func (s *SQLStore) getCollationAndCharset() (string, string, error) { + if s.dbType != model.MysqlDBType { + return "", "", newErrInvalidDBType("getCollationAndCharset requires MySQL") + } + + query := s.getQueryBuilder(s.db). + Select("table_collation"). + From("information_schema.tables"). + Where(sq.Eq{"table_name": "Channels"}). + Where("table_schema=(SELECT DATABASE())") + + row := query.QueryRow() + + var collation string + err := row.Scan(&collation) + if err != nil { + return "", "", fmt.Errorf("error fetching collation: %w", err) + } + + query = s.getQueryBuilder(s.db). + Select("CHARACTER_SET_NAME"). + From("information_schema.columns"). + Where(sq.Eq{ + "table_name": "Channels", + "COLUMN_NAME": "Name", + }). + Where("table_schema=(SELECT DATABASE())") + + row = query.QueryRow() + + var charSet string + err = row.Scan(&charSet) + if err != nil { + return "", "", fmt.Errorf("error fetching charSet: %w", err) + } + + return collation, charSet, nil +} diff --git a/server/services/store/sqlstore/data_migrations_test.go b/server/services/store/sqlstore/data_migrations_test.go index 1d157e31a..778fc0cd8 100644 --- a/server/services/store/sqlstore/data_migrations_test.go +++ b/server/services/store/sqlstore/data_migrations_test.go @@ -6,6 +6,7 @@ import ( "github.com/mattermost/focalboard/server/model" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -234,3 +235,39 @@ func TestRunUniqueIDsMigration(t *testing.T) { require.NotEqual(t, block4.ID, newBlock4.BoardID) require.NotEqual(t, block5.ID, newBlock5.ParentID) } + +func TestCheckForMismatchedCollation(t *testing.T) { + store, tearDown := SetupTests(t) + sqlStore := store.(*SQLStore) + defer tearDown() + + if sqlStore.dbType != model.MysqlDBType { + return + } + + // make sure all collations are consistent. + tableNames, err := sqlStore.getFocalBoardTableNames() + require.NoError(t, err) + + sqlCollation := "SELECT table_collation FROM information_schema.tables WHERE table_name=? and table_schema=(SELECT DATABASE())" + stmtCollation, err := sqlStore.db.Prepare(sqlCollation) + require.NoError(t, err) + defer stmtCollation.Close() + + var collation string + + // make sure the correct charset is applied to each table. + for i, name := range tableNames { + row := stmtCollation.QueryRow(name) + + var actualCollation string + err = row.Scan(&actualCollation) + require.NoError(t, err) + + if collation == "" { + collation = actualCollation + } + + assert.Equalf(t, collation, actualCollation, "for table_name='%s', index=%d", name, i) + } +} diff --git a/server/services/store/sqlstore/helpers_test.go b/server/services/store/sqlstore/helpers_test.go index d05eba944..d3bf2d496 100644 --- a/server/services/store/sqlstore/helpers_test.go +++ b/server/services/store/sqlstore/helpers_test.go @@ -12,6 +12,9 @@ import ( ) func SetupTests(t *testing.T) (store.Store, func()) { + origUnitTesting := os.Getenv("FOCALBOARD_UNIT_TESTING") + os.Setenv("FOCALBOARD_UNIT_TESTING", "1") + dbType, connectionString, err := PrepareNewTestDatabase() require.NoError(t, err) @@ -40,6 +43,7 @@ func SetupTests(t *testing.T) (store.Store, func()) { if err = os.Remove(connectionString); err == nil { logger.Debug("Removed test database", mlog.String("file", connectionString)) } + os.Setenv("FOCALBOARD_UNIT_TESTING", origUnitTesting) } return store, tearDown diff --git a/server/services/store/sqlstore/migrate.go b/server/services/store/sqlstore/migrate.go index 5105e9fca..bf81d14ab 100644 --- a/server/services/store/sqlstore/migrate.go +++ b/server/services/store/sqlstore/migrate.go @@ -242,9 +242,18 @@ func (s *SQLStore) runMigrationSequence(engine *morph.Morph, driver drivers.Driv } s.logger.Debug("== Applying all remaining migrations ====================", - mlog.Int("current_version", len(appliedMigrations))) + mlog.Int("current_version", len(appliedMigrations)), + ) - return engine.ApplyAll() + if err := engine.ApplyAll(); err != nil { + return err + } + + // always run the collations & charset fix-ups + if mErr := s.RunFixCollationsAndCharsetsMigration(); mErr != nil { + return fmt.Errorf("error running fix collations and charsets migration: %w", mErr) + } + return nil } func (s *SQLStore) ensureMigrationsAppliedUpToVersion(engine *morph.Morph, driver drivers.Driver, version int) error {