Fix collation and charset after migrations (every run) (#4002)

* fix collation and charset after migrations (every run)
This commit is contained in:
Doug Lauder 2022-10-14 18:39:25 -04:00 committed by GitHub
parent ed3197ca62
commit 2b984d45b2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 177 additions and 4 deletions

View file

@ -45,8 +45,7 @@ const manifestStr = `
"type": "bool",
"help_text": "This allows board editors to share boards that can be accessed by anyone with the link.",
"placeholder": "",
"default": false,
"hosting": ""
"default": false
}
]
}

View file

@ -3,9 +3,11 @@ package sqlstore
import (
"context"
"fmt"
"os"
"strconv"
sq "github.com/Masterminds/squirrel"
"github.com/wiggin77/merror"
"github.com/mattermost/focalboard/server/model"
"github.com/mattermost/focalboard/server/utils"
@ -645,3 +647,125 @@ func (s *SQLStore) getDeletedMembershipBoards(tx sq.BaseRunner) ([]*model.Board,
return boards, err
}
func (s *SQLStore) RunFixCollationsAndCharsetsMigration() error {
// This is for MySQL only
if s.dbType != model.MysqlDBType {
return nil
}
// get collation and charSet setting that Channels is using.
// when unit testing, no channels tables exist so just set to a default.
var collation string
var charSet string
var err error
if os.Getenv("FOCALBOARD_UNIT_TESTING") == "1" {
collation = "utf8mb4_general_ci"
charSet = "utf8mb4"
} else {
collation, charSet, err = s.getCollationAndCharset()
if err != nil {
return err
}
}
// get all FocalBoard tables
tableNames, err := s.getFocalBoardTableNames()
if err != nil {
return err
}
merr := merror.New()
// alter each table; this is idempotent
for _, name := range tableNames {
sql := fmt.Sprintf("ALTER TABLE %s CONVERT TO CHARACTER SET '%s' COLLATE '%s'", name, charSet, collation)
result, err := s.db.Exec(sql)
if err != nil {
merr.Append(err)
continue
}
num, err := result.RowsAffected()
if err != nil {
merr.Append(err)
}
if num > 0 {
s.logger.Debug("table collation and/or charSet fixed",
mlog.String("table_name", name),
)
}
}
return merr.ErrorOrNil()
}
func (s *SQLStore) getFocalBoardTableNames() ([]string, error) {
if s.dbType != model.MysqlDBType {
return nil, newErrInvalidDBType("getFocalBoardTableNames requires MySQL")
}
query := s.getQueryBuilder(s.db).
Select("table_name").
From("information_schema.tables").
Where(sq.Like{"table_name": s.tablePrefix + "%"}).
Where("table_schema=(SELECT DATABASE())")
rows, err := query.Query()
if err != nil {
return nil, fmt.Errorf("error fetching FocalBoard table names: %w", err)
}
defer rows.Close()
names := make([]string, 0)
for rows.Next() {
var tableName string
err := rows.Scan(&tableName)
if err != nil {
return nil, fmt.Errorf("cannot scan result while fetching table names: %w", err)
}
names = append(names, tableName)
}
return names, nil
}
func (s *SQLStore) getCollationAndCharset() (string, string, error) {
if s.dbType != model.MysqlDBType {
return "", "", newErrInvalidDBType("getCollationAndCharset requires MySQL")
}
query := s.getQueryBuilder(s.db).
Select("table_collation").
From("information_schema.tables").
Where(sq.Eq{"table_name": "Channels"}).
Where("table_schema=(SELECT DATABASE())")
row := query.QueryRow()
var collation string
err := row.Scan(&collation)
if err != nil {
return "", "", fmt.Errorf("error fetching collation: %w", err)
}
query = s.getQueryBuilder(s.db).
Select("CHARACTER_SET_NAME").
From("information_schema.columns").
Where(sq.Eq{
"table_name": "Channels",
"COLUMN_NAME": "Name",
}).
Where("table_schema=(SELECT DATABASE())")
row = query.QueryRow()
var charSet string
err = row.Scan(&charSet)
if err != nil {
return "", "", fmt.Errorf("error fetching charSet: %w", err)
}
return collation, charSet, nil
}

View file

@ -6,6 +6,7 @@ import (
"github.com/mattermost/focalboard/server/model"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -234,3 +235,39 @@ func TestRunUniqueIDsMigration(t *testing.T) {
require.NotEqual(t, block4.ID, newBlock4.BoardID)
require.NotEqual(t, block5.ID, newBlock5.ParentID)
}
func TestCheckForMismatchedCollation(t *testing.T) {
store, tearDown := SetupTests(t)
sqlStore := store.(*SQLStore)
defer tearDown()
if sqlStore.dbType != model.MysqlDBType {
return
}
// make sure all collations are consistent.
tableNames, err := sqlStore.getFocalBoardTableNames()
require.NoError(t, err)
sqlCollation := "SELECT table_collation FROM information_schema.tables WHERE table_name=? and table_schema=(SELECT DATABASE())"
stmtCollation, err := sqlStore.db.Prepare(sqlCollation)
require.NoError(t, err)
defer stmtCollation.Close()
var collation string
// make sure the correct charset is applied to each table.
for i, name := range tableNames {
row := stmtCollation.QueryRow(name)
var actualCollation string
err = row.Scan(&actualCollation)
require.NoError(t, err)
if collation == "" {
collation = actualCollation
}
assert.Equalf(t, collation, actualCollation, "for table_name='%s', index=%d", name, i)
}
}

View file

@ -12,6 +12,9 @@ import (
)
func SetupTests(t *testing.T) (store.Store, func()) {
origUnitTesting := os.Getenv("FOCALBOARD_UNIT_TESTING")
os.Setenv("FOCALBOARD_UNIT_TESTING", "1")
dbType, connectionString, err := PrepareNewTestDatabase()
require.NoError(t, err)
@ -40,6 +43,7 @@ func SetupTests(t *testing.T) (store.Store, func()) {
if err = os.Remove(connectionString); err == nil {
logger.Debug("Removed test database", mlog.String("file", connectionString))
}
os.Setenv("FOCALBOARD_UNIT_TESTING", origUnitTesting)
}
return store, tearDown

View file

@ -242,9 +242,18 @@ func (s *SQLStore) runMigrationSequence(engine *morph.Morph, driver drivers.Driv
}
s.logger.Debug("== Applying all remaining migrations ====================",
mlog.Int("current_version", len(appliedMigrations)))
mlog.Int("current_version", len(appliedMigrations)),
)
return engine.ApplyAll()
if err := engine.ApplyAll(); err != nil {
return err
}
// always run the collations & charset fix-ups
if mErr := s.RunFixCollationsAndCharsetsMigration(); mErr != nil {
return fmt.Errorf("error running fix collations and charsets migration: %w", mErr)
}
return nil
}
func (s *SQLStore) ensureMigrationsAppliedUpToVersion(engine *morph.Morph, driver drivers.Driver, version int) error {