Fix collation and charset after migrations (every run) (#4002)
* fix collation and charset after migrations (every run)
This commit is contained in:
parent
ed3197ca62
commit
2b984d45b2
5 changed files with 177 additions and 4 deletions
3
mattermost-plugin/server/manifest.go
generated
3
mattermost-plugin/server/manifest.go
generated
|
@ -45,8 +45,7 @@ const manifestStr = `
|
|||
"type": "bool",
|
||||
"help_text": "This allows board editors to share boards that can be accessed by anyone with the link.",
|
||||
"placeholder": "",
|
||||
"default": false,
|
||||
"hosting": ""
|
||||
"default": false
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
@ -3,9 +3,11 @@ package sqlstore
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
sq "github.com/Masterminds/squirrel"
|
||||
"github.com/wiggin77/merror"
|
||||
|
||||
"github.com/mattermost/focalboard/server/model"
|
||||
"github.com/mattermost/focalboard/server/utils"
|
||||
|
@ -645,3 +647,125 @@ func (s *SQLStore) getDeletedMembershipBoards(tx sq.BaseRunner) ([]*model.Board,
|
|||
|
||||
return boards, err
|
||||
}
|
||||
|
||||
func (s *SQLStore) RunFixCollationsAndCharsetsMigration() error {
|
||||
// This is for MySQL only
|
||||
if s.dbType != model.MysqlDBType {
|
||||
return nil
|
||||
}
|
||||
|
||||
// get collation and charSet setting that Channels is using.
|
||||
// when unit testing, no channels tables exist so just set to a default.
|
||||
var collation string
|
||||
var charSet string
|
||||
var err error
|
||||
if os.Getenv("FOCALBOARD_UNIT_TESTING") == "1" {
|
||||
collation = "utf8mb4_general_ci"
|
||||
charSet = "utf8mb4"
|
||||
} else {
|
||||
collation, charSet, err = s.getCollationAndCharset()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// get all FocalBoard tables
|
||||
tableNames, err := s.getFocalBoardTableNames()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
merr := merror.New()
|
||||
|
||||
// alter each table; this is idempotent
|
||||
for _, name := range tableNames {
|
||||
sql := fmt.Sprintf("ALTER TABLE %s CONVERT TO CHARACTER SET '%s' COLLATE '%s'", name, charSet, collation)
|
||||
result, err := s.db.Exec(sql)
|
||||
if err != nil {
|
||||
merr.Append(err)
|
||||
continue
|
||||
}
|
||||
num, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
merr.Append(err)
|
||||
}
|
||||
if num > 0 {
|
||||
s.logger.Debug("table collation and/or charSet fixed",
|
||||
mlog.String("table_name", name),
|
||||
)
|
||||
}
|
||||
}
|
||||
return merr.ErrorOrNil()
|
||||
}
|
||||
|
||||
func (s *SQLStore) getFocalBoardTableNames() ([]string, error) {
|
||||
if s.dbType != model.MysqlDBType {
|
||||
return nil, newErrInvalidDBType("getFocalBoardTableNames requires MySQL")
|
||||
}
|
||||
|
||||
query := s.getQueryBuilder(s.db).
|
||||
Select("table_name").
|
||||
From("information_schema.tables").
|
||||
Where(sq.Like{"table_name": s.tablePrefix + "%"}).
|
||||
Where("table_schema=(SELECT DATABASE())")
|
||||
|
||||
rows, err := query.Query()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error fetching FocalBoard table names: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
names := make([]string, 0)
|
||||
|
||||
for rows.Next() {
|
||||
var tableName string
|
||||
|
||||
err := rows.Scan(&tableName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot scan result while fetching table names: %w", err)
|
||||
}
|
||||
|
||||
names = append(names, tableName)
|
||||
}
|
||||
|
||||
return names, nil
|
||||
}
|
||||
|
||||
func (s *SQLStore) getCollationAndCharset() (string, string, error) {
|
||||
if s.dbType != model.MysqlDBType {
|
||||
return "", "", newErrInvalidDBType("getCollationAndCharset requires MySQL")
|
||||
}
|
||||
|
||||
query := s.getQueryBuilder(s.db).
|
||||
Select("table_collation").
|
||||
From("information_schema.tables").
|
||||
Where(sq.Eq{"table_name": "Channels"}).
|
||||
Where("table_schema=(SELECT DATABASE())")
|
||||
|
||||
row := query.QueryRow()
|
||||
|
||||
var collation string
|
||||
err := row.Scan(&collation)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("error fetching collation: %w", err)
|
||||
}
|
||||
|
||||
query = s.getQueryBuilder(s.db).
|
||||
Select("CHARACTER_SET_NAME").
|
||||
From("information_schema.columns").
|
||||
Where(sq.Eq{
|
||||
"table_name": "Channels",
|
||||
"COLUMN_NAME": "Name",
|
||||
}).
|
||||
Where("table_schema=(SELECT DATABASE())")
|
||||
|
||||
row = query.QueryRow()
|
||||
|
||||
var charSet string
|
||||
err = row.Scan(&charSet)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("error fetching charSet: %w", err)
|
||||
}
|
||||
|
||||
return collation, charSet, nil
|
||||
}
|
||||
|
|
|
@ -6,6 +6,7 @@ import (
|
|||
|
||||
"github.com/mattermost/focalboard/server/model"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
@ -234,3 +235,39 @@ func TestRunUniqueIDsMigration(t *testing.T) {
|
|||
require.NotEqual(t, block4.ID, newBlock4.BoardID)
|
||||
require.NotEqual(t, block5.ID, newBlock5.ParentID)
|
||||
}
|
||||
|
||||
func TestCheckForMismatchedCollation(t *testing.T) {
|
||||
store, tearDown := SetupTests(t)
|
||||
sqlStore := store.(*SQLStore)
|
||||
defer tearDown()
|
||||
|
||||
if sqlStore.dbType != model.MysqlDBType {
|
||||
return
|
||||
}
|
||||
|
||||
// make sure all collations are consistent.
|
||||
tableNames, err := sqlStore.getFocalBoardTableNames()
|
||||
require.NoError(t, err)
|
||||
|
||||
sqlCollation := "SELECT table_collation FROM information_schema.tables WHERE table_name=? and table_schema=(SELECT DATABASE())"
|
||||
stmtCollation, err := sqlStore.db.Prepare(sqlCollation)
|
||||
require.NoError(t, err)
|
||||
defer stmtCollation.Close()
|
||||
|
||||
var collation string
|
||||
|
||||
// make sure the correct charset is applied to each table.
|
||||
for i, name := range tableNames {
|
||||
row := stmtCollation.QueryRow(name)
|
||||
|
||||
var actualCollation string
|
||||
err = row.Scan(&actualCollation)
|
||||
require.NoError(t, err)
|
||||
|
||||
if collation == "" {
|
||||
collation = actualCollation
|
||||
}
|
||||
|
||||
assert.Equalf(t, collation, actualCollation, "for table_name='%s', index=%d", name, i)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -12,6 +12,9 @@ import (
|
|||
)
|
||||
|
||||
func SetupTests(t *testing.T) (store.Store, func()) {
|
||||
origUnitTesting := os.Getenv("FOCALBOARD_UNIT_TESTING")
|
||||
os.Setenv("FOCALBOARD_UNIT_TESTING", "1")
|
||||
|
||||
dbType, connectionString, err := PrepareNewTestDatabase()
|
||||
require.NoError(t, err)
|
||||
|
||||
|
@ -40,6 +43,7 @@ func SetupTests(t *testing.T) (store.Store, func()) {
|
|||
if err = os.Remove(connectionString); err == nil {
|
||||
logger.Debug("Removed test database", mlog.String("file", connectionString))
|
||||
}
|
||||
os.Setenv("FOCALBOARD_UNIT_TESTING", origUnitTesting)
|
||||
}
|
||||
|
||||
return store, tearDown
|
||||
|
|
|
@ -242,9 +242,18 @@ func (s *SQLStore) runMigrationSequence(engine *morph.Morph, driver drivers.Driv
|
|||
}
|
||||
|
||||
s.logger.Debug("== Applying all remaining migrations ====================",
|
||||
mlog.Int("current_version", len(appliedMigrations)))
|
||||
mlog.Int("current_version", len(appliedMigrations)),
|
||||
)
|
||||
|
||||
return engine.ApplyAll()
|
||||
if err := engine.ApplyAll(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// always run the collations & charset fix-ups
|
||||
if mErr := s.RunFixCollationsAndCharsetsMigration(); mErr != nil {
|
||||
return fmt.Errorf("error running fix collations and charsets migration: %w", mErr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *SQLStore) ensureMigrationsAppliedUpToVersion(engine *morph.Morph, driver drivers.Driver, version int) error {
|
||||
|
|
Loading…
Reference in a new issue