418 lines
11 KiB
Go
418 lines
11 KiB
Go
package app
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"path"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"github.com/krolaw/zipstream"
|
|
|
|
"github.com/mattermost/focalboard/server/model"
|
|
"github.com/mattermost/focalboard/server/utils"
|
|
|
|
"github.com/mattermost/mattermost-server/v6/shared/mlog"
|
|
)
|
|
|
|
const (
|
|
archiveVersion = 2
|
|
legacyFileBegin = "{\"version\":1"
|
|
)
|
|
|
|
var (
|
|
errBlockIsNotABoard = errors.New("block is not a board")
|
|
)
|
|
|
|
// ImportArchive imports an archive containing zero or more boards, plus all
|
|
// associated content, including cards, content blocks, views, and images.
|
|
//
|
|
// Archives are ZIP files containing a `version.json` file and zero or more
|
|
// directories, each containing a `board.jsonl` and zero or more image files.
|
|
func (a *App) ImportArchive(r io.Reader, opt model.ImportArchiveOptions) error {
|
|
// peek at the first bytes to see if this is a legacy archive format
|
|
br := bufio.NewReader(r)
|
|
peek, err := br.Peek(len(legacyFileBegin))
|
|
if err == nil && string(peek) == legacyFileBegin {
|
|
a.logger.Debug("importing legacy archive")
|
|
_, errImport := a.ImportBoardJSONL(br, opt)
|
|
|
|
go func() {
|
|
if err := a.UpdateCardLimitTimestamp(); err != nil {
|
|
a.logger.Error(
|
|
"UpdateCardLimitTimestamp failed after importing a legacy file",
|
|
mlog.Err(err),
|
|
)
|
|
}
|
|
}()
|
|
|
|
return errImport
|
|
}
|
|
|
|
a.logger.Debug("importing archive")
|
|
zr := zipstream.NewReader(br)
|
|
|
|
boardMap := make(map[string]string) // maps old board ids to new
|
|
|
|
for {
|
|
hdr, err := zr.Next()
|
|
if err != nil {
|
|
if errors.Is(err, io.EOF) {
|
|
a.logger.Debug("import archive - done", mlog.Int("boards_imported", len(boardMap)))
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
|
|
dir, filename := filepath.Split(hdr.Name)
|
|
dir = path.Clean(dir)
|
|
|
|
switch filename {
|
|
case "version.json":
|
|
ver, errVer := parseVersionFile(zr)
|
|
if errVer != nil {
|
|
return errVer
|
|
}
|
|
if ver != archiveVersion {
|
|
return model.NewErrUnsupportedArchiveVersion(ver, archiveVersion)
|
|
}
|
|
case "board.jsonl":
|
|
boardID, err := a.ImportBoardJSONL(zr, opt)
|
|
if err != nil {
|
|
return fmt.Errorf("cannot import board %s: %w", dir, err)
|
|
}
|
|
boardMap[dir] = boardID
|
|
default:
|
|
// import file/image; dir is the old board id
|
|
boardID, ok := boardMap[dir]
|
|
if !ok {
|
|
a.logger.Warn("skipping orphan image in archive",
|
|
mlog.String("dir", dir),
|
|
mlog.String("filename", filename),
|
|
)
|
|
continue
|
|
}
|
|
// save file with original filename so it matches name in image block.
|
|
filePath := filepath.Join(opt.TeamID, boardID, filename)
|
|
_, err := a.filesBackend.WriteFile(zr, filePath)
|
|
if err != nil {
|
|
return fmt.Errorf("cannot import file %s for board %s: %w", filename, dir, err)
|
|
}
|
|
}
|
|
|
|
a.logger.Trace("import archive file",
|
|
mlog.String("dir", dir),
|
|
mlog.String("filename", filename),
|
|
)
|
|
|
|
go func() {
|
|
if err := a.UpdateCardLimitTimestamp(); err != nil {
|
|
a.logger.Error(
|
|
"UpdateCardLimitTimestamp failed after importing an archive",
|
|
mlog.Err(err),
|
|
)
|
|
}
|
|
}()
|
|
}
|
|
}
|
|
|
|
// ImportBoardJSONL imports a JSONL file containing blocks for one board. The resulting
|
|
// board id is returned.
|
|
func (a *App) ImportBoardJSONL(r io.Reader, opt model.ImportArchiveOptions) (string, error) {
|
|
// TODO: Stream this once `model.GenerateBlockIDs` can take a stream of blocks.
|
|
// We don't want to load the whole file in memory, even though it's a single board.
|
|
boardsAndBlocks := &model.BoardsAndBlocks{
|
|
Blocks: make([]*model.Block, 0, 10),
|
|
Boards: make([]*model.Board, 0, 10),
|
|
}
|
|
lineReader := bufio.NewReader(r)
|
|
|
|
userID := opt.ModifiedBy
|
|
if userID == model.SingleUser {
|
|
userID = ""
|
|
}
|
|
now := utils.GetMillis()
|
|
var boardID string
|
|
|
|
lineNum := 1
|
|
firstLine := true
|
|
for {
|
|
line, errRead := readLine(lineReader)
|
|
if len(line) != 0 {
|
|
var skip bool
|
|
if firstLine {
|
|
// first line might be a header tag (old archive format)
|
|
if strings.HasPrefix(string(line), legacyFileBegin) {
|
|
skip = true
|
|
}
|
|
}
|
|
|
|
if !skip {
|
|
var archiveLine model.ArchiveLine
|
|
if err := json.Unmarshal(line, &archiveLine); err != nil {
|
|
return "", fmt.Errorf("error parsing archive line %d: %w", lineNum, err)
|
|
}
|
|
|
|
// first line must be a board
|
|
if firstLine && archiveLine.Type == "block" {
|
|
archiveLine.Type = "board_block"
|
|
}
|
|
|
|
switch archiveLine.Type {
|
|
case "board":
|
|
var board model.Board
|
|
if err2 := json.Unmarshal(archiveLine.Data, &board); err2 != nil {
|
|
return "", fmt.Errorf("invalid board in archive line %d: %w", lineNum, err2)
|
|
}
|
|
board.ModifiedBy = userID
|
|
board.UpdateAt = now
|
|
board.TeamID = opt.TeamID
|
|
boardsAndBlocks.Boards = append(boardsAndBlocks.Boards, &board)
|
|
boardID = board.ID
|
|
case "board_block":
|
|
// legacy archives encoded boards as blocks; we need to convert them to real boards.
|
|
var block *model.Block
|
|
if err2 := json.Unmarshal(archiveLine.Data, &block); err2 != nil {
|
|
return "", fmt.Errorf("invalid board block in archive line %d: %w", lineNum, err2)
|
|
}
|
|
block.ModifiedBy = userID
|
|
block.UpdateAt = now
|
|
board, err := a.blockToBoard(block, opt)
|
|
if err != nil {
|
|
return "", fmt.Errorf("cannot convert archive line %d to block: %w", lineNum, err)
|
|
}
|
|
boardsAndBlocks.Boards = append(boardsAndBlocks.Boards, board)
|
|
boardID = board.ID
|
|
case "block":
|
|
var block *model.Block
|
|
if err2 := json.Unmarshal(archiveLine.Data, &block); err2 != nil {
|
|
return "", fmt.Errorf("invalid block in archive line %d: %w", lineNum, err2)
|
|
}
|
|
block.ModifiedBy = userID
|
|
block.UpdateAt = now
|
|
block.BoardID = boardID
|
|
boardsAndBlocks.Blocks = append(boardsAndBlocks.Blocks, block)
|
|
default:
|
|
return "", model.NewErrUnsupportedArchiveLineType(lineNum, archiveLine.Type)
|
|
}
|
|
firstLine = false
|
|
}
|
|
}
|
|
|
|
if errRead != nil {
|
|
if errors.Is(errRead, io.EOF) {
|
|
break
|
|
}
|
|
return "", fmt.Errorf("error reading archive line %d: %w", lineNum, errRead)
|
|
}
|
|
lineNum++
|
|
}
|
|
|
|
a.fixBoardsandBlocks(boardsAndBlocks, opt)
|
|
|
|
var err error
|
|
boardsAndBlocks, err = model.GenerateBoardsAndBlocksIDs(boardsAndBlocks, a.logger)
|
|
if err != nil {
|
|
return "", fmt.Errorf("error generating archive block IDs: %w", err)
|
|
}
|
|
|
|
boardsAndBlocks, err = a.CreateBoardsAndBlocks(boardsAndBlocks, opt.ModifiedBy, false)
|
|
if err != nil {
|
|
return "", fmt.Errorf("error inserting archive blocks: %w", err)
|
|
}
|
|
|
|
// add user to all the new boards (if not the fake system user).
|
|
if opt.ModifiedBy != model.SystemUserID {
|
|
for _, board := range boardsAndBlocks.Boards {
|
|
boardMember := &model.BoardMember{
|
|
BoardID: board.ID,
|
|
UserID: opt.ModifiedBy,
|
|
SchemeAdmin: true,
|
|
}
|
|
if _, err := a.AddMemberToBoard(boardMember); err != nil {
|
|
return "", fmt.Errorf("cannot add member to board: %w", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// find new board id
|
|
for _, board := range boardsAndBlocks.Boards {
|
|
return board.ID, nil
|
|
}
|
|
return "", fmt.Errorf("missing board in archive: %w", model.ErrInvalidBoardBlock)
|
|
}
|
|
|
|
// fixBoardsandBlocks allows the caller of `ImportArchive` to modify or filters boards and blocks being
|
|
// imported via callbacks.
|
|
func (a *App) fixBoardsandBlocks(boardsAndBlocks *model.BoardsAndBlocks, opt model.ImportArchiveOptions) {
|
|
if opt.BlockModifier == nil && opt.BoardModifier == nil {
|
|
return
|
|
}
|
|
|
|
modInfoCache := make(map[string]interface{})
|
|
modBoards := make([]*model.Board, 0, len(boardsAndBlocks.Boards))
|
|
modBlocks := make([]*model.Block, 0, len(boardsAndBlocks.Blocks))
|
|
|
|
for _, board := range boardsAndBlocks.Boards {
|
|
b := *board
|
|
if opt.BoardModifier != nil && !opt.BoardModifier(&b, modInfoCache) {
|
|
a.logger.Debug("skipping insert board per board modifier",
|
|
mlog.String("boardID", board.ID),
|
|
)
|
|
continue
|
|
}
|
|
modBoards = append(modBoards, &b)
|
|
}
|
|
|
|
for _, block := range boardsAndBlocks.Blocks {
|
|
b := block
|
|
if opt.BlockModifier != nil && !opt.BlockModifier(b, modInfoCache) {
|
|
a.logger.Debug("skipping insert block per block modifier",
|
|
mlog.String("blockID", block.ID),
|
|
)
|
|
continue
|
|
}
|
|
modBlocks = append(modBlocks, b)
|
|
}
|
|
|
|
boardsAndBlocks.Boards = modBoards
|
|
boardsAndBlocks.Blocks = modBlocks
|
|
}
|
|
|
|
// blockToBoard converts a `model.Block` to `model.Board`. Legacy archive formats encode boards as blocks
|
|
// and need conversion during import.
|
|
func (a *App) blockToBoard(block *model.Block, opt model.ImportArchiveOptions) (*model.Board, error) {
|
|
if block.Type != model.TypeBoard {
|
|
return nil, errBlockIsNotABoard
|
|
}
|
|
|
|
board := &model.Board{
|
|
ID: block.ID,
|
|
TeamID: opt.TeamID,
|
|
CreatedBy: block.CreatedBy,
|
|
ModifiedBy: block.ModifiedBy,
|
|
Type: model.BoardTypePrivate,
|
|
Title: block.Title,
|
|
CreateAt: block.CreateAt,
|
|
UpdateAt: block.UpdateAt,
|
|
DeleteAt: block.DeleteAt,
|
|
Properties: make(map[string]interface{}),
|
|
CardProperties: make([]map[string]interface{}, 0),
|
|
}
|
|
|
|
if icon, ok := stringValue(block.Fields, "icon"); ok {
|
|
board.Icon = icon
|
|
}
|
|
if description, ok := stringValue(block.Fields, "description"); ok {
|
|
board.Description = description
|
|
}
|
|
if showDescription, ok := boolValue(block.Fields, "showDescription"); ok {
|
|
board.ShowDescription = showDescription
|
|
}
|
|
if isTemplate, ok := boolValue(block.Fields, "isTemplate"); ok {
|
|
board.IsTemplate = isTemplate
|
|
}
|
|
if templateVer, ok := intValue(block.Fields, "templateVer"); ok {
|
|
board.TemplateVersion = templateVer
|
|
}
|
|
if properties, ok := mapValue(block.Fields, "properties"); ok {
|
|
board.Properties = properties
|
|
}
|
|
if cardProperties, ok := arrayMapsValue(block.Fields, "cardProperties"); ok {
|
|
board.CardProperties = cardProperties
|
|
}
|
|
return board, nil
|
|
}
|
|
|
|
func stringValue(m map[string]interface{}, key string) (string, bool) {
|
|
v, ok := m[key]
|
|
if !ok {
|
|
return "", false
|
|
}
|
|
s, ok := v.(string)
|
|
if !ok {
|
|
return "", false
|
|
}
|
|
return s, true
|
|
}
|
|
|
|
func boolValue(m map[string]interface{}, key string) (bool, bool) {
|
|
v, ok := m[key]
|
|
if !ok {
|
|
return false, false
|
|
}
|
|
b, ok := v.(bool)
|
|
if !ok {
|
|
return false, false
|
|
}
|
|
return b, true
|
|
}
|
|
|
|
func intValue(m map[string]interface{}, key string) (int, bool) {
|
|
v, ok := m[key]
|
|
if !ok {
|
|
return 0, false
|
|
}
|
|
i, ok := v.(int)
|
|
if !ok {
|
|
return 0, false
|
|
}
|
|
return i, true
|
|
}
|
|
|
|
func mapValue(m map[string]interface{}, key string) (map[string]interface{}, bool) {
|
|
v, ok := m[key]
|
|
if !ok {
|
|
return nil, false
|
|
}
|
|
mm, ok := v.(map[string]interface{})
|
|
if !ok {
|
|
return nil, false
|
|
}
|
|
return mm, true
|
|
}
|
|
|
|
func arrayMapsValue(m map[string]interface{}, key string) ([]map[string]interface{}, bool) {
|
|
v, ok := m[key]
|
|
if !ok {
|
|
return nil, false
|
|
}
|
|
ai, ok := v.([]interface{})
|
|
if !ok {
|
|
return nil, false
|
|
}
|
|
|
|
arr := make([]map[string]interface{}, 0, len(ai))
|
|
for _, mi := range ai {
|
|
mm, ok := mi.(map[string]interface{})
|
|
if !ok {
|
|
return nil, false
|
|
}
|
|
arr = append(arr, mm)
|
|
}
|
|
return arr, true
|
|
}
|
|
|
|
func parseVersionFile(r io.Reader) (int, error) {
|
|
file, err := io.ReadAll(r)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("cannot read version.json: %w", err)
|
|
}
|
|
|
|
var header model.ArchiveHeader
|
|
if err := json.Unmarshal(file, &header); err != nil {
|
|
return 0, fmt.Errorf("cannot parse version.json: %w", err)
|
|
}
|
|
return header.Version, nil
|
|
}
|
|
|
|
func readLine(r *bufio.Reader) ([]byte, error) {
|
|
line, err := r.ReadBytes('\n')
|
|
line = bytes.TrimSpace(line)
|
|
return line, err
|
|
}
|