From 1f461adbf88c54d3b14a1775dfd57bae64df8141 Mon Sep 17 00:00:00 2001 From: Chen-I Lim Date: Tue, 2 Mar 2021 15:35:44 -0800 Subject: [PATCH] Cleanup archive import --- webapp/src/archiver.ts | 66 +++++++++------------------------------- webapp/src/lineReader.ts | 20 ++++++------ webapp/src/octoClient.ts | 7 +++-- 3 files changed, 29 insertions(+), 64 deletions(-) diff --git a/webapp/src/archiver.ts b/webapp/src/archiver.ts index 9fc6f416d..bb18c70f7 100644 --- a/webapp/src/archiver.ts +++ b/webapp/src/archiver.ts @@ -1,7 +1,7 @@ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. import {ArchiveUtils, IArchiveHeader, IArchiveLine, IBlockArchiveLine} from './blocks/archive' -import {IBlock, IMutableBlock} from './blocks/block' +import {IBlock} from './blocks/block' import {LineReader} from './lineReader' import mutator from './mutator' import {Utils} from './utils' @@ -65,7 +65,9 @@ class Archiver { case 'block': { const blockLine = row as IBlockArchiveLine const block = blockLine.data - blocks.push(block) + if (Archiver.isValidBlock(block)) { + blocks.push(block) + } break } } @@ -74,6 +76,14 @@ class Archiver { }) } + static isValidBlock(block: IBlock): boolean { + if (!block.id || !block.rootId) { + return false + } + + return true + } + static importFullArchive(onComplete?: () => void): void { const input = document.createElement('input') input.type = 'file' @@ -83,19 +93,9 @@ class Archiver { if (file) { const blocks = await Archiver.readBlocksFromFile(file) - // Basic error checking - let filteredBlocks = blocks.filter((o) => Boolean(o.id)) - - Utils.log(`Import ${filteredBlocks.length} filtered blocks with ids.`) - - this.fixRootIds(filteredBlocks) - - filteredBlocks = filteredBlocks.filter((o) => Boolean(o.rootId)) - - Utils.log(`Import ${filteredBlocks.length} filtered blocks with rootIds.`) - - await mutator.importFullArchive(filteredBlocks) - Utils.log('Import completed') + Utils.log(`Importing ${blocks.length} blocks...`) + await mutator.importFullArchive(blocks) + Utils.log(`Imported ${blocks.length} blocks.`) } onComplete?.() @@ -107,42 +107,6 @@ class Archiver { // TODO: Remove or reuse input } - - private static fixRootIds(blocks: IMutableBlock[]) { - const blockMap = new Map(blocks.map((o) => [o.id, o])) - const maxLevels = 5 - for (let i = 0; i < maxLevels; i++) { - let missingRootIds = false - blocks.forEach((o) => { - if (o.parentId) { - const parent = blockMap.get(o.parentId) - if (parent) { - o.rootId = parent.rootId - } else { - Utils.assert(`No parent for ${o.type}: ${o.id} (${o.title})`) - } - if (!o.rootId) { - missingRootIds = true - } - } else { - o.rootId = o.id - } - }) - - if (!missingRootIds) { - Utils.log(`fixRootIds in ${i} levels`) - break - } - } - - // Check and log remaining errors - blocks.forEach((o) => { - if (!o.rootId) { - const parent = blockMap.get(o.parentId) - Utils.logError(`RootId is null: ${o.type} ${o.id}, parentId ${o.parentId}: ${o.title}, parent: ${parent?.type}, parent.rootId: ${parent?.rootId}, parent.title: ${parent?.title}`) - } - }) - } } export {Archiver} diff --git a/webapp/src/lineReader.ts b/webapp/src/lineReader.ts index c8b395ef3..b3d06badb 100644 --- a/webapp/src/lineReader.ts +++ b/webapp/src/lineReader.ts @@ -2,17 +2,17 @@ // See LICENSE.txt for license information. class LineReader { - private static appendBuffer(buffer1: ArrayBuffer, buffer2: ArrayBuffer) { + private static appendBuffer(buffer1: Uint8Array, buffer2: Uint8Array): Uint8Array { const tmp = new Uint8Array(buffer1.byteLength + buffer2.byteLength) - tmp.set(new Uint8Array(buffer1), 0) - tmp.set(new Uint8Array(buffer2), buffer1.byteLength) - return tmp.buffer + tmp.set(buffer1, 0) + tmp.set(buffer2, buffer1.byteLength) + + return tmp } - private static arrayBufferIndexOf(buffer: ArrayBuffer, charCode: number): number { - const view = new Uint8Array(buffer) - for (let i = 0; i < view.byteLength; ++i) { - if (view[i] === charCode) { + private static arrayBufferIndexOf(buffer: Uint8Array, charCode: number): number { + for (let i = 0; i < buffer.byteLength; ++i) { + if (buffer[i] === charCode) { return i } } @@ -21,7 +21,7 @@ class LineReader { } static readFile(file: File, callback: (line: string, completed: boolean) => void): void { - let buffer = new ArrayBuffer(0) + let buffer = new Uint8Array(0) const chunkSize = 1024 * 1000 let offset = 0 @@ -29,7 +29,7 @@ class LineReader { const decoder = new TextDecoder() fr.onload = () => { - const chunk = fr.result as ArrayBuffer + const chunk = new Uint8Array(fr.result as ArrayBuffer) buffer = LineReader.appendBuffer(buffer, chunk) const newlineChar = 10 // '\n' diff --git a/webapp/src/octoClient.ts b/webapp/src/octoClient.ts index cebfb5daa..e35b66ba8 100644 --- a/webapp/src/octoClient.ts +++ b/webapp/src/octoClient.ts @@ -138,9 +138,10 @@ class OctoClient { async importFullArchive(blocks: readonly IBlock[]): Promise { Utils.log(`importFullArchive: ${blocks.length} blocks(s)`) - blocks.forEach((block) => { - Utils.log(`\t ${block.type}, ${block.id}`) - }) + + // blocks.forEach((block) => { + // Utils.log(`\t ${block.type}, ${block.id}`) + // }) const body = JSON.stringify(blocks) return fetch(this.serverUrl + '/api/v1/blocks/import', { method: 'POST',