From 5d050abd09281935d4596c2c9cc9b1e34da94dcf Mon Sep 17 00:00:00 2001 From: Chen-I Lim Date: Tue, 2 Mar 2021 13:21:55 -0800 Subject: [PATCH 1/7] Use JSONL format for archive --- import/asana/importAsana.ts | 21 +++----- import/asana/package.json | 4 +- import/notion/importNotion.ts | 19 +++---- import/notion/package.json | 4 +- import/trello/importTrello.ts | 17 +++---- import/trello/package.json | 4 +- webapp/src/archiver.ts | 93 +++++++++++++++++++++++------------ webapp/src/blocks/archive.ts | 39 +++++++++++++-- 8 files changed, 123 insertions(+), 78 deletions(-) diff --git a/import/asana/importAsana.ts b/import/asana/importAsana.ts index 38c3e3e4b..19d119af7 100644 --- a/import/asana/importAsana.ts +++ b/import/asana/importAsana.ts @@ -3,7 +3,7 @@ import * as fs from 'fs' import minimist from 'minimist' import {exit} from 'process' -import {IArchive} from '../../webapp/src/blocks/archive' +import {ArchiveUtils} from '../../webapp/src/blocks/archive' import {IBlock} from '../../webapp/src/blocks/block' import {IPropertyOption, IPropertyTemplate, MutableBoard} from '../../webapp/src/blocks/board' import {MutableBoardView} from '../../webapp/src/blocks/boardView' @@ -49,10 +49,11 @@ function main() { const input = JSON.parse(inputData) as Asana // Convert - const output = convert(input) + const blocks = convert(input) // Save output - const outputData = JSON.stringify(output) + // TODO: Stream output + const outputData = ArchiveUtils.buildBlockArchive(blocks) fs.writeFileSync(outputFile, outputData) console.log(`Exported to ${outputFile}`) @@ -87,17 +88,11 @@ function getSections(input: Asana, projectId: string): Workspace[] { return [...sectionMap.values()] } -function convert(input: Asana): IArchive { - const archive: IArchive = { - version: 1, - date: Date.now(), - blocks: [] - } - +function convert(input: Asana): IBlock[] { const projects = getProjects(input) if (projects.length < 1) { console.error('No projects found') - return archive + return [] } // TODO: Handle multiple projects @@ -181,12 +176,10 @@ function convert(input: Asana): IArchive { } }) - archive.blocks = blocks - console.log('') console.log(`Found ${input.data.length} card(s).`) - return archive + return blocks } function showHelp() { diff --git a/import/asana/package.json b/import/asana/package.json index d29ed69f2..a8fc76ba6 100644 --- a/import/asana/package.json +++ b/import/asana/package.json @@ -7,8 +7,8 @@ "scripts": { "lint": "eslint --ext .tsx,.ts . --quiet --cache", "fix": "eslint --ext .tsx,.ts . --quiet --fix --cache", - "test": "ts-node importAsana.ts -i test/asana.json -o test/archive.focalboard", - "debug:test": "node --inspect=5858 -r ts-node/register importAsana.ts -i test/asana.json -o test/archive.focalboard" + "test": "ts-node importAsana.ts -i test/asana.json -o test/asana-import.focalboard", + "debug:test": "node --inspect=5858 -r ts-node/register importAsana.ts -i test/asana.json -o test/asana-import.focalboard" }, "keywords": [], "author": "", diff --git a/import/notion/importNotion.ts b/import/notion/importNotion.ts index fdd1ad743..7b6512964 100644 --- a/import/notion/importNotion.ts +++ b/import/notion/importNotion.ts @@ -3,7 +3,7 @@ import * as fs from 'fs' import minimist from 'minimist' import path from 'path' import {exit} from 'process' -import {IArchive} from '../../webapp/src/blocks/archive' +import {ArchiveUtils} from '../../webapp/src/blocks/archive' import {IBlock} from '../../webapp/src/blocks/block' import {IPropertyTemplate, MutableBoard} from '../../webapp/src/blocks/board' import {MutableBoardView} from '../../webapp/src/blocks/boardView' @@ -70,10 +70,11 @@ async function main() { markdownFolder = path.join(inputFolder, basename) // Convert - const output = convert(input, title) + const blocks = convert(input, title) // Save output - const outputData = JSON.stringify(output) + // TODO: Stream output + const outputData = ArchiveUtils.buildBlockArchive(blocks) fs.writeFileSync(outputFile, outputData) console.log(`Exported to ${outputFile}`) @@ -115,15 +116,9 @@ function getColumns(input: any[]) { return keys.slice(1) } -function convert(input: any[], title: string): IArchive { +function convert(input: any[], title: string): IBlock[] { const blocks: IBlock[] = [] - const archive: IArchive = { - version: 1, - date: Date.now(), - blocks - } - // Board const board = new MutableBoard() console.log(`Board: ${title}`) @@ -160,7 +155,7 @@ function convert(input: any[], title: string): IArchive { console.log(keys) if (keys.length < 1) { console.error(`Expected at least one column`) - return archive + return blocks } const titleKey = keys[0] @@ -216,7 +211,7 @@ function convert(input: any[], title: string): IArchive { console.log('') console.log(`Found ${input.length} card(s).`) - return archive + return blocks } function showHelp() { diff --git a/import/notion/package.json b/import/notion/package.json index f5d485dc2..6dfc1a83c 100644 --- a/import/notion/package.json +++ b/import/notion/package.json @@ -7,8 +7,8 @@ "scripts": { "lint": "eslint --ext .tsx,.ts . --quiet --cache", "fix": "eslint --ext .tsx,.ts . --quiet --fix --cache", - "test": "ts-node importNotion.ts -i test/export -o test/archive.focalboard", - "debug:test": "node --inspect=5858 -r ts-node/register importNotion.ts -i test/export -o test/archive.focalboard" + "test": "ts-node importNotion.ts -i test/export -o test/notion-import.focalboard", + "debug:test": "node --inspect=5858 -r ts-node/register importNotion.ts -i test/export -o test/notion-import.focalboard" }, "keywords": [], "author": "", diff --git a/import/trello/importTrello.ts b/import/trello/importTrello.ts index 96d1f3f77..07bcf0077 100644 --- a/import/trello/importTrello.ts +++ b/import/trello/importTrello.ts @@ -3,7 +3,7 @@ import * as fs from 'fs' import minimist from 'minimist' import {exit} from 'process' -import {IArchive} from '../../webapp/src/blocks/archive' +import {ArchiveUtils} from '../../webapp/src/blocks/archive' import {IBlock} from '../../webapp/src/blocks/block' import {IPropertyOption, IPropertyTemplate, MutableBoard} from '../../webapp/src/blocks/board' import {MutableBoardView} from '../../webapp/src/blocks/boardView' @@ -49,16 +49,17 @@ function main() { const input = JSON.parse(inputData) as Trello // Convert - const output = convert(input) + const blocks = convert(input) // Save output - const outputData = JSON.stringify(output) + // TODO: Stream output + const outputData = ArchiveUtils.buildBlockArchive(blocks) fs.writeFileSync(outputFile, outputData) console.log(`Exported to ${outputFile}`) } -function convert(input: Trello): IArchive { +function convert(input: Trello): IBlock[] { const blocks: IBlock[] = [] // Board @@ -136,16 +137,10 @@ function convert(input: Trello): IArchive { } }) - const archive: IArchive = { - version: 1, - date: Date.now(), - blocks - } - console.log('') console.log(`Found ${input.cards.length} card(s).`) - return archive + return blocks } function showHelp() { diff --git a/import/trello/package.json b/import/trello/package.json index 6cc1eba08..27cec842e 100644 --- a/import/trello/package.json +++ b/import/trello/package.json @@ -7,8 +7,8 @@ "scripts": { "lint": "eslint --ext .tsx,.ts . --quiet --cache", "fix": "eslint --ext .tsx,.ts . --quiet --fix --cache", - "test": "ts-node importTrello.ts -i test/trello.json -o test/archive.focalboard", - "debug:test": "node --inspect=5858 -r ts-node/register importTrello.ts -i test/trello.json -o test/archive.focalboard" + "test": "ts-node importTrello.ts -i test/trello.json -o test/trello-import.focalboard", + "debug:test": "node --inspect=5858 -r ts-node/register importTrello.ts -i test/trello.json -o test/trello-import.focalboard" }, "keywords": [], "author": "", diff --git a/webapp/src/archiver.ts b/webapp/src/archiver.ts index c4d9aed28..1359e5935 100644 --- a/webapp/src/archiver.ts +++ b/webapp/src/archiver.ts @@ -1,7 +1,7 @@ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. -import {IArchive} from './blocks/archive' -import {IMutableBlock} from './blocks/block' +import {ArchiveUtils, IArchiveHeader, IArchiveLine, IBlockArchiveLine} from './blocks/archive' +import {IBlock, IMutableBlock} from './blocks/block' import mutator from './mutator' import {Utils} from './utils' import {BoardTree} from './viewModel/boardTree' @@ -9,28 +9,16 @@ import {BoardTree} from './viewModel/boardTree' class Archiver { static async exportBoardTree(boardTree: BoardTree): Promise { const blocks = boardTree.allBlocks - const archive: IArchive = { - version: 1, - date: Date.now(), - blocks, - } - - this.exportArchive(archive) + this.exportArchive(blocks) } static async exportFullArchive(): Promise { const blocks = await mutator.exportFullArchive() - const archive: IArchive = { - version: 1, - date: Date.now(), - blocks, - } - - this.exportArchive(archive) + this.exportArchive(blocks) } - private static exportArchive(archive: IArchive): void { - const content = JSON.stringify(archive) + private static exportArchive(blocks: readonly IBlock[]): void { + const content = ArchiveUtils.buildBlockArchive(blocks) const date = new Date() const filename = `archive-${date.getFullYear()}-${date.getMonth() + 1}-${date.getDate()}.focalboard` @@ -48,32 +36,73 @@ class Archiver { // TODO: Remove or reuse link } + private static async readBlocksFromFile(file: File): Promise { + // TODO: Read input as a stream, line by line + const contents = await (new Response(file)).text() + Utils.log(`Import ${contents.length} bytes.`) + + const blocks: IBlock[] = [] + const allLineStrings = contents.split('\n') + if (allLineStrings.length >= 2) { + const headerString = allLineStrings[0] + const header = JSON.parse(headerString) as IArchiveHeader + if (header.date && header.version >= 1) { + const date = new Date(header.date) + Utils.log(`Import archive, version: ${header.version}, date/time: ${date.toLocaleString()}, ${blocks.length} block(s).`) + + const lineStrings = allLineStrings.slice(1) + for (const lineString of lineStrings) { + if (!lineString) { + // Ignore empty lines, e.g. last line + continue + } + + const line = JSON.parse(lineString) as IArchiveLine + if (!line || !line.type || !line.data) { + Utils.logError('importFullArchive ERROR parsing line') + continue + } + switch (line.type) { + case 'block': { + const blockLine = line as IBlockArchiveLine + const block = blockLine.data + blocks.push(block) + break + } + } + } + } else { + Utils.logError('importFullArchive ERROR parsing header') + } + } + + return blocks + } + static importFullArchive(onComplete?: () => void): void { const input = document.createElement('input') input.type = 'file' input.accept = '.focalboard' input.onchange = async () => { const file = input.files && input.files[0] - const contents = await (new Response(file)).text() - Utils.log(`Import ${contents.length} bytes.`) - const archive: IArchive = JSON.parse(contents) - const {blocks} = archive - const date = new Date(archive.date) - Utils.log(`Import archive, version: ${archive.version}, date/time: ${date.toLocaleString()}, ${blocks.length} block(s).`) + if (file) { + const blocks = await Archiver.readBlocksFromFile(file) - // Basic error checking - let filteredBlocks = blocks.filter((o) => Boolean(o.id)) + // Basic error checking + let filteredBlocks = blocks.filter((o) => Boolean(o.id)) - Utils.log(`Import ${filteredBlocks.length} filtered blocks with ids.`) + Utils.log(`Import ${filteredBlocks.length} filtered blocks with ids.`) - this.fixRootIds(filteredBlocks) + this.fixRootIds(filteredBlocks) - filteredBlocks = filteredBlocks.filter((o) => Boolean(o.rootId)) + filteredBlocks = filteredBlocks.filter((o) => Boolean(o.rootId)) - Utils.log(`Import ${filteredBlocks.length} filtered blocks with rootIds.`) + Utils.log(`Import ${filteredBlocks.length} filtered blocks with rootIds.`) + + await mutator.importFullArchive(filteredBlocks) + Utils.log('Import completed') + } - await mutator.importFullArchive(filteredBlocks) - Utils.log('Import completed') onComplete?.() } diff --git a/webapp/src/blocks/archive.ts b/webapp/src/blocks/archive.ts index 1a1b08a55..88a91ecee 100644 --- a/webapp/src/blocks/archive.ts +++ b/webapp/src/blocks/archive.ts @@ -2,10 +2,43 @@ // See LICENSE.txt for license information. import {IBlock} from './block' -interface IArchive { +interface IArchiveHeader { version: number date: number - blocks: readonly IBlock[] } -export {IArchive} +interface IArchiveLine { + type: string, + data: any, +} + +// This schema allows the expansion of additional line types in the future +interface IBlockArchiveLine extends IArchiveLine { + type: 'block', + data: IBlock +} + +class ArchiveUtils { + static buildBlockArchive(blocks: readonly IBlock[]): string { + const header: IArchiveHeader = { + version: 1, + date: Date.now(), + } + + const headerString = JSON.stringify(header) + let content = headerString + '\n' + for (const block of blocks) { + const line: IBlockArchiveLine = { + type: 'block', + data: block, + } + const lineString = JSON.stringify(line) + content += lineString + content += '\n' + } + + return content + } +} + +export {IArchiveHeader, IArchiveLine, IBlockArchiveLine, ArchiveUtils} From 8e1c5941bbfdb04fdc185f127fa4bcacf90591b8 Mon Sep 17 00:00:00 2001 From: Chen-I Lim Date: Tue, 2 Mar 2021 15:14:37 -0800 Subject: [PATCH 2/7] LineReader --- webapp/src/archiver.ts | 51 ++++++++++++--------------- webapp/src/lineReader.ts | 75 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 28 deletions(-) create mode 100644 webapp/src/lineReader.ts diff --git a/webapp/src/archiver.ts b/webapp/src/archiver.ts index 1359e5935..9fc6f416d 100644 --- a/webapp/src/archiver.ts +++ b/webapp/src/archiver.ts @@ -2,6 +2,7 @@ // See LICENSE.txt for license information. import {ArchiveUtils, IArchiveHeader, IArchiveLine, IBlockArchiveLine} from './blocks/archive' import {IBlock, IMutableBlock} from './blocks/block' +import {LineReader} from './lineReader' import mutator from './mutator' import {Utils} from './utils' import {BoardTree} from './viewModel/boardTree' @@ -37,46 +38,40 @@ class Archiver { } private static async readBlocksFromFile(file: File): Promise { - // TODO: Read input as a stream, line by line - const contents = await (new Response(file)).text() - Utils.log(`Import ${contents.length} bytes.`) - const blocks: IBlock[] = [] - const allLineStrings = contents.split('\n') - if (allLineStrings.length >= 2) { - const headerString = allLineStrings[0] - const header = JSON.parse(headerString) as IArchiveHeader - if (header.date && header.version >= 1) { - const date = new Date(header.date) - Utils.log(`Import archive, version: ${header.version}, date/time: ${date.toLocaleString()}, ${blocks.length} block(s).`) - const lineStrings = allLineStrings.slice(1) - for (const lineString of lineStrings) { - if (!lineString) { - // Ignore empty lines, e.g. last line - continue + let isFirstLine = true + return new Promise((resolve) => { + LineReader.readFile(file, (line, completed) => { + if (completed) { + resolve(blocks) + return + } + + if (isFirstLine) { + isFirstLine = false + const header = JSON.parse(line) as IArchiveHeader + if (header.date && header.version >= 1) { + const date = new Date(header.date) + Utils.log(`Import archive, version: ${header.version}, date/time: ${date.toLocaleString()}.`) } - - const line = JSON.parse(lineString) as IArchiveLine - if (!line || !line.type || !line.data) { + } else { + const row = JSON.parse(line) as IArchiveLine + if (!row || !row.type || !row.data) { Utils.logError('importFullArchive ERROR parsing line') - continue + return } - switch (line.type) { + switch (row.type) { case 'block': { - const blockLine = line as IBlockArchiveLine + const blockLine = row as IBlockArchiveLine const block = blockLine.data blocks.push(block) break } } } - } else { - Utils.logError('importFullArchive ERROR parsing header') - } - } - - return blocks + }) + }) } static importFullArchive(onComplete?: () => void): void { diff --git a/webapp/src/lineReader.ts b/webapp/src/lineReader.ts new file mode 100644 index 000000000..c8b395ef3 --- /dev/null +++ b/webapp/src/lineReader.ts @@ -0,0 +1,75 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +class LineReader { + private static appendBuffer(buffer1: ArrayBuffer, buffer2: ArrayBuffer) { + const tmp = new Uint8Array(buffer1.byteLength + buffer2.byteLength) + tmp.set(new Uint8Array(buffer1), 0) + tmp.set(new Uint8Array(buffer2), buffer1.byteLength) + return tmp.buffer + } + + private static arrayBufferIndexOf(buffer: ArrayBuffer, charCode: number): number { + const view = new Uint8Array(buffer) + for (let i = 0; i < view.byteLength; ++i) { + if (view[i] === charCode) { + return i + } + } + + return -1 + } + + static readFile(file: File, callback: (line: string, completed: boolean) => void): void { + let buffer = new ArrayBuffer(0) + + const chunkSize = 1024 * 1000 + let offset = 0 + const fr = new FileReader() + const decoder = new TextDecoder() + + fr.onload = () => { + const chunk = fr.result as ArrayBuffer + buffer = LineReader.appendBuffer(buffer, chunk) + + const newlineChar = 10 // '\n' + let newlineIndex = LineReader.arrayBufferIndexOf(buffer, newlineChar) + while (newlineIndex >= 0) { + const result = decoder.decode(buffer.slice(0, newlineIndex)) + buffer = buffer.slice(newlineIndex + 1) + callback(result, false) + newlineIndex = LineReader.arrayBufferIndexOf(buffer, newlineChar) + } + + offset += chunkSize + if (offset >= file.size) { + // Completed + + if (buffer.byteLength > 0) { + // Handle last line + callback(decoder.decode(buffer), false) + } + + callback('', true) + return + } + + seek() + } + + fr.onerror = () => { + callback('', true) + } + + seek() + + function seek() { + const slice = file.slice(offset, offset + chunkSize) + + // Need to read as an ArrayBuffer (instead of text) to handle unicode boundaries + fr.readAsArrayBuffer(slice) + } + } +} + +export {LineReader} From 1f461adbf88c54d3b14a1775dfd57bae64df8141 Mon Sep 17 00:00:00 2001 From: Chen-I Lim Date: Tue, 2 Mar 2021 15:35:44 -0800 Subject: [PATCH 3/7] Cleanup archive import --- webapp/src/archiver.ts | 66 +++++++++------------------------------- webapp/src/lineReader.ts | 20 ++++++------ webapp/src/octoClient.ts | 7 +++-- 3 files changed, 29 insertions(+), 64 deletions(-) diff --git a/webapp/src/archiver.ts b/webapp/src/archiver.ts index 9fc6f416d..bb18c70f7 100644 --- a/webapp/src/archiver.ts +++ b/webapp/src/archiver.ts @@ -1,7 +1,7 @@ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. import {ArchiveUtils, IArchiveHeader, IArchiveLine, IBlockArchiveLine} from './blocks/archive' -import {IBlock, IMutableBlock} from './blocks/block' +import {IBlock} from './blocks/block' import {LineReader} from './lineReader' import mutator from './mutator' import {Utils} from './utils' @@ -65,7 +65,9 @@ class Archiver { case 'block': { const blockLine = row as IBlockArchiveLine const block = blockLine.data - blocks.push(block) + if (Archiver.isValidBlock(block)) { + blocks.push(block) + } break } } @@ -74,6 +76,14 @@ class Archiver { }) } + static isValidBlock(block: IBlock): boolean { + if (!block.id || !block.rootId) { + return false + } + + return true + } + static importFullArchive(onComplete?: () => void): void { const input = document.createElement('input') input.type = 'file' @@ -83,19 +93,9 @@ class Archiver { if (file) { const blocks = await Archiver.readBlocksFromFile(file) - // Basic error checking - let filteredBlocks = blocks.filter((o) => Boolean(o.id)) - - Utils.log(`Import ${filteredBlocks.length} filtered blocks with ids.`) - - this.fixRootIds(filteredBlocks) - - filteredBlocks = filteredBlocks.filter((o) => Boolean(o.rootId)) - - Utils.log(`Import ${filteredBlocks.length} filtered blocks with rootIds.`) - - await mutator.importFullArchive(filteredBlocks) - Utils.log('Import completed') + Utils.log(`Importing ${blocks.length} blocks...`) + await mutator.importFullArchive(blocks) + Utils.log(`Imported ${blocks.length} blocks.`) } onComplete?.() @@ -107,42 +107,6 @@ class Archiver { // TODO: Remove or reuse input } - - private static fixRootIds(blocks: IMutableBlock[]) { - const blockMap = new Map(blocks.map((o) => [o.id, o])) - const maxLevels = 5 - for (let i = 0; i < maxLevels; i++) { - let missingRootIds = false - blocks.forEach((o) => { - if (o.parentId) { - const parent = blockMap.get(o.parentId) - if (parent) { - o.rootId = parent.rootId - } else { - Utils.assert(`No parent for ${o.type}: ${o.id} (${o.title})`) - } - if (!o.rootId) { - missingRootIds = true - } - } else { - o.rootId = o.id - } - }) - - if (!missingRootIds) { - Utils.log(`fixRootIds in ${i} levels`) - break - } - } - - // Check and log remaining errors - blocks.forEach((o) => { - if (!o.rootId) { - const parent = blockMap.get(o.parentId) - Utils.logError(`RootId is null: ${o.type} ${o.id}, parentId ${o.parentId}: ${o.title}, parent: ${parent?.type}, parent.rootId: ${parent?.rootId}, parent.title: ${parent?.title}`) - } - }) - } } export {Archiver} diff --git a/webapp/src/lineReader.ts b/webapp/src/lineReader.ts index c8b395ef3..b3d06badb 100644 --- a/webapp/src/lineReader.ts +++ b/webapp/src/lineReader.ts @@ -2,17 +2,17 @@ // See LICENSE.txt for license information. class LineReader { - private static appendBuffer(buffer1: ArrayBuffer, buffer2: ArrayBuffer) { + private static appendBuffer(buffer1: Uint8Array, buffer2: Uint8Array): Uint8Array { const tmp = new Uint8Array(buffer1.byteLength + buffer2.byteLength) - tmp.set(new Uint8Array(buffer1), 0) - tmp.set(new Uint8Array(buffer2), buffer1.byteLength) - return tmp.buffer + tmp.set(buffer1, 0) + tmp.set(buffer2, buffer1.byteLength) + + return tmp } - private static arrayBufferIndexOf(buffer: ArrayBuffer, charCode: number): number { - const view = new Uint8Array(buffer) - for (let i = 0; i < view.byteLength; ++i) { - if (view[i] === charCode) { + private static arrayBufferIndexOf(buffer: Uint8Array, charCode: number): number { + for (let i = 0; i < buffer.byteLength; ++i) { + if (buffer[i] === charCode) { return i } } @@ -21,7 +21,7 @@ class LineReader { } static readFile(file: File, callback: (line: string, completed: boolean) => void): void { - let buffer = new ArrayBuffer(0) + let buffer = new Uint8Array(0) const chunkSize = 1024 * 1000 let offset = 0 @@ -29,7 +29,7 @@ class LineReader { const decoder = new TextDecoder() fr.onload = () => { - const chunk = fr.result as ArrayBuffer + const chunk = new Uint8Array(fr.result as ArrayBuffer) buffer = LineReader.appendBuffer(buffer, chunk) const newlineChar = 10 // '\n' diff --git a/webapp/src/octoClient.ts b/webapp/src/octoClient.ts index cebfb5daa..e35b66ba8 100644 --- a/webapp/src/octoClient.ts +++ b/webapp/src/octoClient.ts @@ -138,9 +138,10 @@ class OctoClient { async importFullArchive(blocks: readonly IBlock[]): Promise { Utils.log(`importFullArchive: ${blocks.length} blocks(s)`) - blocks.forEach((block) => { - Utils.log(`\t ${block.type}, ${block.id}`) - }) + + // blocks.forEach((block) => { + // Utils.log(`\t ${block.type}, ${block.id}`) + // }) const body = JSON.stringify(blocks) return fetch(this.serverUrl + '/api/v1/blocks/import', { method: 'POST', From ab7d936e9f2f8a1a43518956dedbb1cb073200a0 Mon Sep 17 00:00:00 2001 From: Chen-I Lim Date: Tue, 2 Mar 2021 16:13:26 -0800 Subject: [PATCH 4/7] async LineReader --- webapp/src/archiver.ts | 26 +++++++++++++++++--------- webapp/src/lineReader.ts | 12 +++++++----- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/webapp/src/archiver.ts b/webapp/src/archiver.ts index bb18c70f7..475795497 100644 --- a/webapp/src/archiver.ts +++ b/webapp/src/archiver.ts @@ -37,14 +37,21 @@ class Archiver { // TODO: Remove or reuse link } - private static async readBlocksFromFile(file: File): Promise { + private static async importBlocksFromFile(file: File): Promise { + let blockCount = 0 + const maxBlocksPerImport = 100 const blocks: IBlock[] = [] let isFirstLine = true - return new Promise((resolve) => { - LineReader.readFile(file, (line, completed) => { + return new Promise((resolve) => { + LineReader.readFile(file, async (line, completed) => { if (completed) { - resolve(blocks) + if (blocks.length > 0) { + await mutator.importFullArchive(blocks) + blockCount += blocks.length + } + Utils.log(`Imported ${blockCount} blocks.`) + resolve() return } @@ -67,6 +74,11 @@ class Archiver { const block = blockLine.data if (Archiver.isValidBlock(block)) { blocks.push(block) + if (blocks.length >= maxBlocksPerImport) { + await mutator.importFullArchive(blocks) + blockCount += blocks.length + blocks.length = 0 + } } break } @@ -91,11 +103,7 @@ class Archiver { input.onchange = async () => { const file = input.files && input.files[0] if (file) { - const blocks = await Archiver.readBlocksFromFile(file) - - Utils.log(`Importing ${blocks.length} blocks...`) - await mutator.importFullArchive(blocks) - Utils.log(`Imported ${blocks.length} blocks.`) + await Archiver.importBlocksFromFile(file) } onComplete?.() diff --git a/webapp/src/lineReader.ts b/webapp/src/lineReader.ts index b3d06badb..e2cc06bbf 100644 --- a/webapp/src/lineReader.ts +++ b/webapp/src/lineReader.ts @@ -20,7 +20,7 @@ class LineReader { return -1 } - static readFile(file: File, callback: (line: string, completed: boolean) => void): void { + static readFile(file: File, callback: (line: string, completed: boolean) => Promise): void { let buffer = new Uint8Array(0) const chunkSize = 1024 * 1000 @@ -28,7 +28,7 @@ class LineReader { const fr = new FileReader() const decoder = new TextDecoder() - fr.onload = () => { + fr.onload = async () => { const chunk = new Uint8Array(fr.result as ArrayBuffer) buffer = LineReader.appendBuffer(buffer, chunk) @@ -37,7 +37,9 @@ class LineReader { while (newlineIndex >= 0) { const result = decoder.decode(buffer.slice(0, newlineIndex)) buffer = buffer.slice(newlineIndex + 1) - callback(result, false) + + // eslint-disable-next-line no-await-in-loop + await callback(result, false) newlineIndex = LineReader.arrayBufferIndexOf(buffer, newlineChar) } @@ -47,10 +49,10 @@ class LineReader { if (buffer.byteLength > 0) { // Handle last line - callback(decoder.decode(buffer), false) + await callback(decoder.decode(buffer), false) } - callback('', true) + await callback('', true) return } From 7760cfa9ae987d1499b318c0e90b04bf6fb8dfb4 Mon Sep 17 00:00:00 2001 From: Chen-I Lim Date: Tue, 2 Mar 2021 18:09:13 -0800 Subject: [PATCH 5/7] cleanup import blocks --- webapp/src/archiver.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/webapp/src/archiver.ts b/webapp/src/archiver.ts index 475795497..dd459d401 100644 --- a/webapp/src/archiver.ts +++ b/webapp/src/archiver.ts @@ -39,8 +39,8 @@ class Archiver { private static async importBlocksFromFile(file: File): Promise { let blockCount = 0 - const maxBlocksPerImport = 100 - const blocks: IBlock[] = [] + const maxBlocksPerImport = 1000 + let blocks: IBlock[] = [] let isFirstLine = true return new Promise((resolve) => { @@ -75,9 +75,10 @@ class Archiver { if (Archiver.isValidBlock(block)) { blocks.push(block) if (blocks.length >= maxBlocksPerImport) { - await mutator.importFullArchive(blocks) - blockCount += blocks.length - blocks.length = 0 + const blocksToSend = blocks + blocks = [] + await mutator.importFullArchive(blocksToSend) + blockCount += blocksToSend.length } } break From 19d6dc1ddf7eec9314488906d5b2ec806b08160f Mon Sep 17 00:00:00 2001 From: Chen-I Lim Date: Tue, 2 Mar 2021 18:13:41 -0800 Subject: [PATCH 6/7] Cleanup OctoListener tracing --- webapp/src/octoListener.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/webapp/src/octoListener.ts b/webapp/src/octoListener.ts index 175a2d799..898eb03b4 100644 --- a/webapp/src/octoListener.ts +++ b/webapp/src/octoListener.ts @@ -108,7 +108,6 @@ class OctoListener { switch (message.action) { case 'UPDATE_BLOCK': - Utils.log(`OctoListener update block: ${message.block?.id}`) this.queueUpdateNotification(message.block!) break default: @@ -207,6 +206,9 @@ class OctoListener { } private flushUpdateNotifications() { + for (const block of this.updatedBlocks) { + Utils.log(`OctoListener flush update block: ${block.id}`) + } this.onChange?.(this.updatedBlocks) this.updatedBlocks = [] } From 655635551dadd753a63efd9a855848d662493c5f Mon Sep 17 00:00:00 2001 From: Chen-I Lim Date: Wed, 3 Mar 2021 10:09:09 -0800 Subject: [PATCH 7/7] parseBlockArchive and tests --- webapp/src/blocks/archive.test.ts | 24 ++++++++++++++++++++ webapp/src/blocks/archive.ts | 37 +++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 webapp/src/blocks/archive.test.ts diff --git a/webapp/src/blocks/archive.test.ts b/webapp/src/blocks/archive.test.ts new file mode 100644 index 000000000..f4270056e --- /dev/null +++ b/webapp/src/blocks/archive.test.ts @@ -0,0 +1,24 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. +import {TestBlockFactory} from '../test/testBlockFactory' + +import {ArchiveUtils} from './archive' +import {IBlock} from './block' + +test('archive: archive and unarchive', async () => { + const blocks: IBlock[] = [] + + const board = TestBlockFactory.createBoard() + blocks.push(board) + blocks.push(TestBlockFactory.createBoardView(board)) + const card = TestBlockFactory.createCard(board) + blocks.push(card) + blocks.push(TestBlockFactory.createText(card)) + blocks.push(TestBlockFactory.createDivider(card)) + blocks.push(TestBlockFactory.createImage(card)) + + const archive = ArchiveUtils.buildBlockArchive(blocks) + const unarchivedBlocks = ArchiveUtils.parseBlockArchive(archive) + + expect(unarchivedBlocks).toEqual(blocks) +}) diff --git a/webapp/src/blocks/archive.ts b/webapp/src/blocks/archive.ts index 88a91ecee..8dd420bdc 100644 --- a/webapp/src/blocks/archive.ts +++ b/webapp/src/blocks/archive.ts @@ -39,6 +39,43 @@ class ArchiveUtils { return content } + + static parseBlockArchive(contents: string): IBlock[] { + const blocks: IBlock[] = [] + const allLineStrings = contents.split('\n') + if (allLineStrings.length >= 2) { + const headerString = allLineStrings[0] + const header = JSON.parse(headerString) as IArchiveHeader + if (header.date && header.version >= 1) { + const lineStrings = allLineStrings.slice(1) + let lineNum = 2 + for (const lineString of lineStrings) { + if (!lineString) { + // Ignore empty lines, e.g. last line + continue + } + const line = JSON.parse(lineString) as IArchiveLine + if (!line || !line.type || !line.data) { + throw new Error(`ERROR parsing line ${lineNum}`) + } + switch (line.type) { + case 'block': { + const blockLine = line as IBlockArchiveLine + const block = blockLine.data + blocks.push(block) + break + } + } + + lineNum += 1 + } + } else { + throw new Error('ERROR parsing header') + } + } + + return blocks + } } export {IArchiveHeader, IArchiveLine, IBlockArchiveLine, ArchiveUtils}