diff --git a/import/asana/importAsana.ts b/import/asana/importAsana.ts index 38c3e3e4b..19d119af7 100644 --- a/import/asana/importAsana.ts +++ b/import/asana/importAsana.ts @@ -3,7 +3,7 @@ import * as fs from 'fs' import minimist from 'minimist' import {exit} from 'process' -import {IArchive} from '../../webapp/src/blocks/archive' +import {ArchiveUtils} from '../../webapp/src/blocks/archive' import {IBlock} from '../../webapp/src/blocks/block' import {IPropertyOption, IPropertyTemplate, MutableBoard} from '../../webapp/src/blocks/board' import {MutableBoardView} from '../../webapp/src/blocks/boardView' @@ -49,10 +49,11 @@ function main() { const input = JSON.parse(inputData) as Asana // Convert - const output = convert(input) + const blocks = convert(input) // Save output - const outputData = JSON.stringify(output) + // TODO: Stream output + const outputData = ArchiveUtils.buildBlockArchive(blocks) fs.writeFileSync(outputFile, outputData) console.log(`Exported to ${outputFile}`) @@ -87,17 +88,11 @@ function getSections(input: Asana, projectId: string): Workspace[] { return [...sectionMap.values()] } -function convert(input: Asana): IArchive { - const archive: IArchive = { - version: 1, - date: Date.now(), - blocks: [] - } - +function convert(input: Asana): IBlock[] { const projects = getProjects(input) if (projects.length < 1) { console.error('No projects found') - return archive + return [] } // TODO: Handle multiple projects @@ -181,12 +176,10 @@ function convert(input: Asana): IArchive { } }) - archive.blocks = blocks - console.log('') console.log(`Found ${input.data.length} card(s).`) - return archive + return blocks } function showHelp() { diff --git a/import/asana/package.json b/import/asana/package.json index d29ed69f2..a8fc76ba6 100644 --- a/import/asana/package.json +++ b/import/asana/package.json @@ -7,8 +7,8 @@ "scripts": { "lint": "eslint --ext .tsx,.ts . --quiet --cache", "fix": "eslint --ext .tsx,.ts . --quiet --fix --cache", - "test": "ts-node importAsana.ts -i test/asana.json -o test/archive.focalboard", - "debug:test": "node --inspect=5858 -r ts-node/register importAsana.ts -i test/asana.json -o test/archive.focalboard" + "test": "ts-node importAsana.ts -i test/asana.json -o test/asana-import.focalboard", + "debug:test": "node --inspect=5858 -r ts-node/register importAsana.ts -i test/asana.json -o test/asana-import.focalboard" }, "keywords": [], "author": "", diff --git a/import/notion/importNotion.ts b/import/notion/importNotion.ts index fdd1ad743..7b6512964 100644 --- a/import/notion/importNotion.ts +++ b/import/notion/importNotion.ts @@ -3,7 +3,7 @@ import * as fs from 'fs' import minimist from 'minimist' import path from 'path' import {exit} from 'process' -import {IArchive} from '../../webapp/src/blocks/archive' +import {ArchiveUtils} from '../../webapp/src/blocks/archive' import {IBlock} from '../../webapp/src/blocks/block' import {IPropertyTemplate, MutableBoard} from '../../webapp/src/blocks/board' import {MutableBoardView} from '../../webapp/src/blocks/boardView' @@ -70,10 +70,11 @@ async function main() { markdownFolder = path.join(inputFolder, basename) // Convert - const output = convert(input, title) + const blocks = convert(input, title) // Save output - const outputData = JSON.stringify(output) + // TODO: Stream output + const outputData = ArchiveUtils.buildBlockArchive(blocks) fs.writeFileSync(outputFile, outputData) console.log(`Exported to ${outputFile}`) @@ -115,15 +116,9 @@ function getColumns(input: any[]) { return keys.slice(1) } -function convert(input: any[], title: string): IArchive { +function convert(input: any[], title: string): IBlock[] { const blocks: IBlock[] = [] - const archive: IArchive = { - version: 1, - date: Date.now(), - blocks - } - // Board const board = new MutableBoard() console.log(`Board: ${title}`) @@ -160,7 +155,7 @@ function convert(input: any[], title: string): IArchive { console.log(keys) if (keys.length < 1) { console.error(`Expected at least one column`) - return archive + return blocks } const titleKey = keys[0] @@ -216,7 +211,7 @@ function convert(input: any[], title: string): IArchive { console.log('') console.log(`Found ${input.length} card(s).`) - return archive + return blocks } function showHelp() { diff --git a/import/notion/package.json b/import/notion/package.json index f5d485dc2..6dfc1a83c 100644 --- a/import/notion/package.json +++ b/import/notion/package.json @@ -7,8 +7,8 @@ "scripts": { "lint": "eslint --ext .tsx,.ts . --quiet --cache", "fix": "eslint --ext .tsx,.ts . --quiet --fix --cache", - "test": "ts-node importNotion.ts -i test/export -o test/archive.focalboard", - "debug:test": "node --inspect=5858 -r ts-node/register importNotion.ts -i test/export -o test/archive.focalboard" + "test": "ts-node importNotion.ts -i test/export -o test/notion-import.focalboard", + "debug:test": "node --inspect=5858 -r ts-node/register importNotion.ts -i test/export -o test/notion-import.focalboard" }, "keywords": [], "author": "", diff --git a/import/trello/importTrello.ts b/import/trello/importTrello.ts index 96d1f3f77..07bcf0077 100644 --- a/import/trello/importTrello.ts +++ b/import/trello/importTrello.ts @@ -3,7 +3,7 @@ import * as fs from 'fs' import minimist from 'minimist' import {exit} from 'process' -import {IArchive} from '../../webapp/src/blocks/archive' +import {ArchiveUtils} from '../../webapp/src/blocks/archive' import {IBlock} from '../../webapp/src/blocks/block' import {IPropertyOption, IPropertyTemplate, MutableBoard} from '../../webapp/src/blocks/board' import {MutableBoardView} from '../../webapp/src/blocks/boardView' @@ -49,16 +49,17 @@ function main() { const input = JSON.parse(inputData) as Trello // Convert - const output = convert(input) + const blocks = convert(input) // Save output - const outputData = JSON.stringify(output) + // TODO: Stream output + const outputData = ArchiveUtils.buildBlockArchive(blocks) fs.writeFileSync(outputFile, outputData) console.log(`Exported to ${outputFile}`) } -function convert(input: Trello): IArchive { +function convert(input: Trello): IBlock[] { const blocks: IBlock[] = [] // Board @@ -136,16 +137,10 @@ function convert(input: Trello): IArchive { } }) - const archive: IArchive = { - version: 1, - date: Date.now(), - blocks - } - console.log('') console.log(`Found ${input.cards.length} card(s).`) - return archive + return blocks } function showHelp() { diff --git a/import/trello/package.json b/import/trello/package.json index 6cc1eba08..27cec842e 100644 --- a/import/trello/package.json +++ b/import/trello/package.json @@ -7,8 +7,8 @@ "scripts": { "lint": "eslint --ext .tsx,.ts . --quiet --cache", "fix": "eslint --ext .tsx,.ts . --quiet --fix --cache", - "test": "ts-node importTrello.ts -i test/trello.json -o test/archive.focalboard", - "debug:test": "node --inspect=5858 -r ts-node/register importTrello.ts -i test/trello.json -o test/archive.focalboard" + "test": "ts-node importTrello.ts -i test/trello.json -o test/trello-import.focalboard", + "debug:test": "node --inspect=5858 -r ts-node/register importTrello.ts -i test/trello.json -o test/trello-import.focalboard" }, "keywords": [], "author": "", diff --git a/webapp/src/archiver.ts b/webapp/src/archiver.ts index c4d9aed28..dd459d401 100644 --- a/webapp/src/archiver.ts +++ b/webapp/src/archiver.ts @@ -1,7 +1,8 @@ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. -import {IArchive} from './blocks/archive' -import {IMutableBlock} from './blocks/block' +import {ArchiveUtils, IArchiveHeader, IArchiveLine, IBlockArchiveLine} from './blocks/archive' +import {IBlock} from './blocks/block' +import {LineReader} from './lineReader' import mutator from './mutator' import {Utils} from './utils' import {BoardTree} from './viewModel/boardTree' @@ -9,28 +10,16 @@ import {BoardTree} from './viewModel/boardTree' class Archiver { static async exportBoardTree(boardTree: BoardTree): Promise { const blocks = boardTree.allBlocks - const archive: IArchive = { - version: 1, - date: Date.now(), - blocks, - } - - this.exportArchive(archive) + this.exportArchive(blocks) } static async exportFullArchive(): Promise { const blocks = await mutator.exportFullArchive() - const archive: IArchive = { - version: 1, - date: Date.now(), - blocks, - } - - this.exportArchive(archive) + this.exportArchive(blocks) } - private static exportArchive(archive: IArchive): void { - const content = JSON.stringify(archive) + private static exportArchive(blocks: readonly IBlock[]): void { + const content = ArchiveUtils.buildBlockArchive(blocks) const date = new Date() const filename = `archive-${date.getFullYear()}-${date.getMonth() + 1}-${date.getDate()}.focalboard` @@ -48,32 +37,76 @@ class Archiver { // TODO: Remove or reuse link } + private static async importBlocksFromFile(file: File): Promise { + let blockCount = 0 + const maxBlocksPerImport = 1000 + let blocks: IBlock[] = [] + + let isFirstLine = true + return new Promise((resolve) => { + LineReader.readFile(file, async (line, completed) => { + if (completed) { + if (blocks.length > 0) { + await mutator.importFullArchive(blocks) + blockCount += blocks.length + } + Utils.log(`Imported ${blockCount} blocks.`) + resolve() + return + } + + if (isFirstLine) { + isFirstLine = false + const header = JSON.parse(line) as IArchiveHeader + if (header.date && header.version >= 1) { + const date = new Date(header.date) + Utils.log(`Import archive, version: ${header.version}, date/time: ${date.toLocaleString()}.`) + } + } else { + const row = JSON.parse(line) as IArchiveLine + if (!row || !row.type || !row.data) { + Utils.logError('importFullArchive ERROR parsing line') + return + } + switch (row.type) { + case 'block': { + const blockLine = row as IBlockArchiveLine + const block = blockLine.data + if (Archiver.isValidBlock(block)) { + blocks.push(block) + if (blocks.length >= maxBlocksPerImport) { + const blocksToSend = blocks + blocks = [] + await mutator.importFullArchive(blocksToSend) + blockCount += blocksToSend.length + } + } + break + } + } + } + }) + }) + } + + static isValidBlock(block: IBlock): boolean { + if (!block.id || !block.rootId) { + return false + } + + return true + } + static importFullArchive(onComplete?: () => void): void { const input = document.createElement('input') input.type = 'file' input.accept = '.focalboard' input.onchange = async () => { const file = input.files && input.files[0] - const contents = await (new Response(file)).text() - Utils.log(`Import ${contents.length} bytes.`) - const archive: IArchive = JSON.parse(contents) - const {blocks} = archive - const date = new Date(archive.date) - Utils.log(`Import archive, version: ${archive.version}, date/time: ${date.toLocaleString()}, ${blocks.length} block(s).`) + if (file) { + await Archiver.importBlocksFromFile(file) + } - // Basic error checking - let filteredBlocks = blocks.filter((o) => Boolean(o.id)) - - Utils.log(`Import ${filteredBlocks.length} filtered blocks with ids.`) - - this.fixRootIds(filteredBlocks) - - filteredBlocks = filteredBlocks.filter((o) => Boolean(o.rootId)) - - Utils.log(`Import ${filteredBlocks.length} filtered blocks with rootIds.`) - - await mutator.importFullArchive(filteredBlocks) - Utils.log('Import completed') onComplete?.() } @@ -83,42 +116,6 @@ class Archiver { // TODO: Remove or reuse input } - - private static fixRootIds(blocks: IMutableBlock[]) { - const blockMap = new Map(blocks.map((o) => [o.id, o])) - const maxLevels = 5 - for (let i = 0; i < maxLevels; i++) { - let missingRootIds = false - blocks.forEach((o) => { - if (o.parentId) { - const parent = blockMap.get(o.parentId) - if (parent) { - o.rootId = parent.rootId - } else { - Utils.assert(`No parent for ${o.type}: ${o.id} (${o.title})`) - } - if (!o.rootId) { - missingRootIds = true - } - } else { - o.rootId = o.id - } - }) - - if (!missingRootIds) { - Utils.log(`fixRootIds in ${i} levels`) - break - } - } - - // Check and log remaining errors - blocks.forEach((o) => { - if (!o.rootId) { - const parent = blockMap.get(o.parentId) - Utils.logError(`RootId is null: ${o.type} ${o.id}, parentId ${o.parentId}: ${o.title}, parent: ${parent?.type}, parent.rootId: ${parent?.rootId}, parent.title: ${parent?.title}`) - } - }) - } } export {Archiver} diff --git a/webapp/src/blocks/archive.test.ts b/webapp/src/blocks/archive.test.ts new file mode 100644 index 000000000..f4270056e --- /dev/null +++ b/webapp/src/blocks/archive.test.ts @@ -0,0 +1,24 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. +import {TestBlockFactory} from '../test/testBlockFactory' + +import {ArchiveUtils} from './archive' +import {IBlock} from './block' + +test('archive: archive and unarchive', async () => { + const blocks: IBlock[] = [] + + const board = TestBlockFactory.createBoard() + blocks.push(board) + blocks.push(TestBlockFactory.createBoardView(board)) + const card = TestBlockFactory.createCard(board) + blocks.push(card) + blocks.push(TestBlockFactory.createText(card)) + blocks.push(TestBlockFactory.createDivider(card)) + blocks.push(TestBlockFactory.createImage(card)) + + const archive = ArchiveUtils.buildBlockArchive(blocks) + const unarchivedBlocks = ArchiveUtils.parseBlockArchive(archive) + + expect(unarchivedBlocks).toEqual(blocks) +}) diff --git a/webapp/src/blocks/archive.ts b/webapp/src/blocks/archive.ts index 1a1b08a55..8dd420bdc 100644 --- a/webapp/src/blocks/archive.ts +++ b/webapp/src/blocks/archive.ts @@ -2,10 +2,80 @@ // See LICENSE.txt for license information. import {IBlock} from './block' -interface IArchive { +interface IArchiveHeader { version: number date: number - blocks: readonly IBlock[] } -export {IArchive} +interface IArchiveLine { + type: string, + data: any, +} + +// This schema allows the expansion of additional line types in the future +interface IBlockArchiveLine extends IArchiveLine { + type: 'block', + data: IBlock +} + +class ArchiveUtils { + static buildBlockArchive(blocks: readonly IBlock[]): string { + const header: IArchiveHeader = { + version: 1, + date: Date.now(), + } + + const headerString = JSON.stringify(header) + let content = headerString + '\n' + for (const block of blocks) { + const line: IBlockArchiveLine = { + type: 'block', + data: block, + } + const lineString = JSON.stringify(line) + content += lineString + content += '\n' + } + + return content + } + + static parseBlockArchive(contents: string): IBlock[] { + const blocks: IBlock[] = [] + const allLineStrings = contents.split('\n') + if (allLineStrings.length >= 2) { + const headerString = allLineStrings[0] + const header = JSON.parse(headerString) as IArchiveHeader + if (header.date && header.version >= 1) { + const lineStrings = allLineStrings.slice(1) + let lineNum = 2 + for (const lineString of lineStrings) { + if (!lineString) { + // Ignore empty lines, e.g. last line + continue + } + const line = JSON.parse(lineString) as IArchiveLine + if (!line || !line.type || !line.data) { + throw new Error(`ERROR parsing line ${lineNum}`) + } + switch (line.type) { + case 'block': { + const blockLine = line as IBlockArchiveLine + const block = blockLine.data + blocks.push(block) + break + } + } + + lineNum += 1 + } + } else { + throw new Error('ERROR parsing header') + } + } + + return blocks + } +} + +export {IArchiveHeader, IArchiveLine, IBlockArchiveLine, ArchiveUtils} diff --git a/webapp/src/lineReader.ts b/webapp/src/lineReader.ts new file mode 100644 index 000000000..e2cc06bbf --- /dev/null +++ b/webapp/src/lineReader.ts @@ -0,0 +1,77 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +class LineReader { + private static appendBuffer(buffer1: Uint8Array, buffer2: Uint8Array): Uint8Array { + const tmp = new Uint8Array(buffer1.byteLength + buffer2.byteLength) + tmp.set(buffer1, 0) + tmp.set(buffer2, buffer1.byteLength) + + return tmp + } + + private static arrayBufferIndexOf(buffer: Uint8Array, charCode: number): number { + for (let i = 0; i < buffer.byteLength; ++i) { + if (buffer[i] === charCode) { + return i + } + } + + return -1 + } + + static readFile(file: File, callback: (line: string, completed: boolean) => Promise): void { + let buffer = new Uint8Array(0) + + const chunkSize = 1024 * 1000 + let offset = 0 + const fr = new FileReader() + const decoder = new TextDecoder() + + fr.onload = async () => { + const chunk = new Uint8Array(fr.result as ArrayBuffer) + buffer = LineReader.appendBuffer(buffer, chunk) + + const newlineChar = 10 // '\n' + let newlineIndex = LineReader.arrayBufferIndexOf(buffer, newlineChar) + while (newlineIndex >= 0) { + const result = decoder.decode(buffer.slice(0, newlineIndex)) + buffer = buffer.slice(newlineIndex + 1) + + // eslint-disable-next-line no-await-in-loop + await callback(result, false) + newlineIndex = LineReader.arrayBufferIndexOf(buffer, newlineChar) + } + + offset += chunkSize + if (offset >= file.size) { + // Completed + + if (buffer.byteLength > 0) { + // Handle last line + await callback(decoder.decode(buffer), false) + } + + await callback('', true) + return + } + + seek() + } + + fr.onerror = () => { + callback('', true) + } + + seek() + + function seek() { + const slice = file.slice(offset, offset + chunkSize) + + // Need to read as an ArrayBuffer (instead of text) to handle unicode boundaries + fr.readAsArrayBuffer(slice) + } + } +} + +export {LineReader} diff --git a/webapp/src/octoClient.ts b/webapp/src/octoClient.ts index cebfb5daa..e35b66ba8 100644 --- a/webapp/src/octoClient.ts +++ b/webapp/src/octoClient.ts @@ -138,9 +138,10 @@ class OctoClient { async importFullArchive(blocks: readonly IBlock[]): Promise { Utils.log(`importFullArchive: ${blocks.length} blocks(s)`) - blocks.forEach((block) => { - Utils.log(`\t ${block.type}, ${block.id}`) - }) + + // blocks.forEach((block) => { + // Utils.log(`\t ${block.type}, ${block.id}`) + // }) const body = JSON.stringify(blocks) return fetch(this.serverUrl + '/api/v1/blocks/import', { method: 'POST', diff --git a/webapp/src/octoListener.ts b/webapp/src/octoListener.ts index 175a2d799..898eb03b4 100644 --- a/webapp/src/octoListener.ts +++ b/webapp/src/octoListener.ts @@ -108,7 +108,6 @@ class OctoListener { switch (message.action) { case 'UPDATE_BLOCK': - Utils.log(`OctoListener update block: ${message.block?.id}`) this.queueUpdateNotification(message.block!) break default: @@ -207,6 +206,9 @@ class OctoListener { } private flushUpdateNotifications() { + for (const block of this.updatedBlocks) { + Utils.log(`OctoListener flush update block: ${block.id}`) + } this.onChange?.(this.updatedBlocks) this.updatedBlocks = [] }