luisgulo/focalboard
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #54 from mattermost/archive-jsonl

Browse source 
Archive in JSONL format
This commit is contained in:
Chen-I Lim 2021-03-04 09:22:55 -08:00 committed by GitHub
commit e5824b0696
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 277 additions and 123 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
import/asana/importAsana.ts
View file

@ -3,7 +3,7 @@
import * as fs from 'fs'
import minimist from 'minimist'
import {exit} from 'process'
import {IArchive} from '../../webapp/src/blocks/archive'
import {ArchiveUtils} from '../../webapp/src/blocks/archive'
import {IBlock} from '../../webapp/src/blocks/block'
import {IPropertyOption, IPropertyTemplate, MutableBoard} from '../../webapp/src/blocks/board'
import {MutableBoardView} from '../../webapp/src/blocks/boardView'
@ -49,10 +49,11 @@ function main() {
const input = JSON.parse(inputData) as Asana
// Convert
const output = convert(input)
const blocks = convert(input)
// Save output
const outputData = JSON.stringify(output)
// TODO: Stream output
const outputData = ArchiveUtils.buildBlockArchive(blocks)
fs.writeFileSync(outputFile, outputData)
console.log(`Exported to ${outputFile}`)
@ -87,17 +88,11 @@ function getSections(input: Asana, projectId: string): Workspace[] {
return [...sectionMap.values()]
}
function convert(input: Asana): IArchive {
const archive: IArchive = {
version: 1,
date: Date.now(),
blocks: []
}
function convert(input: Asana): IBlock[] {
const projects = getProjects(input)
if (projects.length < 1) {
console.error('No projects found')
return archive
return []
}
// TODO: Handle multiple projects
@ -181,12 +176,10 @@ function convert(input: Asana): IArchive {
}
})
archive.blocks = blocks
console.log('')
console.log(`Found ${input.data.length} card(s).`)
return archive
return blocks
}
function showHelp() {

4
import/asana/package.json
View file

@ -7,8 +7,8 @@
"scripts": {
"lint": "eslint --ext .tsx,.ts . --quiet --cache",
"fix": "eslint --ext .tsx,.ts . --quiet --fix --cache",
"test": "ts-node importAsana.ts -i test/asana.json -o test/archive.focalboard",
"debug:test": "node --inspect=5858 -r ts-node/register importAsana.ts -i test/asana.json -o test/archive.focalboard"
"test": "ts-node importAsana.ts -i test/asana.json -o test/asana-import.focalboard",
"debug:test": "node --inspect=5858 -r ts-node/register importAsana.ts -i test/asana.json -o test/asana-import.focalboard"
},
"keywords": [],
"author": "",

19
import/notion/importNotion.ts
View file

@ -3,7 +3,7 @@ import * as fs from 'fs'
import minimist from 'minimist'
import path from 'path'
import {exit} from 'process'
import {IArchive} from '../../webapp/src/blocks/archive'
import {ArchiveUtils} from '../../webapp/src/blocks/archive'
import {IBlock} from '../../webapp/src/blocks/block'
import {IPropertyTemplate, MutableBoard} from '../../webapp/src/blocks/board'
import {MutableBoardView} from '../../webapp/src/blocks/boardView'
@ -70,10 +70,11 @@ async function main() {
markdownFolder = path.join(inputFolder, basename)
// Convert
const output = convert(input, title)
const blocks = convert(input, title)
// Save output
const outputData = JSON.stringify(output)
// TODO: Stream output
const outputData = ArchiveUtils.buildBlockArchive(blocks)
fs.writeFileSync(outputFile, outputData)
console.log(`Exported to ${outputFile}`)
@ -115,15 +116,9 @@ function getColumns(input: any[]) {
return keys.slice(1)
}
function convert(input: any[], title: string): IArchive {
function convert(input: any[], title: string): IBlock[] {
const blocks: IBlock[] = []
const archive: IArchive = {
version: 1,
date: Date.now(),
blocks
}
// Board
const board = new MutableBoard()
console.log(`Board: ${title}`)
@ -160,7 +155,7 @@ function convert(input: any[], title: string): IArchive {
console.log(keys)
if (keys.length < 1) {
console.error(`Expected at least one column`)
return archive
return blocks
}
const titleKey = keys[0]
@ -216,7 +211,7 @@ function convert(input: any[], title: string): IArchive {
console.log('')
console.log(`Found ${input.length} card(s).`)
return archive
return blocks
}
function showHelp() {

4
import/notion/package.json
View file

@ -7,8 +7,8 @@
"scripts": {
"lint": "eslint --ext .tsx,.ts . --quiet --cache",
"fix": "eslint --ext .tsx,.ts . --quiet --fix --cache",
"test": "ts-node importNotion.ts -i test/export -o test/archive.focalboard",
"debug:test": "node --inspect=5858 -r ts-node/register importNotion.ts -i test/export -o test/archive.focalboard"
"test": "ts-node importNotion.ts -i test/export -o test/notion-import.focalboard",
"debug:test": "node --inspect=5858 -r ts-node/register importNotion.ts -i test/export -o test/notion-import.focalboard"
},
"keywords": [],
"author": "",

17
import/trello/importTrello.ts
View file

@ -3,7 +3,7 @@
import * as fs from 'fs'
import minimist from 'minimist'
import {exit} from 'process'
import {IArchive} from '../../webapp/src/blocks/archive'
import {ArchiveUtils} from '../../webapp/src/blocks/archive'
import {IBlock} from '../../webapp/src/blocks/block'
import {IPropertyOption, IPropertyTemplate, MutableBoard} from '../../webapp/src/blocks/board'
import {MutableBoardView} from '../../webapp/src/blocks/boardView'
@ -49,16 +49,17 @@ function main() {
const input = JSON.parse(inputData) as Trello
// Convert
const output = convert(input)
const blocks = convert(input)
// Save output
const outputData = JSON.stringify(output)
// TODO: Stream output
const outputData = ArchiveUtils.buildBlockArchive(blocks)
fs.writeFileSync(outputFile, outputData)
console.log(`Exported to ${outputFile}`)
}
function convert(input: Trello): IArchive {
function convert(input: Trello): IBlock[] {
const blocks: IBlock[] = []
// Board
@ -136,16 +137,10 @@ function convert(input: Trello): IArchive {
}
})
const archive: IArchive = {
version: 1,
date: Date.now(),
blocks
}
console.log('')
console.log(`Found ${input.cards.length} card(s).`)
return archive
return blocks
}
function showHelp() {

4
import/trello/package.json
View file

@ -7,8 +7,8 @@
"scripts": {
"lint": "eslint --ext .tsx,.ts . --quiet --cache",
"fix": "eslint --ext .tsx,.ts . --quiet --fix --cache",
"test": "ts-node importTrello.ts -i test/trello.json -o test/archive.focalboard",
"debug:test": "node --inspect=5858 -r ts-node/register importTrello.ts -i test/trello.json -o test/archive.focalboard"
"test": "ts-node importTrello.ts -i test/trello.json -o test/trello-import.focalboard",
"debug:test": "node --inspect=5858 -r ts-node/register importTrello.ts -i test/trello.json -o test/trello-import.focalboard"
},
"keywords": [],
"author": "",

143
webapp/src/archiver.ts
View file

@ -1,7 +1,8 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
import {IArchive} from './blocks/archive'
import {IMutableBlock} from './blocks/block'
import {ArchiveUtils, IArchiveHeader, IArchiveLine, IBlockArchiveLine} from './blocks/archive'
import {IBlock} from './blocks/block'
import {LineReader} from './lineReader'
import mutator from './mutator'
import {Utils} from './utils'
import {BoardTree} from './viewModel/boardTree'
@ -9,28 +10,16 @@ import {BoardTree} from './viewModel/boardTree'
class Archiver {
static async exportBoardTree(boardTree: BoardTree): Promise<void> {
const blocks = boardTree.allBlocks
const archive: IArchive = {
version: 1,
date: Date.now(),
blocks,
}
this.exportArchive(archive)
this.exportArchive(blocks)
}
static async exportFullArchive(): Promise<void> {
const blocks = await mutator.exportFullArchive()
const archive: IArchive = {
version: 1,
date: Date.now(),
blocks,
}
this.exportArchive(archive)
this.exportArchive(blocks)
}
private static exportArchive(archive: IArchive): void {
const content = JSON.stringify(archive)
private static exportArchive(blocks: readonly IBlock[]): void {
const content = ArchiveUtils.buildBlockArchive(blocks)
const date = new Date()
const filename = `archive-${date.getFullYear()}-${date.getMonth() + 1}-${date.getDate()}.focalboard`
@ -48,32 +37,76 @@ class Archiver {
// TODO: Remove or reuse link
}
private static async importBlocksFromFile(file: File): Promise<void> {
let blockCount = 0
const maxBlocksPerImport = 1000
let blocks: IBlock[] = []
let isFirstLine = true
return new Promise<void>((resolve) => {
LineReader.readFile(file, async (line, completed) => {
if (completed) {
if (blocks.length > 0) {
await mutator.importFullArchive(blocks)
blockCount += blocks.length
}
Utils.log(`Imported ${blockCount} blocks.`)
resolve()
return
}
if (isFirstLine) {
isFirstLine = false
const header = JSON.parse(line) as IArchiveHeader
if (header.date && header.version >= 1) {
const date = new Date(header.date)
Utils.log(`Import archive, version: ${header.version}, date/time: ${date.toLocaleString()}.`)
}
} else {
const row = JSON.parse(line) as IArchiveLine
if (!row || !row.type || !row.data) {
Utils.logError('importFullArchive ERROR parsing line')
return
}
switch (row.type) {
case 'block': {
const blockLine = row as IBlockArchiveLine
const block = blockLine.data
if (Archiver.isValidBlock(block)) {
blocks.push(block)
if (blocks.length >= maxBlocksPerImport) {
const blocksToSend = blocks
blocks = []
await mutator.importFullArchive(blocksToSend)
blockCount += blocksToSend.length
}
}
break
}
}
}
})
})
}
static isValidBlock(block: IBlock): boolean {
if (!block.id || !block.rootId) {
return false
}
return true
}
static importFullArchive(onComplete?: () => void): void {
const input = document.createElement('input')
input.type = 'file'
input.accept = '.focalboard'
input.onchange = async () => {
const file = input.files && input.files[0]
const contents = await (new Response(file)).text()
Utils.log(`Import ${contents.length} bytes.`)
const archive: IArchive = JSON.parse(contents)
const {blocks} = archive
const date = new Date(archive.date)
Utils.log(`Import archive, version: ${archive.version}, date/time: ${date.toLocaleString()}, ${blocks.length} block(s).`)
if (file) {
await Archiver.importBlocksFromFile(file)
}
// Basic error checking
let filteredBlocks = blocks.filter((o) => Boolean(o.id))
Utils.log(`Import ${filteredBlocks.length} filtered blocks with ids.`)
this.fixRootIds(filteredBlocks)
filteredBlocks = filteredBlocks.filter((o) => Boolean(o.rootId))
Utils.log(`Import ${filteredBlocks.length} filtered blocks with rootIds.`)
await mutator.importFullArchive(filteredBlocks)
Utils.log('Import completed')
onComplete?.()
}
@ -83,42 +116,6 @@ class Archiver {
// TODO: Remove or reuse input
}
private static fixRootIds(blocks: IMutableBlock[]) {
const blockMap = new Map(blocks.map((o) => [o.id, o]))
const maxLevels = 5
for (let i = 0; i < maxLevels; i++) {
let missingRootIds = false
blocks.forEach((o) => {
if (o.parentId) {
const parent = blockMap.get(o.parentId)
if (parent) {
o.rootId = parent.rootId
} else {
Utils.assert(`No parent for ${o.type}: ${o.id} (${o.title})`)
}
if (!o.rootId) {
missingRootIds = true
}
} else {
o.rootId = o.id
}
})
if (!missingRootIds) {
Utils.log(`fixRootIds in ${i} levels`)
break
}
}
// Check and log remaining errors
blocks.forEach((o) => {
if (!o.rootId) {
const parent = blockMap.get(o.parentId)
Utils.logError(`RootId is null: ${o.type} ${o.id}, parentId ${o.parentId}: ${o.title}, parent: ${parent?.type}, parent.rootId: ${parent?.rootId}, parent.title: ${parent?.title}`)
}
})
}
}
export {Archiver}

24
webapp/src/blocks/archive.test.ts Normal file
View file

@ -0,0 +1,24 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
import {TestBlockFactory} from '../test/testBlockFactory'
import {ArchiveUtils} from './archive'
import {IBlock} from './block'
test('archive: archive and unarchive', async () => {
const blocks: IBlock[] = []
const board = TestBlockFactory.createBoard()
blocks.push(board)
blocks.push(TestBlockFactory.createBoardView(board))
const card = TestBlockFactory.createCard(board)
blocks.push(card)
blocks.push(TestBlockFactory.createText(card))
blocks.push(TestBlockFactory.createDivider(card))
blocks.push(TestBlockFactory.createImage(card))
const archive = ArchiveUtils.buildBlockArchive(blocks)
const unarchivedBlocks = ArchiveUtils.parseBlockArchive(archive)
expect(unarchivedBlocks).toEqual(blocks)
})

76
webapp/src/blocks/archive.ts
View file

@ -2,10 +2,80 @@
// See LICENSE.txt for license information.
import {IBlock} from './block'
interface IArchive {
interface IArchiveHeader {
version: number
date: number
blocks: readonly IBlock[]
}
export {IArchive}
interface IArchiveLine {
type: string,
data: any,
}
// This schema allows the expansion of additional line types in the future
interface IBlockArchiveLine extends IArchiveLine {
type: 'block',
data: IBlock
}
class ArchiveUtils {
static buildBlockArchive(blocks: readonly IBlock[]): string {
const header: IArchiveHeader = {
version: 1,
date: Date.now(),
}
const headerString = JSON.stringify(header)
let content = headerString + '\n'
for (const block of blocks) {
const line: IBlockArchiveLine = {
type: 'block',
data: block,
}
const lineString = JSON.stringify(line)
content += lineString
content += '\n'
}
return content
}
static parseBlockArchive(contents: string): IBlock[] {
const blocks: IBlock[] = []
const allLineStrings = contents.split('\n')
if (allLineStrings.length >= 2) {
const headerString = allLineStrings[0]
const header = JSON.parse(headerString) as IArchiveHeader
if (header.date && header.version >= 1) {
const lineStrings = allLineStrings.slice(1)
let lineNum = 2
for (const lineString of lineStrings) {
if (!lineString) {
// Ignore empty lines, e.g. last line
continue
}
const line = JSON.parse(lineString) as IArchiveLine
if (!line || !line.type || !line.data) {
throw new Error(`ERROR parsing line ${lineNum}`)
}
switch (line.type) {
case 'block': {
const blockLine = line as IBlockArchiveLine
const block = blockLine.data
blocks.push(block)
break
}
}
lineNum += 1
}
} else {
throw new Error('ERROR parsing header')
}
}
return blocks
}
}
export {IArchiveHeader, IArchiveLine, IBlockArchiveLine, ArchiveUtils}

77
webapp/src/lineReader.ts Normal file
View file

@ -0,0 +1,77 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
class LineReader {
private static appendBuffer(buffer1: Uint8Array, buffer2: Uint8Array): Uint8Array {
const tmp = new Uint8Array(buffer1.byteLength + buffer2.byteLength)
tmp.set(buffer1, 0)
tmp.set(buffer2, buffer1.byteLength)
return tmp
}
private static arrayBufferIndexOf(buffer: Uint8Array, charCode: number): number {
for (let i = 0; i < buffer.byteLength; ++i) {
if (buffer[i] === charCode) {
return i
}
}
return -1
}
static readFile(file: File, callback: (line: string, completed: boolean) => Promise<void>): void {
let buffer = new Uint8Array(0)
const chunkSize = 1024 * 1000
let offset = 0
const fr = new FileReader()
const decoder = new TextDecoder()
fr.onload = async () => {
const chunk = new Uint8Array(fr.result as ArrayBuffer)
buffer = LineReader.appendBuffer(buffer, chunk)
const newlineChar = 10 // '\n'
let newlineIndex = LineReader.arrayBufferIndexOf(buffer, newlineChar)
while (newlineIndex >= 0) {
const result = decoder.decode(buffer.slice(0, newlineIndex))
buffer = buffer.slice(newlineIndex + 1)
// eslint-disable-next-line no-await-in-loop
await callback(result, false)
newlineIndex = LineReader.arrayBufferIndexOf(buffer, newlineChar)
}
offset += chunkSize
if (offset >= file.size) {
// Completed
if (buffer.byteLength > 0) {
// Handle last line
await callback(decoder.decode(buffer), false)
}
await callback('', true)
return
}
seek()
}
fr.onerror = () => {
callback('', true)
}
seek()
function seek() {
const slice = file.slice(offset, offset + chunkSize)
// Need to read as an ArrayBuffer (instead of text) to handle unicode boundaries
fr.readAsArrayBuffer(slice)
}
}
}
export {LineReader}

7
webapp/src/octoClient.ts
View file

@ -138,9 +138,10 @@ class OctoClient {
async importFullArchive(blocks: readonly IBlock[]): Promise<Response> {
Utils.log(`importFullArchive: ${blocks.length} blocks(s)`)
blocks.forEach((block) => {
Utils.log(`\t ${block.type}, ${block.id}`)
})
// blocks.forEach((block) => {
// Utils.log(`\t ${block.type}, ${block.id}`)
// })
const body = JSON.stringify(blocks)
return fetch(this.serverUrl + '/api/v1/blocks/import', {
method: 'POST',

4
webapp/src/octoListener.ts
View file

@ -108,7 +108,6 @@ class OctoListener {
switch (message.action) {
case 'UPDATE_BLOCK':
Utils.log(`OctoListener update block: ${message.block?.id}`)
this.queueUpdateNotification(message.block!)
break
default:
@ -207,6 +206,9 @@ class OctoListener {
}
private flushUpdateNotifications() {
for (const block of this.updatedBlocks) {
Utils.log(`OctoListener flush update block: ${block.id}`)
}
this.onChange?.(this.updatedBlocks)
this.updatedBlocks = []
}