From ab7d936e9f2f8a1a43518956dedbb1cb073200a0 Mon Sep 17 00:00:00 2001
From: Chen-I Lim <chenilim@gmail.com>
Date: Tue, 2 Mar 2021 16:13:26 -0800
Subject: [PATCH] async LineReader

---
 webapp/src/archiver.ts   | 26 +++++++++++++++++---------
 webapp/src/lineReader.ts | 12 +++++++-----
 2 files changed, 24 insertions(+), 14 deletions(-)

diff --git a/webapp/src/archiver.ts b/webapp/src/archiver.ts
index bb18c70f7..475795497 100644
--- a/webapp/src/archiver.ts
+++ b/webapp/src/archiver.ts
@@ -37,14 +37,21 @@ class Archiver {
         // TODO: Remove or reuse link
     }
 
-    private static async readBlocksFromFile(file: File): Promise<IBlock[]> {
+    private static async importBlocksFromFile(file: File): Promise<void> {
+        let blockCount = 0
+        const maxBlocksPerImport = 100
         const blocks: IBlock[] = []
 
         let isFirstLine = true
-        return new Promise<IBlock[]>((resolve) => {
-            LineReader.readFile(file, (line, completed) => {
+        return new Promise<void>((resolve) => {
+            LineReader.readFile(file, async (line, completed) => {
                 if (completed) {
-                    resolve(blocks)
+                    if (blocks.length > 0) {
+                        await mutator.importFullArchive(blocks)
+                        blockCount += blocks.length
+                    }
+                    Utils.log(`Imported ${blockCount} blocks.`)
+                    resolve()
                     return
                 }
 
@@ -67,6 +74,11 @@ class Archiver {
                         const block = blockLine.data
                         if (Archiver.isValidBlock(block)) {
                             blocks.push(block)
+                            if (blocks.length >= maxBlocksPerImport) {
+                                await mutator.importFullArchive(blocks)
+                                blockCount += blocks.length
+                                blocks.length = 0
+                            }
                         }
                         break
                     }
@@ -91,11 +103,7 @@ class Archiver {
         input.onchange = async () => {
             const file = input.files && input.files[0]
             if (file) {
-                const blocks = await Archiver.readBlocksFromFile(file)
-
-                Utils.log(`Importing ${blocks.length} blocks...`)
-                await mutator.importFullArchive(blocks)
-                Utils.log(`Imported ${blocks.length} blocks.`)
+                await Archiver.importBlocksFromFile(file)
             }
 
             onComplete?.()
diff --git a/webapp/src/lineReader.ts b/webapp/src/lineReader.ts
index b3d06badb..e2cc06bbf 100644
--- a/webapp/src/lineReader.ts
+++ b/webapp/src/lineReader.ts
@@ -20,7 +20,7 @@ class LineReader {
         return -1
     }
 
-    static readFile(file: File, callback: (line: string, completed: boolean) => void): void {
+    static readFile(file: File, callback: (line: string, completed: boolean) => Promise<void>): void {
         let buffer = new Uint8Array(0)
 
         const chunkSize = 1024 * 1000
@@ -28,7 +28,7 @@ class LineReader {
         const fr = new FileReader()
         const decoder = new TextDecoder()
 
-        fr.onload = () => {
+        fr.onload = async () => {
             const chunk = new Uint8Array(fr.result as ArrayBuffer)
             buffer = LineReader.appendBuffer(buffer, chunk)
 
@@ -37,7 +37,9 @@ class LineReader {
             while (newlineIndex >= 0) {
                 const result = decoder.decode(buffer.slice(0, newlineIndex))
                 buffer = buffer.slice(newlineIndex + 1)
-                callback(result, false)
+
+                // eslint-disable-next-line no-await-in-loop
+                await callback(result, false)
                 newlineIndex = LineReader.arrayBufferIndexOf(buffer, newlineChar)
             }
 
@@ -47,10 +49,10 @@ class LineReader {
 
                 if (buffer.byteLength > 0) {
                     // Handle last line
-                    callback(decoder.decode(buffer), false)
+                    await callback(decoder.decode(buffer), false)
                 }
 
-                callback('', true)
+                await callback('', true)
                 return
             }