[TEST] try to improve sync

src/lansync-manifest.js

@@ -22,7 +22,9 @@ function createManifestBuilder({ filesPath, filesRoot, hash, mapLimit, concurren
 
   // Walk without following symlinks (dosdevices etc. are part of the sync).
   // Iterative, with one directory's stat/readlink calls in flight together.
-  async function scanTree(abs, rel, out, mtimes) {
+  // `emit` (optional) streams dirs and links out while the walk continues;
+  // file entries only land in `out` (their digest comes later).
+  async function scanTree(abs, rel, out, mtimes, emit) {
     const queue = [[abs, rel]]
     while (queue.length) {
       const [dirAbs, dirRel] = queue.pop()
@@ -37,7 +39,9 @@ function createManifestBuilder({ filesPath, filesRoot, hash, mapLimit, concurren
       for (const entry of entries) {
         const relChild = `${dirRel}/${entry.name}`
         if (entry.isDirectory()) {
-          out.push({ path: relChild, type: 'dir' })
+          const record = { path: relChild, type: 'dir' }
+          out.push(record)
+          if (emit) await emit(record)
           dirs.push([path.join(dirAbs, entry.name), relChild])
         } else {
           leaves.push(entry)
@@ -51,7 +55,9 @@ function createManifestBuilder({ filesPath, filesRoot, hash, mapLimit, concurren
           try {
             target = await fsp.readlink(absChild)
           } catch {}
-          out.push({ path: relChild, type: 'link', target: portableLinkTarget(absChild, target) })
+          const record = { path: relChild, type: 'link', target: portableLinkTarget(absChild, target) }
+          out.push(record)
+          if (emit) await emit(record)
           return
         }
         let st
@@ -70,8 +76,14 @@ function createManifestBuilder({ filesPath, filesRoot, hash, mapLimit, concurren
     }
   }
 
-  // Manifest of one unit; `force` skips hashing (receiver replaces everything).
-  async function buildManifest(roots, force, onProgress) {
+  // Manifest of one unit, streamed: `handlers` may carry onEntry (a Promise
+  // per entry, awaited - backpressure), onTotal and onProgress. File entries
+  // go out as their digests finish, so the receiver compares while this side
+  // is still hashing; hashing itself stays parallel, a batch at a time.
+  async function buildManifestStream(roots, force, handlers) {
+    const onEntry = handlers && handlers.onEntry
+    const onTotal = handlers && handlers.onTotal
+    const onProgress = handlers && handlers.onProgress
     const entries = []
     const mtimes = new Map()
     for (const rel of roots) {
@@ -88,45 +100,69 @@ function createManifestBuilder({ filesPath, filesRoot, hash, mapLimit, concurren
         try {
           target = await fsp.readlink(abs)
         } catch {}
-        entries.push({ path: rel, type: 'link', target: portableLinkTarget(abs, target) })
+        const record = { path: rel, type: 'link', target: portableLinkTarget(abs, target) }
+        entries.push(record)
+        if (onEntry) await onEntry(record)
       } else if (st.isDirectory()) {
-        entries.push({ path: rel, type: 'dir' })
-        await scanTree(abs, rel, entries, mtimes)
+        const record = { path: rel, type: 'dir' }
+        entries.push(record)
+        if (onEntry) await onEntry(record)
+        await scanTree(abs, rel, entries, mtimes, onEntry)
       } else if (st.isFile()) {
         entries.push({ path: rel, type: 'file', size: st.size, mode: st.mode & 0o777, hash: null })
         mtimes.set(rel, st.mtimeMs || 0)
       }
     }
     const files = entries.filter(e => e.type === 'file')
-    // total size is known before hashing, so the bar starts empty
+    // total size is known before hashing: the receiver sizes its compare bar
+    // from this, the sender's popup from onProgress
     const totalBytes = files.reduce((sum, entry) => sum + (entry.size || 0), 0)
-    if (!force && files.length) {
-      let done = 0
-      let hashed = 0
-      await mapLimit(files, concurrency, async entry => {
-        const abs = filesPath(entry.path)
-        try {
-          // unchanged size + mtime: the cached digest still holds
-          const mtimeMs = mtimes.get(entry.path) || 0
-          entry.hash = abs
-            ? await hash.cachedHash(entry.path, abs, entry.size, mtimeMs, bytes => {
-              hashed += bytes
-              if (onProgress) onProgress(done, files.length, hashed, totalBytes)
-            })
-            : null
-        } catch {
-          entry.hash = null
-        }
+    if (onTotal) await onTotal(totalBytes)
+    let done = 0
+    let hashed = 0
+    // one batch hashes with the whole disk queue behind it; its entries go
+    // on the wire in order before the next batch starts
+    for (let i = 0; i < files.length; i += concurrency) {
+      const batch = files.slice(i, i + concurrency)
+      if (!force) {
+        await Promise.all(batch.map(async entry => {
+          const abs = filesPath(entry.path)
+          try {
+            // unchanged size + mtime: the cached digest still holds
+            const mtimeMs = mtimes.get(entry.path) || 0
+            entry.hash = abs
+              ? await hash.cachedHash(entry.path, abs, entry.size, mtimeMs, bytes => {
+                hashed += bytes
+                if (onProgress) onProgress(done, files.length, hashed, totalBytes)
+              })
+              : null
+          } catch {
+            entry.hash = null
+          }
+        }))
+      }
+      for (const entry of batch) {
         done++
         if (!entry.hash) hashed += entry.size || 0 // nothing read: still report the file as passed over
         if (onProgress) onProgress(done, files.length, hashed, totalBytes)
-      })
+        if (onEntry) await onEntry(entry)
+      }
     }
     hash.save()
     return entries
   }
 
-  return { buildManifest, scanTree, portableLinkTarget }
+  // Buffering variant for callers that want the whole manifest first.
+  async function buildManifest(roots, force, onProgress) {
+    const entries = []
+    await buildManifestStream(roots, force, {
+      onProgress,
+      onEntry: entry => { entries.push(entry) }
+    })
+    return entries
+  }
+
+  return { buildManifest, buildManifestStream, scanTree, portableLinkTarget }
 }
 
 module.exports = { createManifestBuilder }