olcxjas-softworks/PLWL
2
0
Fork 0
Code Issues Pull requests Projects Releases 16 Packages Wiki Activity Actions

[TEST] try to improve sync

Browse source
This commit is contained in:
olcxja 2026-10-06 07:28:47 +02:00
commit 0f3fbaa418
8 changed files with 374 additions and 145 deletions
Download patch file Download diff file Expand all files Collapse all files

256
src/lansyncpush.js
View file

@ -1,8 +1,9 @@
'use strict'
// The pushing side of a LAN sync: per unit, send a manifest, learn what the
// receiver has, upload the rest, delete the surplus. Exported as a factory;
// src/lansync.js hands in its own helpers and state.
// The pushing side of a LAN sync: per unit, stream a manifest while it is
// still being hashed, learn what the receiver has, upload the rest, delete
// the surplus. Exported as a factory; src/lansync.js hands in its own
// helpers and state.
const http = require('http')
const os = require('os')
@ -12,10 +13,22 @@ const path = require('path')
// Files sent at once: one at a time wastes LAN round trips on small files.
const UPLOAD_CONCURRENCY = 4
// Two lost heartbeats in a row: the receiver is gone (40 s of silence).
const MAX_MISSED_PINGS = 2
// One file, one PUT. A file that has not finished on its own after this long
// (stalled receiver, lost socket while the OS still reports it open, a file
// that changed size after the manifest) fails the whole sync instead of
// hanging it forever. Env override for tests.
const UPLOAD_TIMEOUT_MS = Number(process.env.PLWL_LANSYNC_UPLOAD_TIMEOUT) || 10 * 60 * 1000
// Keeps the receiver alive while this side hashes (minutes of silence); env
// override for tests.
const HEARTBEAT_MS = Number(process.env.PLWL_LANSYNC_HEARTBEAT) || 20000
// One ping round trip; env override for tests.
const PING_TIMEOUT_MS = Number(process.env.PLWL_LANSYNC_PING_TIMEOUT) || 10000
function createPusher(api) {
const {
CATEGORIES,
@ -26,7 +39,7 @@ function createPusher(api) {
httpJson,
filesPath,
ownedRoot,
buildManifest,
buildManifestStream,
listDirs,
DIRS,
toPortablePath,
@ -35,8 +48,10 @@ function createPusher(api) {
prefixes
} = api
// Manifest in, upload list out; NDJSON avoids one giant string.
function planUnit(device, token, unit, entries, force, meta) {
// Manifest in, upload list out; NDJSON avoids one giant string. The body
// is written by `produce` WHILE the sender hashes, so the receiver can
// compare entries as they arrive (both sides hash at the same time).
function planUnit(device, token, unit, force, meta, produce) {
// announced up front, so the receiver's popup can say "Apps: 2/5"
const totals = meta && meta.totals ? `&totals=${encodeURIComponent(JSON.stringify(meta.totals))}` : ''
return new Promise((resolve, reject) => {
@ -62,23 +77,47 @@ function createPusher(api) {
})
}
)
req.on('error', reject)
let index = 0
const pump = () => {
while (index < entries.length) {
if (!req.write(`${JSON.stringify(entries[index++])}\n`)) {
req.once('drain', pump)
return
}
let dead = null
req.on('error', err => {
dead = err
reject(err)
})
// ordered writes with backpressure: the receiver reads continuously,
// but a slow socket must not buffer the whole manifest in memory
const write = obj => new Promise((resolveWrite, rejectWrite) => {
if (dead) return rejectWrite(dead)
const onDrain = () => {
cleanup()
resolveWrite()
}
req.end()
}
pump()
const onErr = () => {
cleanup()
rejectWrite(dead || new Error('plan-write-failed'))
}
const cleanup = () => {
req.off('drain', onDrain)
req.off('error', onErr)
}
if (req.write(`${JSON.stringify(obj)}\n`)) {
cleanup()
return resolveWrite()
}
req.once('drain', onDrain)
req.once('error', onErr)
})
produce(write).then(() => req.end(), err => {
try { req.destroy() } catch {}
reject(err)
})
})
}
function putFile(device, token, unitKey, rel, abs) {
return new Promise((resolve, reject) => {
const settle = (fn, value) => {
clearTimeout(timer)
fn(value)
}
// unit and path travel as headers: constant per request
const req = http.request(
{
@ -96,12 +135,26 @@ function createPusher(api) {
},
res => {
res.resume()
res.on('end', () => resolve({ status: res.statusCode }))
res.on('end', () => settle(resolve, { status: res.statusCode }))
res.on('aborted', () => settle(reject, new Error('response-aborted')))
}
)
req.on('error', reject)
// The timer spans the WHOLE transfer, response included: a stalled
// receiver or a silently lost socket must fail the sync instead of
// hanging the popup on the last file. Generous - a file would have to
// take over ten minutes at 20 MB/s to hit it.
const timer = setTimeout(() => {
req.destroy(new Error('upload-timeout'))
settle(reject, new Error('upload-timeout'))
}, UPLOAD_TIMEOUT_MS)
if (timer.unref) timer.unref()
req.on('error', err => settle(reject, err))
const stream = fs.createReadStream(abs)
stream.on('error', reject)
stream.on('error', err => settle(reject, err))
// pipe ends the request when the file ends; a body shorter than the
// declared content-length (file changed mid-sync) wedges the receiver
// - its own idle watchdog breaks that, this side's timer breaks every
// other silent case
stream.pipe(req)
})
}
@ -265,8 +318,9 @@ function createPusher(api) {
// units are known before anything is read: announce them with the verify
const units = buildUnits(cats)
const bucketOf = kind => (kind === 'wine' ? 'wines' : kind === 'prefix' ? 'prefixes' : 'apps')
const totals = { apps: 0, wines: 0, prefixes: 0 }
const totals = { apps: 0, wines: 0, prefixes: 0, settings: 0 }
for (const unit of units) totals[bucketOf(unit.kind)] += 1
if (cats.has('settings')) totals.settings = 1
// every progress event is kept, so callers can report without subscribing
const events = []
const progress = data => {
@ -274,32 +328,6 @@ function createPusher(api) {
if (onProgress) onProgress(data)
}
let verified
try {
// longer than the receiver's incoming-request timeout (popup wait)
verified = await httpJson(device, 'POST', '/plwl/verify', {
json: { deviceId: identity().deviceId, name: os.hostname(), code: String(code == null ? '' : code).trim().toUpperCase(), totals },
timeout: 90000
})
} catch (err) {
return fail(String((err && err.message) || err))
}
if (!verified.data || !verified.data.ok) {
const error = (verified.data && verified.data.error) || `http-${verified.status}`
// a rotated code drops the pairing; a plain 'rejected' keeps it
if (error === 'auth-failed' || error === 'bad-code') forgetDevice(device.id || deviceId)
return fail(error)
}
const token = verified.data.token
// paired now: the next sync needs no typing
rememberDevice(device.id || deviceId, code, device.name)
// hashing precedes any byte: keep the receiver's popup alive meanwhile
const heartbeat = setInterval(() => {
httpJson(device, 'POST', `/plwl/ping?token=${encodeURIComponent(token)}`, { json: {}, timeout: 10000 }).catch(() => {})
}, HEARTBEAT_MS)
if (heartbeat.unref) heartbeat.unref()
const summary = newSummary(units.length)
const finished = { apps: 0, wines: 0, prefixes: 0 }
// planned bytes so far, so the popup can show "1.2 GB / 3.4 GB"
@ -341,13 +369,58 @@ function createPusher(api) {
// snapshot per file, forward a few times per second (phase changes always)
const gate = typeof api.progressGate === 'function' ? api.progressGate(progress) : data => progress(data)
const emit = (phase, unit, done, total, force) => gate(snapshot(phase, unit, done, total), !!force)
// consent may wait on a human: say so before anything else happens
emit('wait', null, null, null, true)
let verified
try {
// longer than the receiver's incoming-request timeout (popup wait)
verified = await httpJson(device, 'POST', '/plwl/verify', {
json: { deviceId: identity().deviceId, name: os.hostname(), code: String(code == null ? '' : code).trim().toUpperCase(), totals },
timeout: 90000
})
} catch (err) {
return fail(String((err && err.message) || err))
}
if (!verified.data || !verified.data.ok) {
const error = (verified.data && verified.data.error) || `http-${verified.status}`
// a rotated code drops the pairing; a plain 'rejected' keeps it
if (error === 'auth-failed' || error === 'bad-code') forgetDevice(device.id || deviceId)
return fail(error)
}
const token = verified.data.token
// paired now: the next sync needs no typing
rememberDevice(device.id || deviceId, code, device.name)
// The heartbeat keeps the receiver's popup alive while this side hashes
// (minutes of silence). Consecutive ping failures mean the peer dropped
// (crash, sleep, lost network): without this a dead receiver hangs the
// sync until the next request notices, which may be never.
let missedPings = 0
const abort = { dead: false, reject: null }
const abortPromise = new Promise((resolve, reject) => { abort.reject = reject })
abortPromise.catch(() => {}) // only awaited via Promise.race below
const heartbeat = setInterval(() => {
httpJson(device, 'POST', `/plwl/ping?token=${encodeURIComponent(token)}`, { json: {}, timeout: PING_TIMEOUT_MS })
.then(() => { missedPings = 0 })
.catch(() => {
missedPings++
if (missedPings >= MAX_MISSED_PINGS && !abort.dead) {
abort.dead = true
abort.reject(new Error('receiver-lost'))
}
})
}, HEARTBEAT_MS)
if (heartbeat.unref) heartbeat.unref()
const end = async () => {
try {
await httpJson(device, 'POST', `/plwl/end?token=${encodeURIComponent(token)}`, { json: {} })
} catch {}
}
try {
const runUnits = async () => {
for (const unit of units) {
if (abort.dead) throw new Error('receiver-lost')
const roots = unit.roots.filter(ownedRoot)
if (!roots.length) continue
const kind = bucketOf(unit.kind)
@ -357,14 +430,36 @@ function createPusher(api) {
barBytes = 0
barTotal = 0
emit('scan', unit, null, null, true)
const entries = await buildManifest(roots, !!force, (done, total, hashedBytes, totalBytes) => {
barBytes = hashedBytes
barTotal = totalBytes
// no file count until the plan returns
emit('hash', unit, null, null)
// The manifest streams WHILE this side hashes its tail: the receiver
// compares entries as they arrive, so both devices hash at the same
// time. Once this side is done hashing, the popup keeps a full bar
// and the waiting text until the plan answers (a cached sender waits
// for a receiver that still has to hash).
const entries = []
let writes = Promise.resolve()
const plan = await planUnit(device, token, { ...unit, roots }, !!force, { totals }, async write => {
await buildManifestStream(roots, !!force, {
// the receiver sizes its compare bar from this, before any hashing
onTotal: totalBytes => {
writes = writes.then(() => write({ unitBytes: totalBytes }))
},
onEntry: entry => {
entries.push(entry)
writes = writes.then(() => write(entry))
},
onProgress: (hashedDone, hashedTotal, hashedBytes, totalBytes) => {
barBytes = hashedBytes
barTotal = totalBytes
emit('hash', unit, null, null)
}
})
await writes
// this side is done hashing; the receiver may still be comparing
barTotal = barTotal || 1
barBytes = barTotal
emit('wait', unit, null, null, true)
})
emit('plan', unit, null, null, true)
const plan = await planUnit(device, token, { ...unit, roots }, entries, !!force, { totals })
if (abort.dead) throw new Error('receiver-lost')
if (!plan.data || !plan.data.ok) throw new Error((plan.data && plan.data.error) || `http-${plan.status}`)
const upload = new Set(plan.data.upload || [])
const same = new Set(plan.data.same || [])
@ -383,32 +478,32 @@ function createPusher(api) {
pending.push(entry)
plannedBytes += entry.size || 0
}
// a few files in flight at once: small files cost round trips
// the bar counts only bytes that move, and starts empty for the copy
// a few files in flight at once: small files cost round trips; the
// bar counts only bytes that move and starts empty for the copy
if (pending.length) {
barBytes = 0
barTotal = pending.reduce((sum, entry) => sum + (entry.size || 0), 0)
emit('copy', unit, allDone, allTotal + pending.length, true)
}
let failed = null
let done = 0
const send = async entry => {
if (failed) return
const res = await putFile(device, token, unit.key, entry.path, filesPath(entry.path))
if (res.status !== 200) {
failed = new Error(`upload-failed-${res.status}`)
return
let failed = null
let done = 0
const send = async entry => {
if (failed || abort.dead) return
const res = await putFile(device, token, unit.key, entry.path, filesPath(entry.path))
if (res.status !== 200) {
failed = new Error(`upload-failed-${res.status}`)
return
}
done++
summary.uploaded++
summary.bytes += entry.size || 0
barBytes += entry.size || 0
emit('copy', unit, allDone + done, allTotal + pending.length)
}
done++
summary.uploaded++
summary.bytes += entry.size || 0
barBytes += entry.size || 0
emit('copy', unit, allDone + done, allTotal + pending.length)
if (typeof api.mapLimit === 'function') await api.mapLimit(pending, UPLOAD_CONCURRENCY, send)
else for (const entry of pending) await send(entry)
if (failed) throw failed
}
if (typeof api.mapLimit === 'function') await api.mapLimit(pending, UPLOAD_CONCURRENCY, send)
else for (const entry of pending) await send(entry)
if (failed) throw failed
emit('copy', unit, allDone + done, allTotal + pending.length, true)
emit('copy', unit, allDone + pending.length, allTotal + pending.length, true)
// the totals count a unit's files only once they are on the wire
allDone += pending.length
allTotal += pending.length
@ -431,9 +526,16 @@ function createPusher(api) {
if (deleted) summary[key].deleted += deleted
}
if (applied.data.deleted) summary.deleted += applied.data.deleted
// a sync with nothing to send still ends with a full bar
barTotal = barTotal || 1
barBytes = barTotal
emit('done', null, allDone, allTotal, true)
await end()
}
try {
await Promise.race([runUnits(), abortPromise])
} catch (err) {
abort.dead = true
await end()
return fail(String((err && err.message) || err), summary, events)
} finally {
@ -442,7 +544,9 @@ function createPusher(api) {
return syncResult(true, null, summary, events)
}
return { syncToDevice, buildUnits, buildRegistryPayload, CATEGORIES }
// putFile is exposed for the stall tests (it is the piece that must never
// leave a request half-open)
return { syncToDevice, buildUnits, buildRegistryPayload, putFile, CATEGORIES }
}
module.exports = createPusher