[TEST] try to improve sync
This commit is contained in:
parent
be439e5421
commit
0f3fbaa418
8 changed files with 374 additions and 145 deletions
128
src/lansync.js
128
src/lansync.js
|
|
@ -240,14 +240,23 @@ function readJsonBody(req, limit = 16 * 1024 * 1024) {
|
|||
function readNdjson(req, onEntry) {
|
||||
return new Promise(resolve => {
|
||||
const rl = readline.createInterface({ input: req, crlfDelay: Infinity })
|
||||
let chain = Promise.resolve()
|
||||
rl.on('line', line => {
|
||||
if (!line.trim()) return
|
||||
// entries are handled strictly in order: a handler may be async (the
|
||||
// streamed plan compares/hashes while the body keeps arriving), but
|
||||
// the response must never reorder the manifest
|
||||
let parsed = null
|
||||
try {
|
||||
onEntry(JSON.parse(line))
|
||||
} catch {}
|
||||
parsed = JSON.parse(line)
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
chain = chain.then(() => onEntry(parsed)).catch(() => {})
|
||||
})
|
||||
rl.on('close', resolve)
|
||||
rl.on('error', resolve)
|
||||
const done = () => chain.then(resolve, () => resolve())
|
||||
rl.on('close', done)
|
||||
rl.on('error', done)
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -801,7 +810,7 @@ async function handleVerify(state, req, res, hooks) {
|
|||
requestId,
|
||||
deviceId: from,
|
||||
name: String(payload.name || ''),
|
||||
totals: { apps: Number(announced.apps) || 0, wines: Number(announced.wines) || 0, prefixes: Number(announced.prefixes) || 0 }
|
||||
totals: { apps: Number(announced.apps) || 0, wines: Number(announced.wines) || 0, prefixes: Number(announced.prefixes) || 0, settings: Number(announced.settings) || 0 }
|
||||
})
|
||||
} catch {
|
||||
resolve('reject') // no window to answer with -> never auto-accept silently
|
||||
|
|
@ -857,13 +866,14 @@ async function handlePlan(state, session, url, req, res, hooks) {
|
|||
} catch {}
|
||||
const owned = (Array.isArray(roots) ? roots : []).map(ownedRoot).filter(Boolean)
|
||||
if (!key || !owned.length) return sendJson(res, 400, { ok: false, error: 'bad-unit' })
|
||||
const entries = []
|
||||
await readNdjson(req, entry => {
|
||||
if (inlineSame(entry)) entries.push(entry)
|
||||
})
|
||||
// the mode each file has to end up with on this side
|
||||
const modes = new Map()
|
||||
for (const entry of entries) if (entry.type === 'file' && typeof entry.mode === 'number') modes.set(entry.path, entry.mode)
|
||||
// comparing is its own phase: the bar restarts for the copy phase
|
||||
const announce = progressGate((data, force) => emitReceive(hooks, session, 'compare', data))
|
||||
// the unit counts from the start of its comparison, not from its commit
|
||||
const kind = UNIT_KINDS[String(key).split(':')[0]]
|
||||
if (kind && !session.counted.has(key)) {
|
||||
session.counted.add(key)
|
||||
session.items[kind] = (session.items[kind] || 0) + 1
|
||||
}
|
||||
// a force sync compares nothing: every entry counts as different
|
||||
const forced = url.searchParams.get('force') === '1'
|
||||
// the sender's totals, so the popup can say "2/5" from the start
|
||||
|
|
@ -877,24 +887,64 @@ async function handlePlan(state, session, url, req, res, hooks) {
|
|||
}
|
||||
}
|
||||
} catch {}
|
||||
const allowed = entries.filter(entry => withinRoots(entry.path, owned))
|
||||
// comparing is its own phase: the bar restarts for the copy phase
|
||||
const unitTotalBytes = allowed.reduce((sum, entry) => sum + (entry.size || 0), 0)
|
||||
const announce = progressGate((data, force) => emitReceive(hooks, session, 'plan', data))
|
||||
// The manifest is compared as it streams in (both sides hash at the same
|
||||
// time). The sender sends its unit total first, so the bar starts with its
|
||||
// full width; files missing from the sender's total compare as the usual
|
||||
// entries - the bar just grows when that happens.
|
||||
const entries = []
|
||||
const results = new Map()
|
||||
let unitTotalBytes = 0
|
||||
let compared = 0
|
||||
let comparedBytes = 0
|
||||
session.barTotal = unitTotalBytes
|
||||
session.barBytes = 0
|
||||
const identical = await mapLimit(allowed, HASH_CONCURRENCY, async entry => {
|
||||
const compareOne = async entry => {
|
||||
const same = forced ? false : await entrySame(entry)
|
||||
results.set(entry.path, same === true)
|
||||
compared++
|
||||
comparedBytes += entry.size || 0
|
||||
session.barBytes = comparedBytes
|
||||
// the sender's announced total (or a provisional full bar until it
|
||||
// arrives) - clamped at the top by the renderer, never over 100%
|
||||
session.barTotal = unitTotalBytes || 1
|
||||
// no file counting yet: dirs/links are compared too
|
||||
announce({ unit: label })
|
||||
return same === true
|
||||
}
|
||||
// files compared concurrently: the sender streams its manifest, this side
|
||||
// keeps at most HASH_CONCURRENCY digests in flight (the sender's hashing
|
||||
// queue is the wider throttle)
|
||||
const jobs = []
|
||||
let inFlight = 0
|
||||
const settleOldest = () => Promise.race(jobs.splice(0, 1))
|
||||
await readNdjson(req, async entry => {
|
||||
if (!inlineSame(entry)) return
|
||||
if (entry.unitBytes != null) {
|
||||
// the sender announces its unit total before hashing starts; it can
|
||||
// arrive after early compares (dirs/links), so it just overwrites
|
||||
unitTotalBytes = Number(entry.unitBytes) || 0
|
||||
session.barTotal = unitTotalBytes || 1
|
||||
return
|
||||
}
|
||||
entries.push(entry)
|
||||
if (!withinRoots(entry.path, owned)) return
|
||||
if (forced) {
|
||||
await compareOne(entry)
|
||||
return
|
||||
}
|
||||
// the sender's hashing queue is the throttle on its side; this side's
|
||||
// own concurrency limit keeps a saturated disk from stacking up here
|
||||
if (inFlight >= HASH_CONCURRENCY) await settleOldest()
|
||||
inFlight++
|
||||
const job = compareOne(entry).finally(() => {
|
||||
inFlight--
|
||||
})
|
||||
jobs.push(job)
|
||||
})
|
||||
await Promise.all(jobs)
|
||||
hashStore.save()
|
||||
const allowed = entries.filter(entry => withinRoots(entry.path, owned))
|
||||
const identical = allowed.map(entry => results.get(entry.path) === true)
|
||||
// the mode each file has to end up with on this side
|
||||
const modes = new Map()
|
||||
for (const entry of allowed) if (entry.type === 'file' && typeof entry.mode === 'number') modes.set(entry.path, entry.mode)
|
||||
const paths = new Set()
|
||||
const upload = []
|
||||
const same = []
|
||||
|
|
@ -922,17 +972,19 @@ async function handlePlan(state, session, url, req, res, hooks) {
|
|||
session.unitWritten = 0
|
||||
session.barTotal = uploadBytes
|
||||
session.barBytes = 0
|
||||
const kind = UNIT_KINDS[String(key).split(':')[0]]
|
||||
if (kind && !session.counted.has(key)) {
|
||||
session.counted.add(key)
|
||||
session.items[kind] = (session.items[kind] || 0) + 1
|
||||
}
|
||||
// the unit's label is announced up front, so file events can name it
|
||||
session.unitLabel = label
|
||||
sendJson(res, 200, { ok: true, upload, same, entries: paths.size })
|
||||
announce({ unit: label }, true)
|
||||
}
|
||||
|
||||
// One PUT, one file. The sender declares a content-length up front; if the
|
||||
// body stops short of it (sender crashed, lost link, file changed size after
|
||||
// the manifest), the request would otherwise stay open forever and wedge
|
||||
// this file, the session and the sync. Any arriving byte re-arms the timer,
|
||||
// so a slow honest transfer never trips it. Env override for tests.
|
||||
const PUT_IDLE_MS = Number(process.env.PLWL_LANSYNC_PUT_IDLE) || 60000
|
||||
|
||||
async function handlePut(session, url, req, res, hooks) {
|
||||
// unit and path travel as headers: constant per request
|
||||
const key = req.headers['x-plwl-unit'] || url.searchParams.get('unit') || ''
|
||||
|
|
@ -945,8 +997,22 @@ async function handlePut(session, url, req, res, hooks) {
|
|||
}
|
||||
// report bytes while the file is still arriving
|
||||
const announce = progressGate((data, force) => emitReceive(hooks, session, 'copy', data))
|
||||
// a ping must not flash "waiting" while transfers are running
|
||||
session.writing = (session.writing || 0) + 1
|
||||
let idleTimer = null
|
||||
const armIdle = () => {
|
||||
if (idleTimer) clearTimeout(idleTimer)
|
||||
idleTimer = setTimeout(() => {
|
||||
// the write's stream 'error' handler removes the tmp file; the sender's
|
||||
// request errors out and its own timer/heartbeat handles the rest
|
||||
req.destroy(new Error('body-stalled'))
|
||||
}, PUT_IDLE_MS)
|
||||
if (idleTimer.unref) idleTimer.unref()
|
||||
}
|
||||
armIdle()
|
||||
try {
|
||||
await writeStream(abs, req, delta => {
|
||||
armIdle()
|
||||
session.bytes = (session.bytes || 0) + delta
|
||||
session.unitWritten = (session.unitWritten || 0) + delta
|
||||
session.barBytes = session.unitWritten
|
||||
|
|
@ -959,6 +1025,9 @@ async function handlePut(session, url, req, res, hooks) {
|
|||
announce({ unit: session.unitLabel || '' }, true)
|
||||
} catch (err) {
|
||||
sendJson(res, 500, { ok: false, error: String((err && err.message) || err) })
|
||||
} finally {
|
||||
if (idleTimer) clearTimeout(idleTimer)
|
||||
session.writing--
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1030,10 +1099,13 @@ function handleEnd(state, session, res, hooks) {
|
|||
emitReceive(hooks, session, 'done', { deleted: session.deleted || 0 })
|
||||
}
|
||||
|
||||
// Liveness ping: hashing can take minutes; re-arms the popup watchdog.
|
||||
// Liveness ping: hashing can take minutes; re-arms the popup watchdog. The
|
||||
// event must repeat on every ping (it is what keeps the popup's watchdog
|
||||
// armed), but never while a file is being written: a copy in flight reports
|
||||
// itself and must not flash the waiting text over the receiving phase.
|
||||
function handlePing(session, res, hooks) {
|
||||
session.lastAt = Date.now()
|
||||
emitReceive(hooks, session, 'wait')
|
||||
if (!session.writing) emitReceive(hooks, session, 'wait')
|
||||
sendJson(res, 200, { ok: true })
|
||||
}
|
||||
|
||||
|
|
@ -1140,6 +1212,7 @@ function getPusher() {
|
|||
filesPath,
|
||||
ownedRoot,
|
||||
buildManifest,
|
||||
buildManifestStream: manifest.buildManifestStream,
|
||||
listDirs,
|
||||
listDevices,
|
||||
mapLimit,
|
||||
|
|
@ -1219,6 +1292,7 @@ const exported = {
|
|||
filesPath,
|
||||
ownedRoot,
|
||||
buildManifest,
|
||||
buildManifestStream: manifest.buildManifestStream,
|
||||
mapLimit,
|
||||
listDirs,
|
||||
syncTo
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue