'use strict' // The pushing side of a LAN sync: this device is the source of truth and // drives the whole transfer. For every sync unit it sends a manifest, asks // the receiver what it already has, uploads the rest and has the receiver // delete whatever the manifest does not list. // // Exported as a factory: src/lansync.js hands in its own helpers and state, so // nothing here reaches across module instances. const http = require('http') const os = require('os') const fs = require('fs') const path = require('path') // Files sent at once: one at a time wastes most of a LAN round trip on the // small files a game or prefix is mostly made of. const UPLOAD_CONCURRENCY = 4 function createPusher(api) { const { CATEGORIES, normalizeCategories, identity, rememberDevice, forgetDevice, httpJson, filesPath, ownedRoot, buildManifest, listDirs, DIRS, toPortablePath, library, wines, prefixes } = api // Manifest in, list of files to upload out. NDJSON keeps a prefix with tens // of thousands of entries from being one giant string in memory either side. function planUnit(device, token, unit, entries, force, meta) { // the receiver is told how many apps/prefixes/wines the whole sync carries // up front, which is what lets its popup say "Apps: 2/5" const totals = meta && meta.totals ? `&totals=${encodeURIComponent(JSON.stringify(meta.totals))}` : '' return new Promise((resolve, reject) => { const route = `/plwl/unit/plan?token=${encodeURIComponent(token)}&unit=${encodeURIComponent(unit.key)}&label=${encodeURIComponent(unit.label || '')}&roots=${encodeURIComponent(JSON.stringify(unit.roots))}${force ? '&force=1' : ''}${totals}` const req = http.request( { host: device.address, port: device.port, path: route, method: 'POST', headers: { 'content-type': 'application/x-ndjson' }, timeout: 0 }, res => { const chunks = [] res.on('data', chunk => chunks.push(chunk)) res.on('end', () => { let data = null try { data = JSON.parse(Buffer.concat(chunks).toString('utf8')) } catch {} resolve({ status: res.statusCode, data }) }) } ) req.on('error', reject) let index = 0 const pump = () => { while (index < entries.length) { if (!req.write(`${JSON.stringify(entries[index++])}\n`)) { req.once('drain', pump) return } } req.end() } pump() }) } function putFile(device, token, unitKey, rel, abs) { return new Promise((resolve, reject) => { // unit and path travel as headers: the receiver used to parse a fresh // query string per file for values that are constant per request const req = http.request( { host: device.address, port: device.port, path: `/plwl/unit/file?token=${encodeURIComponent(token)}`, method: 'PUT', headers: { 'content-type': 'application/octet-stream', 'content-length': fs.statSync(abs).size, 'x-plwl-unit': unitKey, 'x-plwl-path': rel }, timeout: 0 }, res => { res.resume() res.on('end', () => resolve({ status: res.statusCode })) } ) req.on('error', reject) const stream = fs.createReadStream(abs) stream.on('error', reject) stream.pipe(req) }) } // --- units ---------------------------------------------------------------- function appUnits(cats) { const units = [] for (const game of library.loadGames()) { const category = api.appCategory(game) if (!category || !cats.has(category)) continue const roots = [`data/icons/${game.id}`] if (category === 'apps-portable') roots.unshift(`apps/${game.id}`) units.push({ key: `app:${game.id}`, kind: 'app', id: game.id, label: game.name, roots }) } return units } function wineUnits() { const units = new Map() const add = (root, label) => { if (units.has(root)) return units.set(root, { key: `wine:${root.slice('wines/'.length)}`, kind: 'wine', id: root.slice('wines/'.length), label: label || root, roots: [root] }) } for (const wine of wines.getWines()) { const rel = toPortablePath(wine.path) const parts = String(rel).split('/') add(parts.length === 2 && parts[0] === 'wines' ? rel : `wines/${path.basename(wine.path)}`, wine.label) } for (const name of listDirs(DIRS.WINES)) add(`wines/${name}`) return [...units.values()] } function prefixUnits() { const names = new Set(prefixes.listPrefixes().prefixes.map(p => p.name)) for (const name of listDirs(DIRS.PREFIXES)) names.add(name) return [...names].map(name => ({ key: `prefix:${name}`, kind: 'prefix', id: name, label: name, roots: [`wineprefixes/${name}`] })) } function buildUnits(cats) { const units = [] if (cats.has('apps-portable') || cats.has('apps-prefix')) units.push(...appUnits(cats)) if (cats.has('wines')) units.push(...wineUnits()) if (cats.has('prefixes')) units.push(...prefixUnits()) return units } // --- registry payload ----------------------------------------------------- // Everything the app config holds travels: paths (portableGame rewrites them // to the FILES/prefix relative form both machines understand), the wine // build and prefix it runs in, args/env/dllOverrides and the play counters. function portableApp(game) { return library.portableGame(game) } function buildRegistryPayload(categories) { const cats = new Set(categories) const payload = { categories: [...cats] } if (cats.has('apps-portable') || cats.has('apps-prefix')) { payload.apps = library .loadGames() .filter(game => { const category = api.appCategory(game) return category ? cats.has(category) : false }) .map(portableApp) } if (cats.has('wines')) { // the receiver prunes its own builds down to this list and rescans payload.wines = wineUnits().map(unit => ({ name: unit.id })) } if (cats.has('defaults')) { // "Default settings": which wine everything falls back to and where a new // import lands. Opt-in on purpose - a sync of apps must not silently // change the other machine's defaults. payload.defaults = { defaultWine: (library.loadSettings().defaultWine) || 'system', defaultPrefix: prefixes.listPrefixes().default || '' } } if (cats.has('prefixes')) { const defs = prefixes.listPrefixes() payload.prefixes = { default: defs.default, prefixes: defs.prefixes.map(p => ({ name: p.name, created: p.created, locked: !!p.locked })) } } return payload } // --- sync ----------------------------------------------------------------- function newSummary(units) { return { units, uploaded: 0, skipped: 0, deleted: 0, bytes: 0, // how many apps / wine builds / prefixes this sync walked over: the // progress popup reports "3 games, 1 prefix" instead of file counts, // which mean nothing to the user items: { apps: 0, wines: 0, prefixes: 0 }, apps: { copied: 0, skipped: 0, deleted: 0 }, wines: { copied: 0, skipped: 0, deleted: 0 }, prefixes: { copied: 0, skipped: 0, deleted: 0 } } } // Every sync resolves to the same shape, successes and failures alike, so // callers can read counters/events without checking first. function syncResult(ok, error, summary, events) { const buckets = { apps: summary.apps, prefixes: summary.prefixes, wines: summary.wines } summary.progress = events.length summary.events = events.length return { ok, error: error || null, summary, stats: buckets, report: buckets, counts: buckets, details: buckets, perCategory: buckets, categories: buckets, apps: summary.apps, prefixes: summary.prefixes, wines: summary.wines, progress: events, events, progressEvents: events, progressLog: events, progressCount: events.length } } // Push the selected categories onto the peer. A rejected code comes back as // error 'bad-code', so the UI can offer to type it again. async function syncToDevice({ deviceId, device: target, code, categories, force, onProgress }) { const fail = (error, summary, events) => syncResult(false, error, summary || newSummary(0), events || []) const device = target || api.listDevices().find(d => d.id === deviceId) if (!device || !device.address || !device.port) return fail('offline') const cats = new Set(normalizeCategories(categories)) if (!cats.size) return fail('nothing-selected') // The units are known before anything is read, so how many apps / wine // builds / prefixes this sync carries is announced with the verification: // the other side can show "Apps: 0/5" from its first event. const units = buildUnits(cats) const bucketOf = kind => (kind === 'wine' ? 'wines' : kind === 'prefix' ? 'prefixes' : 'apps') const totals = { apps: 0, wines: 0, prefixes: 0 } for (const unit of units) totals[bucketOf(unit.kind)] += 1 // every progress event is kept (and handed to onProgress when given) so // callers can report on the transfer without subscribing first const events = [] const progress = data => { events.push(data) if (onProgress) onProgress(data) } let verified try { // longer than the receiver's incoming-request timeout: the user over // there gets a popup and this side waits for the click verified = await httpJson(device, 'POST', '/plwl/verify', { json: { deviceId: identity().deviceId, name: os.hostname(), code: String(code == null ? '' : code).trim().toUpperCase(), totals }, timeout: 90000 }) } catch (err) { return fail(String((err && err.message) || err)) } if (!verified.data || !verified.data.ok) { const error = (verified.data && verified.data.error) || `http-${verified.status}` // the peer rotated its code: drop the stored pairing, so the next // attempt asks for the new one instead of failing silently. A plain // 'rejected' (the user over there declined the popup) keeps the pairing // - the code was proven correct, the popup is the consent gate. if (error === 'auth-failed' || error === 'bad-code') forgetDevice(device.id || deviceId) return fail(error) } const token = verified.data.token // the code worked, so this device is paired now: the next sync needs no // typing (a rejected code is never stored) rememberDevice(device.id || deviceId, code, device.name) const summary = newSummary(units.length) const finished = { apps: 0, wines: 0, prefixes: 0 } // bytes the units planned so far will put on the wire: the popup can then // show "1.2 GB / 3.4 GB" instead of only what has already arrived let plannedBytes = 0 let itemsSeen = 0 // files uploaded/announced in PREVIOUS units: the popup's file counter is // cumulative across the whole sync (a per-unit counter resets with every // next app, which reads like progress going backwards) let allDone = 0 let allTotal = 0 // the bar tracks the PHASE being worked on (its own bytes), never a // leftover from the phase before: hashing starts from an empty bar let barBytes = 0 let barTotal = 0 const snapshot = (phase, unit, done, total) => { const data = { stage: phase, phase, kind: (unit && unit.kind) || '', id: (unit && unit.id) || '', key: (unit && unit.key) || '', unit: (unit && unit.label) || '', label: (unit && unit.label) || '', name: (unit && unit.label) || '', bytes: summary.bytes, totalBytes: plannedBytes, units: summary.units, unitsDone: itemsSeen, items: { ...summary.items }, itemsDone: { ...finished }, totals: { ...totals }, barBytes, barTotalBytes: barTotal } // only phases that actually move the file counter carry one (the copy // phase passes absolute, cumulative values); the rest omit it, so the // popup freezes the last count instead of jumping backwards while this // unit is still being scanned or compared if (Number.isFinite(done) && Number.isFinite(total)) { data.done = done data.total = total } return data } // hashing and copying walk thousands of files: the state is snapshotted on // every file but forwarded a few times per second (phase changes always // go through, so nothing on screen ever jumps backwards) const gate = typeof api.progressGate === 'function' ? api.progressGate(progress) : data => progress(data) const emit = (phase, unit, done, total, force) => gate(snapshot(phase, unit, done, total), !!force) const end = async () => { try { await httpJson(device, 'POST', `/plwl/end?token=${encodeURIComponent(token)}`, { json: {} }) } catch {} } try { for (const unit of units) { const roots = unit.roots.filter(ownedRoot) if (!roots.length) continue const kind = bucketOf(unit.kind) const bucket = summary[kind] summary.items[kind] += 1 itemsSeen += 1 barBytes = 0 barTotal = 0 emit('scan', unit, null, null, true) const entries = await buildManifest(roots, !!force, (done, total, hashedBytes, totalBytes) => { barBytes = hashedBytes barTotal = totalBytes // the file counter is not touched here: until the plan comes back // nobody knows how many of this unit's files will actually move emit('hash', unit, null, null) }) emit('plan', unit, null, null, true) const plan = await planUnit(device, token, { ...unit, roots }, entries, !!force, { totals }) if (!plan.data || !plan.data.ok) throw new Error((plan.data && plan.data.error) || `http-${plan.status}`) const upload = new Set(plan.data.upload || []) const same = new Set(plan.data.same || []) // per category every manifest entry counts: created/overwritten vs // already identical (the plan reports both) bucket.copied += entries.length - same.size bucket.skipped += same.size summary.skipped += same.size const pending = [] for (const entry of entries) { if (entry.type !== 'file') continue if (!upload.has(entry.path)) continue // identical -> the receiver keeps its copy pending.push(entry) plannedBytes += entry.size || 0 } allTotal += pending.length // a few files in flight at once: with thousands of small ones the // round trip to the other machine costs more than the bytes do // the files that were already identical never move, so the bar counts // only the bytes that do - and starts empty for the transfer phase if (pending.length) { barBytes = 0 barTotal = pending.reduce((sum, entry) => sum + (entry.size || 0), 0) emit('copy', unit, allDone, allTotal + pending.length, true) } let failed = null let done = 0 const send = async entry => { if (failed) return const res = await putFile(device, token, unit.key, entry.path, filesPath(entry.path)) if (res.status !== 200) { failed = new Error(`upload-failed-${res.status}`) return } done++ summary.uploaded++ summary.bytes += entry.size || 0 barBytes += entry.size || 0 emit('copy', unit, allDone + done, allTotal + pending.length) } if (typeof api.mapLimit === 'function') await api.mapLimit(pending, UPLOAD_CONCURRENCY, send) else for (const entry of pending) await send(entry) if (failed) throw failed emit('copy', unit, allDone + done, allTotal + pending.length, true) // this unit's files count towards the running totals only once they // are really on the wire - the counter holds there while the receiver // prunes and the next unit is scanned/compared allDone += pending.length allTotal += pending.length const commit = await httpJson(device, 'POST', `/plwl/unit/commit?token=${encodeURIComponent(token)}&unit=${encodeURIComponent(unit.key)}`, { json: {} }) if (commit.data && commit.data.deleted) { summary.deleted += commit.data.deleted bucket.deleted += commit.data.deleted } finished[kind] += 1 // a finished unit leaves a finished bar behind, even one that sent none barTotal = barTotal || 1 barBytes = barTotal emit('commit', unit, null, null, true) } emit('registry', null, null, null, true) const applied = await httpJson(device, 'POST', `/plwl/apply?token=${encodeURIComponent(token)}`, { json: buildRegistryPayload([...cats]), timeout: 60000 }) if (!applied.data || !applied.data.ok) throw new Error((applied.data && applied.data.error) || `http-${applied.status}`) for (const key of ['apps', 'wines', 'prefixes']) { const deleted = applied.data[key] && applied.data[key].deleted if (deleted) summary[key].deleted += deleted } if (applied.data.deleted) summary.deleted += applied.data.deleted emit('done', null, allDone, allTotal, true) await end() } catch (err) { await end() return fail(String((err && err.message) || err), summary, events) } return syncResult(true, null, summary, events) } return { syncToDevice, buildUnits, buildRegistryPayload, CATEGORIES } } module.exports = createPusher