code refactor & sync bugfixes

src/hashcache.js

@@ -0,0 +1,161 @@
+'use strict'
+
+// Content hashing for the LAN sync, plus the per-file digest cache.
+//
+// Syncing a library means hashing tens of thousands of files, so the digest of
+// every file is remembered on disk (files/data/lansync-hashes.json), keyed by
+// its FILES-relative path with the size and mtime it was computed from: a file
+// whose size and mtime are unchanged is never read again, only stat'ed. The
+// cache is machine-local and never travels in a sync.
+
+const crypto = require('crypto')
+const fs = require('fs')
+const path = require('path')
+
+// BLAKE2b resists collisions like SHA-2 but is noticeably faster. Older
+// runtimes fall back to SHA-2.
+const HASH_ALGO = (() => {
+  for (const algo of ['blake2b512', 'sha512', 'sha256']) {
+    try {
+      crypto.createHash(algo)
+      return algo
+    } catch {}
+  }
+  return 'sha256'
+})()
+
+// Streaming through one callback instead of accumulating 'data' chunks keeps
+// memory flat for huge files, and a small highWaterMark gives the read pump
+// real backpressure. Digest speed does not care about the chunk size.
+function hashFile(file, onBytes) {
+  return new Promise((resolve, reject) => {
+    const hash = crypto.createHash(HASH_ALGO)
+    const stream = fs.createReadStream(file, { highWaterMark: 64 * 1024 })
+    stream.on('error', reject)
+    if (onBytes) stream.on('data', chunk => onBytes(chunk.length))
+    stream.on('data', chunk => hash.update(chunk))
+    stream.on('end', () => resolve(hash.digest('hex')))
+    stream.resume()
+  })
+}
+
+// A quarter million entries is a whole game library; beyond that, entries the
+// longest time untouched are dropped first.
+const HASH_CACHE_MAX = 250000
+// Mid-sync saves are throttled to this (the file can hold tens of megabytes);
+// callers that must not lose entries (sync end, shutdown) pass force=true.
+const SAVE_MIN_MS = 60000
+
+function createHashCache(dataDir) {
+  const FILE = path.join(dataDir, 'lansync-hashes.json')
+  let cache = null
+  let dirty = false
+  let savedAt = 0
+  const stats = { hits: 0, misses: 0 }
+
+  function load() {
+    if (cache) return cache
+    try {
+      const stored = JSON.parse(fs.readFileSync(FILE, 'utf8'))
+      cache = stored && typeof stored === 'object' && stored.files && typeof stored.files === 'object' ? stored.files : {}
+    } catch {
+      cache = {}
+    }
+    return cache
+  }
+
+  // Compact JSON (the entries are arrays) keeps a full cache at tens of
+  // milliseconds instead of the seconds pretty-printed JSON needs.
+  function save(force) {
+    if (!dirty || !cache) return
+    if (!force && Date.now() - savedAt < SAVE_MIN_MS) return
+    const names = Object.keys(cache)
+    if (names.length > HASH_CACHE_MAX) {
+      names.sort((a, b) => (cache[b][3] || 0) - (cache[a][3] || 0))
+      const trimmed = {}
+      for (const name of names.slice(0, HASH_CACHE_MAX)) trimmed[name] = cache[name]
+      cache = trimmed
+    }
+    dirty = false
+    savedAt = Date.now()
+    try {
+      fs.mkdirSync(path.dirname(FILE), { recursive: true })
+      const tmp = `${FILE}.tmp`
+      fs.writeFileSync(tmp, JSON.stringify({ v: 1, algo: HASH_ALGO, files: cache }))
+      fs.renameSync(tmp, FILE)
+    } catch {}
+  }
+
+  // The remembered digest of a file, or null when it has to be read again.
+  // A hit only marks the cache dirty when its timestamp is actually stale:
+  // marking it on every lookup would force a full rewrite of the file per
+  // sync even though nothing a second sync could reuse has changed.
+  function hit(rel, size, mtimeMs) {
+    const known = load()[rel]
+    if (!known || known[0] !== size || known[1] !== mtimeMs || typeof known[2] !== 'string') {
+      stats.misses++
+      return null
+    }
+    stats.hits++
+    if (Date.now() - (known[3] || 0) > 86400000) {
+      known[3] = Date.now()
+      dirty = true
+    }
+    return known[2]
+  }
+
+  function remember(rel, size, mtimeMs, hash) {
+    load()[rel] = [size, mtimeMs, hash, Date.now()]
+    dirty = true
+  }
+
+  // Digest of a file we already stat'ed: size and mtime unchanged means the
+  // bytes are the same, and the digest from an earlier sync still describes
+  // them. Otherwise the file is read (progress through onBytes) and stored.
+  async function cachedHash(rel, abs, size, mtimeMs, onBytes) {
+    const known = hit(rel, size, mtimeMs)
+    if (known) return known
+    const hash = await hashFile(abs, onBytes)
+    if (hash) remember(rel, size, mtimeMs, hash)
+    return hash
+  }
+
+  // The user can drop the cache from the settings view; a corrupted or stale
+  // file is also replaced wholesale this way.
+  function clear() {
+    cache = null
+    dirty = false
+    savedAt = 0
+    stats.hits = 0
+    stats.misses = 0
+    try {
+      fs.rmSync(FILE, { force: true })
+      fs.rmSync(`${FILE}.tmp`, { force: true })
+    } catch {}
+  }
+
+  // Dropping the in-memory copy makes the next lookup re-read the file from
+  // disk - what tests use to prove a file was hashed instead of remembered.
+  function forget() {
+    cache = null
+    dirty = false
+  }
+
+  return {
+    HASH_ALGO,
+    hashFile,
+    hit,
+    remember,
+    cachedHash,
+    save,
+    clear,
+    forget,
+    stats: () => ({ ...stats, size: cache ? Object.keys(cache).length : 0, algo: HASH_ALGO }),
+    resetStats: () => {
+      stats.hits = 0
+      stats.misses = 0
+    }
+  }
+}
+
+module.exports = { createHashCache, hashFile, HASH_ALGO }