Update gitignore (sorry)

This commit is contained in:
olcxja 2026-05-10 14:02:17 +02:00
commit cca8b02fea
6604 changed files with 1219661 additions and 4 deletions

View file

@ -0,0 +1,33 @@
/// <reference types="node" />
import { Writable } from "stream";
import { Operation } from "./downloadPlanBuilder";
export interface PartListDataTask {
readonly oldFileFd: number;
readonly tasks: Array<Operation>;
readonly start: number;
readonly end: number;
}
export declare function copyData(task: Operation, out: Writable, oldFileFd: number, reject: (error: Error) => void, resolve: () => void): void;
export declare class DataSplitter extends Writable {
private readonly out;
private readonly options;
private readonly partIndexToTaskIndex;
private readonly partIndexToLength;
private readonly finishHandler;
partIndex: number;
private headerListBuffer;
private readState;
private ignoreByteCount;
private remainingPartDataCount;
private readonly boundaryLength;
constructor(out: Writable, options: PartListDataTask, partIndexToTaskIndex: Map<number, number>, boundary: string, partIndexToLength: Array<number>, finishHandler: () => any);
get isFinished(): boolean;
_write(data: Buffer, encoding: string, callback: (error?: Error) => void): void;
private handleData;
private copyExistingData;
private searchHeaderListEnd;
private actualPartLength;
private onPartEnd;
private processPartStarted;
private processPartData;
}

View file

@ -0,0 +1,202 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.DataSplitter = exports.copyData = void 0;
const builder_util_runtime_1 = require("builder-util-runtime");
const fs_1 = require("fs");
const stream_1 = require("stream");
const downloadPlanBuilder_1 = require("./downloadPlanBuilder");
const DOUBLE_CRLF = Buffer.from("\r\n\r\n");
var ReadState;
(function (ReadState) {
ReadState[ReadState["INIT"] = 0] = "INIT";
ReadState[ReadState["HEADER"] = 1] = "HEADER";
ReadState[ReadState["BODY"] = 2] = "BODY";
})(ReadState || (ReadState = {}));
function copyData(task, out, oldFileFd, reject, resolve) {
const readStream = fs_1.createReadStream("", {
fd: oldFileFd,
autoClose: false,
start: task.start,
// end is inclusive
end: task.end - 1,
});
readStream.on("error", reject);
readStream.once("end", resolve);
readStream.pipe(out, {
end: false,
});
}
exports.copyData = copyData;
class DataSplitter extends stream_1.Writable {
constructor(out, options, partIndexToTaskIndex, boundary, partIndexToLength, finishHandler) {
super();
this.out = out;
this.options = options;
this.partIndexToTaskIndex = partIndexToTaskIndex;
this.partIndexToLength = partIndexToLength;
this.finishHandler = finishHandler;
this.partIndex = -1;
this.headerListBuffer = null;
this.readState = ReadState.INIT;
this.ignoreByteCount = 0;
this.remainingPartDataCount = 0;
this.actualPartLength = 0;
this.boundaryLength = boundary.length + 4; /* size of \r\n-- */
// first chunk doesn't start with \r\n
this.ignoreByteCount = this.boundaryLength - 2;
}
get isFinished() {
return this.partIndex === this.partIndexToLength.length;
}
// noinspection JSUnusedGlobalSymbols
_write(data, encoding, callback) {
if (this.isFinished) {
console.error(`Trailing ignored data: ${data.length} bytes`);
return;
}
this.handleData(data).then(callback).catch(callback);
}
async handleData(chunk) {
let start = 0;
if (this.ignoreByteCount !== 0 && this.remainingPartDataCount !== 0) {
throw builder_util_runtime_1.newError("Internal error", "ERR_DATA_SPLITTER_BYTE_COUNT_MISMATCH");
}
if (this.ignoreByteCount > 0) {
const toIgnore = Math.min(this.ignoreByteCount, chunk.length);
this.ignoreByteCount -= toIgnore;
start = toIgnore;
}
else if (this.remainingPartDataCount > 0) {
const toRead = Math.min(this.remainingPartDataCount, chunk.length);
this.remainingPartDataCount -= toRead;
await this.processPartData(chunk, 0, toRead);
start = toRead;
}
if (start === chunk.length) {
return;
}
if (this.readState === ReadState.HEADER) {
const headerListEnd = this.searchHeaderListEnd(chunk, start);
if (headerListEnd === -1) {
return;
}
start = headerListEnd;
this.readState = ReadState.BODY;
// header list is ignored, we don't need it
this.headerListBuffer = null;
}
while (true) {
if (this.readState === ReadState.BODY) {
this.readState = ReadState.INIT;
}
else {
this.partIndex++;
let taskIndex = this.partIndexToTaskIndex.get(this.partIndex);
if (taskIndex == null) {
if (this.isFinished) {
taskIndex = this.options.end;
}
else {
throw builder_util_runtime_1.newError("taskIndex is null", "ERR_DATA_SPLITTER_TASK_INDEX_IS_NULL");
}
}
const prevTaskIndex = this.partIndex === 0 ? this.options.start : this.partIndexToTaskIndex.get(this.partIndex - 1) + 1; /* prev part is download, next maybe copy */
if (prevTaskIndex < taskIndex) {
await this.copyExistingData(prevTaskIndex, taskIndex);
}
else if (prevTaskIndex > taskIndex) {
throw builder_util_runtime_1.newError("prevTaskIndex must be < taskIndex", "ERR_DATA_SPLITTER_TASK_INDEX_ASSERT_FAILED");
}
if (this.isFinished) {
this.onPartEnd();
this.finishHandler();
return;
}
start = this.searchHeaderListEnd(chunk, start);
if (start === -1) {
this.readState = ReadState.HEADER;
return;
}
}
const partLength = this.partIndexToLength[this.partIndex];
const end = start + partLength;
const effectiveEnd = Math.min(end, chunk.length);
await this.processPartStarted(chunk, start, effectiveEnd);
this.remainingPartDataCount = partLength - (effectiveEnd - start);
if (this.remainingPartDataCount > 0) {
return;
}
start = end + this.boundaryLength;
if (start >= chunk.length) {
this.ignoreByteCount = this.boundaryLength - (chunk.length - end);
return;
}
}
}
copyExistingData(index, end) {
return new Promise((resolve, reject) => {
const w = () => {
if (index === end) {
resolve();
return;
}
const task = this.options.tasks[index];
if (task.kind !== downloadPlanBuilder_1.OperationKind.COPY) {
reject(new Error("Task kind must be COPY"));
return;
}
copyData(task, this.out, this.options.oldFileFd, reject, () => {
index++;
w();
});
};
w();
});
}
searchHeaderListEnd(chunk, readOffset) {
const headerListEnd = chunk.indexOf(DOUBLE_CRLF, readOffset);
if (headerListEnd !== -1) {
return headerListEnd + DOUBLE_CRLF.length;
}
// not all headers data were received, save to buffer
const partialChunk = readOffset === 0 ? chunk : chunk.slice(readOffset);
if (this.headerListBuffer == null) {
this.headerListBuffer = partialChunk;
}
else {
this.headerListBuffer = Buffer.concat([this.headerListBuffer, partialChunk]);
}
return -1;
}
onPartEnd() {
const expectedLength = this.partIndexToLength[this.partIndex - 1];
if (this.actualPartLength !== expectedLength) {
throw builder_util_runtime_1.newError(`Expected length: ${expectedLength} differs from actual: ${this.actualPartLength}`, "ERR_DATA_SPLITTER_LENGTH_MISMATCH");
}
this.actualPartLength = 0;
}
processPartStarted(data, start, end) {
if (this.partIndex !== 0) {
this.onPartEnd();
}
return this.processPartData(data, start, end);
}
processPartData(data, start, end) {
this.actualPartLength += end - start;
const out = this.out;
if (out.write(start === 0 && data.length === end ? data : data.slice(start, end))) {
return Promise.resolve();
}
else {
return new Promise((resolve, reject) => {
out.on("error", reject);
out.once("drain", () => {
out.removeListener("error", reject);
resolve();
});
});
}
}
}
exports.DataSplitter = DataSplitter;
//# sourceMappingURL=DataSplitter.js.map

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1,31 @@
/// <reference types="node" />
import { BlockMapDataHolder, HttpExecutor } from "builder-util-runtime";
import { BlockMap } from "builder-util-runtime/out/blockMapApi";
import { OutgoingHttpHeaders, RequestOptions } from "http";
import { ProgressInfo, CancellationToken } from "builder-util-runtime";
import { Logger } from "../main";
import { URL } from "url";
export interface DifferentialDownloaderOptions {
readonly oldFile: string;
readonly newUrl: URL;
readonly logger: Logger;
readonly newFile: string;
readonly requestHeaders: OutgoingHttpHeaders | null;
readonly isUseMultipleRangeRequest?: boolean;
readonly cancellationToken: CancellationToken;
onProgress?: (progress: ProgressInfo) => void;
}
export declare abstract class DifferentialDownloader {
protected readonly blockAwareFileInfo: BlockMapDataHolder;
readonly httpExecutor: HttpExecutor<any>;
readonly options: DifferentialDownloaderOptions;
fileMetadataBuffer: Buffer | null;
private readonly logger;
constructor(blockAwareFileInfo: BlockMapDataHolder, httpExecutor: HttpExecutor<any>, options: DifferentialDownloaderOptions);
createRequestOptions(): RequestOptions;
protected doDownload(oldBlockMap: BlockMap, newBlockMap: BlockMap): Promise<any>;
private downloadFile;
private doDownloadFile;
protected readRemoteBytes(start: number, endInclusive: number): Promise<Buffer>;
private request;
}

View file

@ -0,0 +1,261 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.DifferentialDownloader = void 0;
const builder_util_runtime_1 = require("builder-util-runtime");
const fs_extra_1 = require("fs-extra");
const fs_1 = require("fs");
const DataSplitter_1 = require("./DataSplitter");
const url_1 = require("url");
const downloadPlanBuilder_1 = require("./downloadPlanBuilder");
const multipleRangeDownloader_1 = require("./multipleRangeDownloader");
const ProgressDifferentialDownloadCallbackTransform_1 = require("./ProgressDifferentialDownloadCallbackTransform");
class DifferentialDownloader {
// noinspection TypeScriptAbstractClassConstructorCanBeMadeProtected
constructor(blockAwareFileInfo, httpExecutor, options) {
this.blockAwareFileInfo = blockAwareFileInfo;
this.httpExecutor = httpExecutor;
this.options = options;
this.fileMetadataBuffer = null;
this.logger = options.logger;
}
createRequestOptions() {
const result = {
headers: {
...this.options.requestHeaders,
accept: "*/*",
},
};
builder_util_runtime_1.configureRequestUrl(this.options.newUrl, result);
// user-agent, cache-control and other common options
builder_util_runtime_1.configureRequestOptions(result);
return result;
}
doDownload(oldBlockMap, newBlockMap) {
// we don't check other metadata like compressionMethod - generic check that it is make sense to differentially update is suitable for it
if (oldBlockMap.version !== newBlockMap.version) {
throw new Error(`version is different (${oldBlockMap.version} - ${newBlockMap.version}), full download is required`);
}
const logger = this.logger;
const operations = downloadPlanBuilder_1.computeOperations(oldBlockMap, newBlockMap, logger);
if (logger.debug != null) {
logger.debug(JSON.stringify(operations, null, 2));
}
let downloadSize = 0;
let copySize = 0;
for (const operation of operations) {
const length = operation.end - operation.start;
if (operation.kind === downloadPlanBuilder_1.OperationKind.DOWNLOAD) {
downloadSize += length;
}
else {
copySize += length;
}
}
const newSize = this.blockAwareFileInfo.size;
if (downloadSize + copySize + (this.fileMetadataBuffer == null ? 0 : this.fileMetadataBuffer.length) !== newSize) {
throw new Error(`Internal error, size mismatch: downloadSize: ${downloadSize}, copySize: ${copySize}, newSize: ${newSize}`);
}
logger.info(`Full: ${formatBytes(newSize)}, To download: ${formatBytes(downloadSize)} (${Math.round(downloadSize / (newSize / 100))}%)`);
return this.downloadFile(operations);
}
downloadFile(tasks) {
const fdList = [];
const closeFiles = () => {
return Promise.all(fdList.map(openedFile => {
return fs_extra_1.close(openedFile.descriptor).catch(e => {
this.logger.error(`cannot close file "${openedFile.path}": ${e}`);
});
}));
};
return this.doDownloadFile(tasks, fdList)
.then(closeFiles)
.catch(e => {
// then must be after catch here (since then always throws error)
return closeFiles()
.catch(closeFilesError => {
// closeFiles never throw error, but just to be sure
try {
this.logger.error(`cannot close files: ${closeFilesError}`);
}
catch (errorOnLog) {
try {
console.error(errorOnLog);
}
catch (ignored) {
// ok, give up and ignore error
}
}
throw e;
})
.then(() => {
throw e;
});
});
}
async doDownloadFile(tasks, fdList) {
const oldFileFd = await fs_extra_1.open(this.options.oldFile, "r");
fdList.push({ descriptor: oldFileFd, path: this.options.oldFile });
const newFileFd = await fs_extra_1.open(this.options.newFile, "w");
fdList.push({ descriptor: newFileFd, path: this.options.newFile });
const fileOut = fs_1.createWriteStream(this.options.newFile, { fd: newFileFd });
await new Promise((resolve, reject) => {
const streams = [];
// Create our download info transformer if we have one
let downloadInfoTransform = undefined;
if (!this.options.isUseMultipleRangeRequest && this.options.onProgress) {
// TODO: Does not support multiple ranges (someone feel free to PR this!)
const expectedByteCounts = [];
let grandTotalBytes = 0;
for (const task of tasks) {
if (task.kind === downloadPlanBuilder_1.OperationKind.DOWNLOAD) {
expectedByteCounts.push(task.end - task.start);
grandTotalBytes += task.end - task.start;
}
}
const progressDifferentialDownloadInfo = {
expectedByteCounts: expectedByteCounts,
grandTotal: grandTotalBytes,
};
downloadInfoTransform = new ProgressDifferentialDownloadCallbackTransform_1.ProgressDifferentialDownloadCallbackTransform(progressDifferentialDownloadInfo, this.options.cancellationToken, this.options.onProgress);
streams.push(downloadInfoTransform);
}
const digestTransform = new builder_util_runtime_1.DigestTransform(this.blockAwareFileInfo.sha512);
// to simply debug, do manual validation to allow file to be fully written
digestTransform.isValidateOnEnd = false;
streams.push(digestTransform);
// noinspection JSArrowFunctionCanBeReplacedWithShorthand
fileOut.on("finish", () => {
;
fileOut.close(() => {
// remove from fd list because closed successfully
fdList.splice(1, 1);
try {
digestTransform.validate();
}
catch (e) {
reject(e);
return;
}
resolve(undefined);
});
});
streams.push(fileOut);
let lastStream = null;
for (const stream of streams) {
stream.on("error", reject);
if (lastStream == null) {
lastStream = stream;
}
else {
lastStream = lastStream.pipe(stream);
}
}
const firstStream = streams[0];
let w;
if (this.options.isUseMultipleRangeRequest) {
w = multipleRangeDownloader_1.executeTasksUsingMultipleRangeRequests(this, tasks, firstStream, oldFileFd, reject);
w(0);
return;
}
let downloadOperationCount = 0;
let actualUrl = null;
this.logger.info(`Differential download: ${this.options.newUrl}`);
const requestOptions = this.createRequestOptions();
requestOptions.redirect = "manual";
w = (index) => {
var _a, _b;
if (index >= tasks.length) {
if (this.fileMetadataBuffer != null) {
firstStream.write(this.fileMetadataBuffer);
}
firstStream.end();
return;
}
const operation = tasks[index++];
if (operation.kind === downloadPlanBuilder_1.OperationKind.COPY) {
// We are copying, let's not send status updates to the UI
if (downloadInfoTransform) {
downloadInfoTransform.beginFileCopy();
}
DataSplitter_1.copyData(operation, firstStream, oldFileFd, reject, () => w(index));
return;
}
const range = `bytes=${operation.start}-${operation.end - 1}`;
requestOptions.headers.range = range;
(_b = (_a = this.logger) === null || _a === void 0 ? void 0 : _a.debug) === null || _b === void 0 ? void 0 : _b.call(_a, `download range: ${range}`);
// We are starting to download
if (downloadInfoTransform) {
downloadInfoTransform.beginRangeDownload();
}
const request = this.httpExecutor.createRequest(requestOptions, response => {
// Electron net handles redirects automatically, our NodeJS test server doesn't use redirects - so, we don't check 3xx codes.
if (response.statusCode >= 400) {
reject(builder_util_runtime_1.createHttpError(response));
}
response.pipe(firstStream, {
end: false,
});
response.once("end", () => {
// Pass on that we are downloading a segment
if (downloadInfoTransform) {
downloadInfoTransform.endRangeDownload();
}
if (++downloadOperationCount === 100) {
downloadOperationCount = 0;
setTimeout(() => w(index), 1000);
}
else {
w(index);
}
});
});
request.on("redirect", (statusCode, method, redirectUrl) => {
this.logger.info(`Redirect to ${removeQuery(redirectUrl)}`);
actualUrl = redirectUrl;
builder_util_runtime_1.configureRequestUrl(new url_1.URL(actualUrl), requestOptions);
request.followRedirect();
});
this.httpExecutor.addErrorAndTimeoutHandlers(request, reject);
request.end();
};
w(0);
});
}
async readRemoteBytes(start, endInclusive) {
const buffer = Buffer.allocUnsafe(endInclusive + 1 - start);
const requestOptions = this.createRequestOptions();
requestOptions.headers.range = `bytes=${start}-${endInclusive}`;
let position = 0;
await this.request(requestOptions, chunk => {
chunk.copy(buffer, position);
position += chunk.length;
});
if (position !== buffer.length) {
throw new Error(`Received data length ${position} is not equal to expected ${buffer.length}`);
}
return buffer;
}
request(requestOptions, dataHandler) {
return new Promise((resolve, reject) => {
const request = this.httpExecutor.createRequest(requestOptions, response => {
if (!multipleRangeDownloader_1.checkIsRangesSupported(response, reject)) {
return;
}
response.on("data", dataHandler);
response.on("end", () => resolve());
});
this.httpExecutor.addErrorAndTimeoutHandlers(request, reject);
request.end();
});
}
}
exports.DifferentialDownloader = DifferentialDownloader;
function formatBytes(value, symbol = " KB") {
return new Intl.NumberFormat("en").format((value / 1024).toFixed(2)) + symbol;
}
// safety
function removeQuery(url) {
const index = url.indexOf("?");
return index < 0 ? url : url.substring(0, index);
}
//# sourceMappingURL=DifferentialDownloader.js.map

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1,4 @@
import { DifferentialDownloader } from "./DifferentialDownloader";
export declare class FileWithEmbeddedBlockMapDifferentialDownloader extends DifferentialDownloader {
download(): Promise<void>;
}

View file

@ -0,0 +1,37 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.FileWithEmbeddedBlockMapDifferentialDownloader = void 0;
const fs_extra_1 = require("fs-extra");
const DifferentialDownloader_1 = require("./DifferentialDownloader");
const zlib_1 = require("zlib");
class FileWithEmbeddedBlockMapDifferentialDownloader extends DifferentialDownloader_1.DifferentialDownloader {
async download() {
const packageInfo = this.blockAwareFileInfo;
const fileSize = packageInfo.size;
const offset = fileSize - (packageInfo.blockMapSize + 4);
this.fileMetadataBuffer = await this.readRemoteBytes(offset, fileSize - 1);
const newBlockMap = readBlockMap(this.fileMetadataBuffer.slice(0, this.fileMetadataBuffer.length - 4));
await this.doDownload(await readEmbeddedBlockMapData(this.options.oldFile), newBlockMap);
}
}
exports.FileWithEmbeddedBlockMapDifferentialDownloader = FileWithEmbeddedBlockMapDifferentialDownloader;
function readBlockMap(data) {
return JSON.parse(zlib_1.inflateRawSync(data).toString());
}
async function readEmbeddedBlockMapData(file) {
const fd = await fs_extra_1.open(file, "r");
try {
const fileSize = (await fs_extra_1.fstat(fd)).size;
const sizeBuffer = Buffer.allocUnsafe(4);
await fs_extra_1.read(fd, sizeBuffer, 0, sizeBuffer.length, fileSize - sizeBuffer.length);
const dataBuffer = Buffer.allocUnsafe(sizeBuffer.readUInt32BE(0));
await fs_extra_1.read(fd, dataBuffer, 0, dataBuffer.length, fileSize - sizeBuffer.length - dataBuffer.length);
await fs_extra_1.close(fd);
return readBlockMap(dataBuffer);
}
catch (e) {
await fs_extra_1.close(fd);
throw e;
}
}
//# sourceMappingURL=FileWithEmbeddedBlockMapDifferentialDownloader.js.map

View file

@ -0,0 +1 @@
{"version":3,"file":"FileWithEmbeddedBlockMapDifferentialDownloader.js","sourceRoot":"","sources":["../../src/differentialDownloader/FileWithEmbeddedBlockMapDifferentialDownloader.ts"],"names":[],"mappings":";;;AACA,uCAAmD;AACnD,qEAAiE;AACjE,+BAAqC;AAErC,MAAa,8CAA+C,SAAQ,+CAAsB;IACxF,KAAK,CAAC,QAAQ;QACZ,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAA;QAC3C,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAK,CAAA;QAClC,MAAM,MAAM,GAAG,QAAQ,GAAG,CAAC,WAAW,CAAC,YAAa,GAAG,CAAC,CAAC,CAAA;QACzD,IAAI,CAAC,kBAAkB,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAA;QAC1E,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAA;QACtG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,wBAAwB,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,WAAW,CAAC,CAAA;IAC1F,CAAC;CACF;AATD,wGASC;AAED,SAAS,YAAY,CAAC,IAAY;IAChC,OAAO,IAAI,CAAC,KAAK,CAAC,qBAAc,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAA;AACpD,CAAC;AAED,KAAK,UAAU,wBAAwB,CAAC,IAAY;IAClD,MAAM,EAAE,GAAG,MAAM,eAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;IAChC,IAAI;QACF,MAAM,QAAQ,GAAG,CAAC,MAAM,gBAAK,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;QACvC,MAAM,UAAU,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAA;QACxC,MAAM,eAAI,CAAC,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,UAAU,CAAC,MAAM,EAAE,QAAQ,GAAG,UAAU,CAAC,MAAM,CAAC,CAAA;QAE9E,MAAM,UAAU,GAAG,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAA;QACjE,MAAM,eAAI,CAAC,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,UAAU,CAAC,MAAM,EAAE,QAAQ,GAAG,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAA;QAClG,MAAM,gBAAK,CAAC,EAAE,CAAC,CAAA;QAEf,OAAO,YAAY,CAAC,UAAU,CAAC,CAAA;KAChC;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,gBAAK,CAAC,EAAE,CAAC,CAAA;QACf,MAAM,CAAC,CAAA;KACR;AACH,CAAC","sourcesContent":["import { BlockMap } from \"builder-util-runtime/out/blockMapApi\"\nimport { close, fstat, open, read } from \"fs-extra\"\nimport { DifferentialDownloader } from \"./DifferentialDownloader\"\nimport { inflateRawSync } from \"zlib\"\n\nexport class FileWithEmbeddedBlockMapDifferentialDownloader extends DifferentialDownloader {\n async download(): Promise<void> {\n const packageInfo = this.blockAwareFileInfo\n const fileSize = packageInfo.size!\n const offset = fileSize - (packageInfo.blockMapSize! + 4)\n this.fileMetadataBuffer = await this.readRemoteBytes(offset, fileSize - 1)\n const newBlockMap = readBlockMap(this.fileMetadataBuffer.slice(0, this.fileMetadataBuffer.length - 4))\n await this.doDownload(await readEmbeddedBlockMapData(this.options.oldFile), newBlockMap)\n }\n}\n\nfunction readBlockMap(data: Buffer): BlockMap {\n return JSON.parse(inflateRawSync(data).toString())\n}\n\nasync function readEmbeddedBlockMapData(file: string): Promise<BlockMap> {\n const fd = await open(file, \"r\")\n try {\n const fileSize = (await fstat(fd)).size\n const sizeBuffer = Buffer.allocUnsafe(4)\n await read(fd, sizeBuffer, 0, sizeBuffer.length, fileSize - sizeBuffer.length)\n\n const dataBuffer = Buffer.allocUnsafe(sizeBuffer.readUInt32BE(0))\n await read(fd, dataBuffer, 0, dataBuffer.length, fileSize - sizeBuffer.length - dataBuffer.length)\n await close(fd)\n\n return readBlockMap(dataBuffer)\n } catch (e) {\n await close(fd)\n throw e\n }\n}\n"]}

View file

@ -0,0 +1,5 @@
import { BlockMap } from "builder-util-runtime/out/blockMapApi";
import { DifferentialDownloader } from "./DifferentialDownloader";
export declare class GenericDifferentialDownloader extends DifferentialDownloader {
download(oldBlockMap: BlockMap, newBlockMap: BlockMap): Promise<any>;
}

View file

@ -0,0 +1,11 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.GenericDifferentialDownloader = void 0;
const DifferentialDownloader_1 = require("./DifferentialDownloader");
class GenericDifferentialDownloader extends DifferentialDownloader_1.DifferentialDownloader {
download(oldBlockMap, newBlockMap) {
return this.doDownload(oldBlockMap, newBlockMap);
}
}
exports.GenericDifferentialDownloader = GenericDifferentialDownloader;
//# sourceMappingURL=GenericDifferentialDownloader.js.map

View file

@ -0,0 +1 @@
{"version":3,"file":"GenericDifferentialDownloader.js","sourceRoot":"","sources":["../../src/differentialDownloader/GenericDifferentialDownloader.ts"],"names":[],"mappings":";;;AACA,qEAAiE;AAEjE,MAAa,6BAA8B,SAAQ,+CAAsB;IACvE,QAAQ,CAAC,WAAqB,EAAE,WAAqB;QACnD,OAAO,IAAI,CAAC,UAAU,CAAC,WAAW,EAAE,WAAW,CAAC,CAAA;IAClD,CAAC;CACF;AAJD,sEAIC","sourcesContent":["import { BlockMap } from \"builder-util-runtime/out/blockMapApi\"\nimport { DifferentialDownloader } from \"./DifferentialDownloader\"\n\nexport class GenericDifferentialDownloader extends DifferentialDownloader {\n download(oldBlockMap: BlockMap, newBlockMap: BlockMap): Promise<any> {\n return this.doDownload(oldBlockMap, newBlockMap)\n }\n}\n"]}

View file

@ -0,0 +1,32 @@
/// <reference types="node" />
import { Transform } from "stream";
import { CancellationToken } from "builder-util-runtime";
export interface ProgressInfo {
total: number;
delta: number;
transferred: number;
percent: number;
bytesPerSecond: number;
}
export interface ProgressDifferentialDownloadInfo {
expectedByteCounts: Array<number>;
grandTotal: number;
}
export declare class ProgressDifferentialDownloadCallbackTransform extends Transform {
private readonly progressDifferentialDownloadInfo;
private readonly cancellationToken;
private readonly onProgress;
private start;
private transferred;
private delta;
private expectedBytes;
private index;
private operationType;
private nextUpdate;
constructor(progressDifferentialDownloadInfo: ProgressDifferentialDownloadInfo, cancellationToken: CancellationToken, onProgress: (info: ProgressInfo) => any);
_transform(chunk: any, encoding: string, callback: any): void;
beginFileCopy(): void;
beginRangeDownload(): void;
endRangeDownload(): void;
_flush(callback: any): void;
}

View file

@ -0,0 +1,90 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.ProgressDifferentialDownloadCallbackTransform = void 0;
const stream_1 = require("stream");
var OperationKind;
(function (OperationKind) {
OperationKind[OperationKind["COPY"] = 0] = "COPY";
OperationKind[OperationKind["DOWNLOAD"] = 1] = "DOWNLOAD";
})(OperationKind || (OperationKind = {}));
class ProgressDifferentialDownloadCallbackTransform extends stream_1.Transform {
constructor(progressDifferentialDownloadInfo, cancellationToken, onProgress) {
super();
this.progressDifferentialDownloadInfo = progressDifferentialDownloadInfo;
this.cancellationToken = cancellationToken;
this.onProgress = onProgress;
this.start = Date.now();
this.transferred = 0;
this.delta = 0;
this.expectedBytes = 0;
this.index = 0;
this.operationType = OperationKind.COPY;
this.nextUpdate = this.start + 1000;
}
_transform(chunk, encoding, callback) {
if (this.cancellationToken.cancelled) {
callback(new Error("cancelled"), null);
return;
}
// Don't send progress update when copying from disk
if (this.operationType == OperationKind.COPY) {
callback(null, chunk);
return;
}
this.transferred += chunk.length;
this.delta += chunk.length;
const now = Date.now();
if (now >= this.nextUpdate &&
this.transferred !== this.expectedBytes /* will be emitted by endRangeDownload() */ &&
this.transferred !== this.progressDifferentialDownloadInfo.grandTotal /* will be emitted on _flush */) {
this.nextUpdate = now + 1000;
this.onProgress({
total: this.progressDifferentialDownloadInfo.grandTotal,
delta: this.delta,
transferred: this.transferred,
percent: (this.transferred / this.progressDifferentialDownloadInfo.grandTotal) * 100,
bytesPerSecond: Math.round(this.transferred / ((now - this.start) / 1000)),
});
this.delta = 0;
}
callback(null, chunk);
}
beginFileCopy() {
this.operationType = OperationKind.COPY;
}
beginRangeDownload() {
this.operationType = OperationKind.DOWNLOAD;
this.expectedBytes += this.progressDifferentialDownloadInfo.expectedByteCounts[this.index++];
}
endRangeDownload() {
// _flush() will doour final 100%
if (this.transferred !== this.progressDifferentialDownloadInfo.grandTotal) {
this.onProgress({
total: this.progressDifferentialDownloadInfo.grandTotal,
delta: this.delta,
transferred: this.transferred,
percent: (this.transferred / this.progressDifferentialDownloadInfo.grandTotal) * 100,
bytesPerSecond: Math.round(this.transferred / ((Date.now() - this.start) / 1000)),
});
}
}
// Called when we are 100% done with the connection/download
_flush(callback) {
if (this.cancellationToken.cancelled) {
callback(new Error("cancelled"));
return;
}
this.onProgress({
total: this.progressDifferentialDownloadInfo.grandTotal,
delta: this.delta,
transferred: this.transferred,
percent: 100,
bytesPerSecond: Math.round(this.transferred / ((Date.now() - this.start) / 1000)),
});
this.delta = 0;
this.transferred = 0;
callback(null);
}
}
exports.ProgressDifferentialDownloadCallbackTransform = ProgressDifferentialDownloadCallbackTransform;
//# sourceMappingURL=ProgressDifferentialDownloadCallbackTransform.js.map

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1,12 @@
import { BlockMap } from "builder-util-runtime/out/blockMapApi";
import { Logger } from "../main";
export declare enum OperationKind {
COPY = 0,
DOWNLOAD = 1
}
export interface Operation {
kind: OperationKind;
start: number;
end: number;
}
export declare function computeOperations(oldBlockMap: BlockMap, newBlockMap: BlockMap, logger: Logger): Array<Operation>;

View file

@ -0,0 +1,114 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.computeOperations = exports.OperationKind = void 0;
var OperationKind;
(function (OperationKind) {
OperationKind[OperationKind["COPY"] = 0] = "COPY";
OperationKind[OperationKind["DOWNLOAD"] = 1] = "DOWNLOAD";
})(OperationKind = exports.OperationKind || (exports.OperationKind = {}));
function computeOperations(oldBlockMap, newBlockMap, logger) {
const nameToOldBlocks = buildBlockFileMap(oldBlockMap.files);
const nameToNewBlocks = buildBlockFileMap(newBlockMap.files);
let lastOperation = null;
// for now only one file is supported in block map
const blockMapFile = newBlockMap.files[0];
const operations = [];
const name = blockMapFile.name;
const oldEntry = nameToOldBlocks.get(name);
if (oldEntry == null) {
// new file (unrealistic case for now, because in any case both blockmap contain the only file named as "file")
throw new Error(`no file ${name} in old blockmap`);
}
const newFile = nameToNewBlocks.get(name);
let changedBlockCount = 0;
const { checksumToOffset: checksumToOldOffset, checksumToOldSize } = buildChecksumMap(nameToOldBlocks.get(name), oldEntry.offset, logger);
let newOffset = blockMapFile.offset;
for (let i = 0; i < newFile.checksums.length; newOffset += newFile.sizes[i], i++) {
const blockSize = newFile.sizes[i];
const checksum = newFile.checksums[i];
let oldOffset = checksumToOldOffset.get(checksum);
if (oldOffset != null && checksumToOldSize.get(checksum) !== blockSize) {
logger.warn(`Checksum ("${checksum}") matches, but size differs (old: ${checksumToOldSize.get(checksum)}, new: ${blockSize})`);
oldOffset = undefined;
}
if (oldOffset === undefined) {
// download data from new file
changedBlockCount++;
if (lastOperation != null && lastOperation.kind === OperationKind.DOWNLOAD && lastOperation.end === newOffset) {
lastOperation.end += blockSize;
}
else {
lastOperation = {
kind: OperationKind.DOWNLOAD,
start: newOffset,
end: newOffset + blockSize,
// oldBlocks: null,
};
validateAndAdd(lastOperation, operations, checksum, i);
}
}
else {
// reuse data from old file
if (lastOperation != null && lastOperation.kind === OperationKind.COPY && lastOperation.end === oldOffset) {
lastOperation.end += blockSize;
// lastOperation.oldBlocks!!.push(checksum)
}
else {
lastOperation = {
kind: OperationKind.COPY,
start: oldOffset,
end: oldOffset + blockSize,
// oldBlocks: [checksum]
};
validateAndAdd(lastOperation, operations, checksum, i);
}
}
}
if (changedBlockCount > 0) {
logger.info(`File${blockMapFile.name === "file" ? "" : " " + blockMapFile.name} has ${changedBlockCount} changed blocks`);
}
return operations;
}
exports.computeOperations = computeOperations;
const isValidateOperationRange = process.env["DIFFERENTIAL_DOWNLOAD_PLAN_BUILDER_VALIDATE_RANGES"] === "true";
function validateAndAdd(operation, operations, checksum, index) {
if (isValidateOperationRange && operations.length !== 0) {
const lastOperation = operations[operations.length - 1];
if (lastOperation.kind === operation.kind && operation.start < lastOperation.end && operation.start > lastOperation.start) {
const min = [lastOperation.start, lastOperation.end, operation.start, operation.end].reduce((p, v) => (p < v ? p : v));
throw new Error(`operation (block index: ${index}, checksum: ${checksum}, kind: ${OperationKind[operation.kind]}) overlaps previous operation (checksum: ${checksum}):\n` +
`abs: ${lastOperation.start} until ${lastOperation.end} and ${operation.start} until ${operation.end}\n` +
`rel: ${lastOperation.start - min} until ${lastOperation.end - min} and ${operation.start - min} until ${operation.end - min}`);
}
}
operations.push(operation);
}
// eslint-disable-next-line @typescript-eslint/explicit-function-return-type
function buildChecksumMap(file, fileOffset, logger) {
const checksumToOffset = new Map();
const checksumToSize = new Map();
let offset = fileOffset;
for (let i = 0; i < file.checksums.length; i++) {
const checksum = file.checksums[i];
const size = file.sizes[i];
const existing = checksumToSize.get(checksum);
if (existing === undefined) {
checksumToOffset.set(checksum, offset);
checksumToSize.set(checksum, size);
}
else if (logger.debug != null) {
const sizeExplanation = existing === size ? "(same size)" : `(size: ${existing}, this size: ${size})`;
logger.debug(`${checksum} duplicated in blockmap ${sizeExplanation}, it doesn't lead to broken differential downloader, just corresponding block will be skipped)`);
}
offset += size;
}
return { checksumToOffset, checksumToOldSize: checksumToSize };
}
function buildBlockFileMap(list) {
const result = new Map();
for (const item of list) {
result.set(item.name, item);
}
return result;
}
//# sourceMappingURL=downloadPlanBuilder.js.map

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1,7 @@
/// <reference types="node" />
import { IncomingMessage } from "http";
import { Writable } from "stream";
import { DifferentialDownloader } from "./DifferentialDownloader";
import { Operation } from "./downloadPlanBuilder";
export declare function executeTasksUsingMultipleRangeRequests(differentialDownloader: DifferentialDownloader, tasks: Array<Operation>, out: Writable, oldFileFd: number, reject: (error: Error) => void): (taskOffset: number) => void;
export declare function checkIsRangesSupported(response: IncomingMessage, reject: (error: Error) => void): boolean;

View file

@ -0,0 +1,112 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.checkIsRangesSupported = exports.executeTasksUsingMultipleRangeRequests = void 0;
const builder_util_runtime_1 = require("builder-util-runtime");
const DataSplitter_1 = require("./DataSplitter");
const downloadPlanBuilder_1 = require("./downloadPlanBuilder");
function executeTasksUsingMultipleRangeRequests(differentialDownloader, tasks, out, oldFileFd, reject) {
const w = (taskOffset) => {
if (taskOffset >= tasks.length) {
if (differentialDownloader.fileMetadataBuffer != null) {
out.write(differentialDownloader.fileMetadataBuffer);
}
out.end();
return;
}
const nextOffset = taskOffset + 1000;
doExecuteTasks(differentialDownloader, {
tasks,
start: taskOffset,
end: Math.min(tasks.length, nextOffset),
oldFileFd,
}, out, () => w(nextOffset), reject);
};
return w;
}
exports.executeTasksUsingMultipleRangeRequests = executeTasksUsingMultipleRangeRequests;
function doExecuteTasks(differentialDownloader, options, out, resolve, reject) {
let ranges = "bytes=";
let partCount = 0;
const partIndexToTaskIndex = new Map();
const partIndexToLength = [];
for (let i = options.start; i < options.end; i++) {
const task = options.tasks[i];
if (task.kind === downloadPlanBuilder_1.OperationKind.DOWNLOAD) {
ranges += `${task.start}-${task.end - 1}, `;
partIndexToTaskIndex.set(partCount, i);
partCount++;
partIndexToLength.push(task.end - task.start);
}
}
if (partCount <= 1) {
// the only remote range - copy
const w = (index) => {
if (index >= options.end) {
resolve();
return;
}
const task = options.tasks[index++];
if (task.kind === downloadPlanBuilder_1.OperationKind.COPY) {
DataSplitter_1.copyData(task, out, options.oldFileFd, reject, () => w(index));
}
else {
const requestOptions = differentialDownloader.createRequestOptions();
requestOptions.headers.Range = `bytes=${task.start}-${task.end - 1}`;
const request = differentialDownloader.httpExecutor.createRequest(requestOptions, response => {
if (!checkIsRangesSupported(response, reject)) {
return;
}
response.pipe(out, {
end: false,
});
response.once("end", () => w(index));
});
differentialDownloader.httpExecutor.addErrorAndTimeoutHandlers(request, reject);
request.end();
}
};
w(options.start);
return;
}
const requestOptions = differentialDownloader.createRequestOptions();
requestOptions.headers.Range = ranges.substring(0, ranges.length - 2);
const request = differentialDownloader.httpExecutor.createRequest(requestOptions, response => {
if (!checkIsRangesSupported(response, reject)) {
return;
}
const contentType = builder_util_runtime_1.safeGetHeader(response, "content-type");
const m = /^multipart\/.+?(?:; boundary=(?:(?:"(.+)")|(?:([^\s]+))))$/i.exec(contentType);
if (m == null) {
reject(new Error(`Content-Type "multipart/byteranges" is expected, but got "${contentType}"`));
return;
}
const dicer = new DataSplitter_1.DataSplitter(out, options, partIndexToTaskIndex, m[1] || m[2], partIndexToLength, resolve);
dicer.on("error", reject);
response.pipe(dicer);
response.on("end", () => {
setTimeout(() => {
request.abort();
reject(new Error("Response ends without calling any handlers"));
}, 10000);
});
});
differentialDownloader.httpExecutor.addErrorAndTimeoutHandlers(request, reject);
request.end();
}
function checkIsRangesSupported(response, reject) {
// Electron net handles redirects automatically, our NodeJS test server doesn't use redirects - so, we don't check 3xx codes.
if (response.statusCode >= 400) {
reject(builder_util_runtime_1.createHttpError(response));
return false;
}
if (response.statusCode !== 206) {
const acceptRanges = builder_util_runtime_1.safeGetHeader(response, "accept-ranges");
if (acceptRanges == null || acceptRanges === "none") {
reject(new Error(`Server doesn't support Accept-Ranges (response code ${response.statusCode})`));
return false;
}
}
return true;
}
exports.checkIsRangesSupported = checkIsRangesSupported;
//# sourceMappingURL=multipleRangeDownloader.js.map

File diff suppressed because one or more lines are too long