/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at
http://mozilla.org/MPL/2.0/. */
const lazy = {};
ChromeUtils.defineESModuleGetters(lazy, {
RemoteSettingsWorker:
"resource://services-settings/RemoteSettingsWorker.sys.mjs",
Utils: "resource://services-settings/Utils.sys.mjs",
});
ChromeUtils.defineLazyGetter(lazy, "console", () => lazy.Utils.log);
class DownloadError extends Error {
constructor(url, resp) {
super(`Could not download ${url}`);
this.name = "DownloadError";
this.resp = resp;
}
}
class DownloadBundleError extends Error {
constructor(url, resp) {
super(`Could not download bundle ${url}`);
this.name = "DownloadBundleError";
this.resp = resp;
}
}
class BadContentError extends Error {
constructor(path) {
super(`${path} content does not match server hash`);
this.name = "BadContentError";
}
}
class ServerInfoError extends Error {
constructor(error) {
super(`Server response is invalid ${error}`);
this.name = "ServerInfoError";
this.original = error;
}
}
class NotFoundError extends Error {
constructor(url, resp) {
super(`Could not find ${url} in cache or dump`);
this.name = "NotFoundError";
this.resp = resp;
}
}
// Helper for the `download` method for commonly used methods, to help with
// lazily accessing the record and attachment content.
class LazyRecordAndBuffer {
constructor(getRecordAndLazyBuffer) {
this.getRecordAndLazyBuffer = getRecordAndLazyBuffer;
}
async _ensureRecordAndLazyBuffer() {
if (!this.recordAndLazyBufferPromise) {
this.recordAndLazyBufferPromise = this.getRecordAndLazyBuffer();
}
return this.recordAndLazyBufferPromise;
}
/**
* @returns {object} The attachment record, if found. null otherwise.
**/
async getRecord() {
try {
return (await this._ensureRecordAndLazyBuffer()).record;
} catch (e) {
return null;
}
}
/**
* @param {object} requestedRecord An attachment record
* @returns {boolean} Whether the requested record matches this record.
**/
async isMatchingRequestedRecord(requestedRecord) {
const record = await this.getRecord();
return (
record &&
record.last_modified === requestedRecord.last_modified &&
record.attachment.size === requestedRecord.attachment.size &&
record.attachment.hash === requestedRecord.attachment.hash
);
}
/**
* Generate the return value for the "download" method.
*
* @throws {*} if the record or attachment content is unavailable.
* @returns {Object} An object with two properties:
* buffer: ArrayBuffer with the file content.
* record: Record associated with the bytes.
**/
async getResult() {
const { record, readBuffer } = await this._ensureRecordAndLazyBuffer();
if (!this.bufferPromise) {
this.bufferPromise = readBuffer();
}
return { record, buffer: await this.bufferPromise };
}
}
export class Downloader {
static get DownloadError() {
return DownloadError;
}
static get DownloadBundleError() {
return DownloadBundleError;
}
static get BadContentError() {
return BadContentError;
}
static get ServerInfoError() {
return ServerInfoError;
}
static get NotFoundError() {
return NotFoundError;
}
constructor(bucketName, collectionName, ...subFolders) {
this.folders = ["settings", bucketName, collectionName, ...subFolders];
this.bucketName = bucketName;
this.collectionName = collectionName;
}
/**
* @returns {Object} An object with async "get", "set" and "delete" methods.
* The keys are strings, the values may be any object that
* can be stored in IndexedDB (including Blob).
*/
get cacheImpl() {
throw new Error("This Downloader does not support caching");
}
/**
* Download attachment and return the result together with the record.
* If the requested record cannot be downloaded and fallbacks are enabled, the
* returned attachment may have a different record than the input record.
*
* @param {Object} record A Remote Settings entry with attachment.
* If omitted, the attachmentId option must be set.
* @param {Object} options Some download options.
* @param {Number} [options.retries] Number of times download should be retried (default: `3`
)
* @param {Boolean} [options.checkHash] Check content integrity (default: `true`)
* @param {string} [options.attachmentId] The attachment identifier to use for
* caching and accessing the attachment.
* (default: `record.id`)
* @param {Boolean} [options.fallbackToCache] Return the cached attachment when the
* input record cannot be fetched.
* (default: `false`)
* @param {Boolean} [options.fallbackToDump] Use the remote settings dump as a
* potential source of the attachment.
* (default: `false`)
* @throws {Downloader.DownloadError} if the file could not be fetched.
* @throws {Downloader.BadContentError} if the downloaded content integrity is not valid.
* @throws {Downloader.ServerInfoError} if the server response is not valid.
* @throws {NetworkError} if fetching the server infos and fetching the attachment fails.
* @returns {Object} An object with two properties:
* `buffer` `ArrayBuffer`: the file content.
* `record` `Object`: record associated with the attachment.
* `_source` `String`: identifies the source of the result. Used for testing.
*/
async download(record, options) {
return this.#fetchAttachment(record, options);
}
/**
* Downloads an attachment bundle for a given collection, if one exists. Fills in the cache
* for all attachments provided by the bundle.
*
* @param {Boolean} force Set to true to force a sync even when local data exists
* @returns {Boolean} True if all attachments were processed successfully, false if failed, null if skipped.
*/
async cacheAll(force = false) {
// If we're offline, don't try
if (lazy.Utils.isOffline) {
return null;
}
// Do nothing if local cache has some data and force is not true
if (!force && (await this.cacheImpl.hasData())) {
return null;
}
// Save attachments in bulks.
const BULK_SAVE_COUNT = 50;
const url =
(await lazy.Utils.baseAttachmentsURL()) +
`bundles/${this.bucketName}--${this.collectionName}.zip`;
const tmpZipFilePath = PathUtils.join(
PathUtils.tempDir,
`${Services.uuid.generateUUID().toString().slice(1, -1)}.zip`
);
let allSuccess = true;
try {
// 1. Download the zip archive to disk
const resp = await lazy.Utils.fetch(url);
if (!resp.ok) {
throw new Downloader.DownloadBundleError(url, resp);
}
const downloaded = await resp.arrayBuffer();
await IOUtils.write(tmpZipFilePath, new Uint8Array(downloaded), {
tmpPath: `${tmpZipFilePath}.tmp`,
});
// 2. Read the zipped content
const zipReader = Cc["@mozilla.org/libjar/zip-reader;1"].createInstance(
Ci.nsIZipReader
);
const tmpZipFile = await IOUtils.getFile(tmpZipFilePath);
zipReader.open(tmpZipFile);
const cacheEntries = [];
const zipFiles = Array.from(zipReader.findEntries("*.meta.json"));
allSuccess = !!zipFiles.length;
for (let i = 0; i < zipFiles.length; i++) {
const lastLoop = i == zipFiles.length - 1;
const entryName = zipFiles[i];
try {
// 3. Read the meta.json entry
const recordZStream = zipReader.getInputStream(entryName);
const recordDataLength = recordZStream.available();
const recordStream = Cc[
"@mozilla.org/scriptableinputstream;1"
].createInstance(Ci.nsIScriptableInputStream);
recordStream.init(recordZStream);
const recordBytes = recordStream.readBytes(recordDataLength);
const recordBlob = new Blob([recordBytes], {
type: "application/json",
});
const record = JSON.parse(await recordBlob.text());
recordZStream.close();
recordStream.close();
// 4. Read the attachment entry
const zStream = zipReader.getInputStream(record.id);
const dataLength = zStream.available();
const stream = Cc[
"@mozilla.org/scriptableinputstream;1"
].createInstance(Ci.nsIScriptableInputStream);
stream.init(zStream);
const fileBytes = stream.readBytes(dataLength);
const blob = new Blob([fileBytes]);
cacheEntries.push([record.id, { record, blob }]);
stream.close();
zStream.close();
} catch (ex) {
lazy.console.warn(
`${this.bucketName}/${this.collectionName}: Unable to extract attachment of ${entryName}.`,
ex
);
allSuccess = false;
}
// 5. Save bulk to cache (last loop or reached count)
if (lastLoop || cacheEntries.length == BULK_SAVE_COUNT) {
try {
await this.cacheImpl.setMultiple(cacheEntries);
} catch (ex) {
lazy.console.warn(
`${this.bucketName}/${this.collectionName}: Unable to save attachments in cache`,
ex
);
allSuccess = false;
}
cacheEntries.splice(0); // start new bulk.
}
}
} catch (ex) {
lazy.console.warn(
`${this.bucketName}/${this.collectionName}: Unable to retrieve remote-settings attachment bundle.`,
ex
);
return false;
}
return allSuccess;
}
/**
* Gets an attachment from the cache or local dump, avoiding requesting it
* from the server.
* If the only found attachment hash does not match the requested record, the
* returned attachment may have a different record, e.g. packaged in binary
* resources or one that is outdated.
*
* @param {Object} record A Remote Settings entry with attachment.
* If omitted, the attachmentId option must be set.
* @param {Object} options Some download options.
* @param {Number} [options.retries] Number of times download should be retried (default: `3`)
* @param {Boolean} [options.checkHash] Check content integrity (default: `true`)
* @param {string} [options.attachmentId] The attachment identifier to use for
* caching and accessing the attachment.
* (default: `record.id`)
* @throws {Downloader.DownloadError} if the file could not be fetched.
* @throws {Downloader.BadContentError} if the downloaded content integrity is not valid.
* @throws {Downloader.ServerInfoError} if the server response is not valid.
* @throws {NetworkError} if fetching the server infos and fetching the attachment fails.
* @returns {Object} An object with two properties:
* `buffer` `ArrayBuffer`: the file content.
* `record` `Object`: record associated with the attachment.
* `_source` `String`: identifies the source of the result. Used for testing.
*/
async get(
record,
options = {
attachmentId: record?.id,
}
) {
return this.#fetchAttachment(record, {
...options,
avoidDownload: true,
fallbackToCache: true,
fallbackToDump: true,
});
}
async #fetchAttachment(record, options) {
let {
retries,
checkHash,
attachmentId = record?.id,
fallbackToCache = false,
fallbackToDump = false,
avoidDownload = false,
} = options || {};
if (!attachmentId) {
// Check for pre-condition. This should not happen, but it is explicitly
// checked to avoid mixing up attachments, which could be dangerous.
throw new Error(
"download() was called without attachmentId or `record.id`"
);
}
if (!lazy.Utils.LOAD_DUMPS) {
if (fallbackToDump) {
lazy.console.warn(
"#fetchAttachment: Forcing fallbackToDump to false due to Utils.LOAD_DUMPS being false"
);
}
fallbackToDump = false;
}
const dumpInfo = new LazyRecordAndBuffer(() =>
this._readAttachmentDump(attachmentId)
);
const cacheInfo = new LazyRecordAndBuffer(() =>
this._readAttachmentCache(attachmentId)
);
// Check if an attachment dump has been packaged with the client.
// The dump is checked before the cache because dumps are expected to match
// the requested record, at least shortly after the release of the client.
if (fallbackToDump && record) {
if (await dumpInfo.isMatchingRequestedRecord(record)) {
try {
return { ...(await dumpInfo.getResult()), _source: "dump_match" };
} catch (e) {
// Failed to read dump: record found but attachment file is missing.
console.error(e);
}
}
}
// Check if the requested attachment has already been cached.
if (record) {
if (await cacheInfo.isMatchingRequestedRecord(record)) {
try {
return { ...(await cacheInfo.getResult()), _source: "cache_match" };
} catch (e) {
// Failed to read cache, e.g. IndexedDB unusable.
console.error(e);
}
}
}
let errorIfAllFails;
// There is no local version that matches the requested record.
// Try to download the attachment specified in record.
if (!avoidDownload && record && record.attachment) {
try {
const newBuffer = await this.downloadAsBytes(record, {
retries,
checkHash,
});
const blob = new Blob([newBuffer]);
// Store in cache but don't wait for it before returning.
this.cacheImpl
.set(attachmentId, { record, blob })
.catch(e => console.error(e));
return { buffer: newBuffer, record, _source: "remote_match" };
} catch (e) {
// No network, corrupted content, etc.
errorIfAllFails = e;
}
}
// Unable to find an attachment that matches the record. Consider falling
// back to local versions, even if their attachment hash do not match the
// one from the requested record.
// Unable to find a valid attachment, fall back to the cached attachment.
const cacheRecord = fallbackToCache && (await cacheInfo.getRecord());
if (cacheRecord) {
const dumpRecord = fallbackToDump && (await dumpInfo.getRecord());
if (dumpRecord?.last_modified >= cacheRecord.last_modified) {
// The dump can be more recent than the cache when the client (and its
// packaged dump) is updated.
try {
return { ...(await dumpInfo.getResult()), _source: "dump_fallback" };
} catch (e) {
// Failed to read dump: record found but attachment file is missing.
console.error(e);
}
}
try {
return { ...(await cacheInfo.getResult()), _source: "cache_fallback" };
} catch (e) {
// Failed to read from cache, e.g. IndexedDB unusable.
console.error(e);
}
}
// Unable to find a valid attachment, fall back to the packaged dump.
if (fallbackToDump && (await dumpInfo.getRecord())) {
try {
return { ...(await dumpInfo.getResult()), _source: "dump_fallback" };
} catch (e) {
errorIfAllFails = e;
}
}
if (errorIfAllFails) {
throw errorIfAllFails;
}
if (avoidDownload) {
throw new Downloader.NotFoundError(attachmentId);
}
throw new Downloader.DownloadError(attachmentId);
}
/**
* Is the record downloaded? This does not check if it was bundled.
*
* @param record A Remote Settings entry with attachment.
* @returns {Promise<boolean>}
*/
isDownloaded(record) {
const cacheInfo = new LazyRecordAndBuffer(() =>
this._readAttachmentCache(record.id)
);
return cacheInfo.isMatchingRequestedRecord(record);
}
/**
* Delete the record attachment downloaded locally.
* No-op if the attachment does not exist.
*
* @param record A Remote Settings entry with attachment.
* @param {Object} options Some options.
* @param {string} options.attachmentId The attachment identifier to use for
* accessing and deleting the attachment.
* (default: `record.id`)
*/
async deleteDownloaded(record, options) {
let { attachmentId = record?.id } = options || {};
if (!attachmentId) {
// Check for pre-condition. This should not happen, but it is explicitly
// checked to avoid mixing up attachments, which could be dangerous.
throw new Error(
"deleteDownloaded() was called without attachmentId or `record.id`"
);
}
return this.cacheImpl.delete(attachmentId);
}
/**
* Clear the cache from obsolete downloaded attachments.
*
* @param {Array<String>} excludeIds List of attachments IDs to exclude from pruning.
*/
async prune(excludeIds) {
return this.cacheImpl.prune(excludeIds);
}
/**
* @deprecated See https://bugzilla.mozilla.org/show_bug.cgi?id=1634127
*
* Download the record attachment into the local profile directory
* and return a file:// URL that points to the local path.
*
* No-op if the file was already downloaded and not corrupted.
*
* @param {Object} record A Remote Settings entry with attachment.
* @param {Object} options Some download options.
* @param {Number} options.retries Number of times download should be retried (default: `3`)
* @throws {Downloader.DownloadError} if the file could not be fetched.
* @throws {Downloader.BadContentError} if the downloaded file integrity is not valid.
* @throws {Downloader.ServerInfoError} if the server response is not valid.
* @throws {NetworkError} if fetching the attachment fails.
* @returns {String} the absolute file path to the downloaded attachment.
*/
async downloadToDisk(record, options = {}) {
const { retries = 3 } = options;
const {
attachment: { filename, size, hash },
} = record;
const localFilePath = PathUtils.join(
PathUtils.localProfileDir,
...this.folders,
filename
);
const localFileUrl = PathUtils.toFileURI(localFilePath);
await this._makeDirs();
let retried = 0;
while (true) {
if (
await lazy.RemoteSettingsWorker.checkFileHash(localFileUrl, size, hash)
) {
return localFileUrl;
}
// File does not exist or is corrupted.
if (retried > retries) {
throw new Downloader.BadContentError(localFilePath);
}
try {
// Download and write on disk.
const buffer = await this.downloadAsBytes(record, {
checkHash: false, // Hash will be checked on file.
retries: 0, // Already in a retry loop.
});
await IOUtils.write(localFilePath, new Uint8Array(buffer), {
tmpPath: `${localFilePath}.tmp`,
});
} catch (e) {
if (retried >= retries) {
throw e;
}
}
retried++;
}
}
/**
* Download the record attachment and return its content as bytes.
*
* @param {Object} record A Remote Settings entry with attachment.
* @param {Object} options Some download options.
* @param {Number} options.retries Number of times download should be retried (default: `3`)
* @param {Boolean} options.checkHash Check content integrity (default: `true`)
* @throws {Downloader.DownloadError} if the file could not be fetched.
* @throws {Downloader.BadContentError} if the downloaded content integrity is not valid.
* @returns {ArrayBuffer} the file content.
*/
async downloadAsBytes(record, options = {}) {
const {
attachment: { location, hash, size },
} = record;
let baseURL;
try {
baseURL = await lazy.Utils.baseAttachmentsURL();
} catch (error) {
throw new Downloader.ServerInfoError(error);
}
const remoteFileUrl = baseURL + location;
const { retries = 3, checkHash = true } = options;
let retried = 0;
while (true) {
try {
const buffer = await this._fetchAttachment(remoteFileUrl);
if (!checkHash) {
return buffer;
}
if (
await lazy.RemoteSettingsWorker.checkContentHash(buffer, size, hash)
) {
return buffer;
}
// Content is corrupted.
throw new Downloader.BadContentError(location);
} catch (e) {
if (retried >= retries) {
throw e;
}
}
retried++;
}
}
/**
* @deprecated See https://bugzilla.mozilla.org/show_bug.cgi?id=1634127
*
* Delete the record attachment downloaded locally.
* This is the counterpart of `downloadToDisk()`.
* Use `deleteDownloaded()` if `download()` was used to retrieve
* the attachment.
*
* No-op if the related file does not exist.
*
* @param record A Remote Settings entry with attachment.
*/
async deleteFromDisk(record) {
const {
attachment: { filename },
} = record;
const path = PathUtils.join(
PathUtils.localProfileDir,
...this.folders,
filename
);
await IOUtils.remove(path);
await this._rmDirs();
}
async _fetchAttachment(url) {
const headers = new Headers();
headers.set("Accept-Encoding", "gzip");
const resp = await lazy.Utils.fetch(url, { headers });
if (!resp.ok) {
throw new Downloader.DownloadError(url, resp);
}
return resp.arrayBuffer();
}
async _readAttachmentCache(attachmentId) {
const cached = await this.cacheImpl.get(attachmentId);
if (!cached) {
throw new Downloader.DownloadError(attachmentId);
}
return {
record: cached.record,
async readBuffer() {
const buffer = await cached.blob.arrayBuffer();
const { size, hash } = cached.record.attachment;
if (
await lazy.RemoteSettingsWorker.checkContentHash(buffer, size, hash)
) {
return buffer;
}
// Really unexpected, could indicate corruption in IndexedDB.
throw new Downloader.BadContentError(attachmentId);
},
};
}
async _readAttachmentDump(attachmentId) {
async function fetchResource(resourceUrl) {
try {
return await fetch(resourceUrl);
} catch (e) {
throw new Downloader.DownloadError(resourceUrl);
}
}
const resourceUrlPrefix =
Downloader._RESOURCE_BASE_URL + "/" + this.folders.join("/") + "/";
const recordUrl = `${resourceUrlPrefix}${attachmentId}.meta.json`;
const attachmentUrl = `${resourceUrlPrefix}${attachmentId}`;
const record = await (await fetchResource(recordUrl)).json();
return {
record,
async readBuffer() {
return (await fetchResource(attachmentUrl)).arrayBuffer();
},
};
}
// Separate variable to allow tests to override this.
static _RESOURCE_BASE_URL = "resource://app/defaults";
async _makeDirs() {
const dirPath = PathUtils.join(PathUtils.localProfileDir, ...this.folders);
await IOUtils.makeDirectory(dirPath, { createAncestors: true });
}
async _rmDirs() {
for (let i = this.folders.length; i > 0; i--) {
const dirPath = PathUtils.join(
PathUtils.localProfileDir,
...this.folders.slice(0, i)
);
try {
await IOUtils.remove(dirPath);
} catch (e) {
// This could fail if there's something in
// the folder we're not permitted to remove.
break;
}
}
}
}
