/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*-
* vim: sw=2 ts=2 sts=2 expandtab filetype=javascript
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at
http://mozilla.org/MPL/2.0/. */
const BYTES_PER_MEBIBYTE = 1048576;
const MS_PER_DAY = 86400000;
// Threshold value for removeOldCorruptDBs.
// Corrupt DBs older than this value are removed.
const CORRUPT_DB_RETAIN_DAYS = 14;
// Seconds between maintenance runs.
const MAINTENANCE_INTERVAL_SECONDS = 7 * 86400;
const lazy = {};
ChromeUtils.defineESModuleGetters(lazy, {
PlacesPreviews: "resource://gre/modules/PlacesPreviews.sys.mjs",
PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs",
Sqlite: "resource://gre/modules/Sqlite.sys.mjs",
});
export var PlacesDBUtils = {
_isShuttingDown: false,
_clearTaskQueue: false,
clearPendingTasks() {
PlacesDBUtils._clearTaskQueue = true;
},
/**
* Executes integrity check and common maintenance tasks.
*
* @return a Map[taskName(String) -> Object]. The Object has the following properties:
* - succeeded: boolean
* - logs: an array of strings containing the messages logged by the task.
*/
async maintenanceOnIdle() {
let tasks = [
this.checkIntegrity,
this.checkCoherence,
this._refreshUI,
this.incrementalVacuum,
this.removeOldCorruptDBs,
this.deleteOrphanPreviews,
];
let telemetryStartTime = Date.now();
let taskStatusMap = await PlacesDBUtils.runTasks(tasks);
Services.prefs.setIntPref(
"places.database.lastMaintenance",
parseInt(Date.now() / 1000)
);
Services.telemetry
.getHistogramById("PLACES_IDLE_MAINTENANCE_TIME_MS")
.add(Date.now() - telemetryStartTime);
return taskStatusMap;
},
/**
* Executes integrity check, common and advanced maintenance tasks (like
* expiration and vacuum). Will also collect statistics on the database.
*
* Note: although this function isn't actually async, we keep it async to
* allow us to maintain a simple, consistent API for the tasks within this object.
*
* @return {Promise}
* A promise that resolves with a Map[taskName(String) -> Object].
* The Object has the following properties:
* - succeeded: boolean
* - logs: an array of strings containing the messages logged by the task.
*/
async checkAndFixDatabase() {
let tasks = [
this.checkIntegrity,
this.checkCoherence,
this.expire,
this.vacuum,
this.stats,
this._refreshUI,
];
return PlacesDBUtils.runTasks(tasks);
},
/**
* Forces a full refresh of Places views.
*
* Note: although this function isn't actually async, we keep it async to
* allow us to maintain a simple, consistent API for the tasks within this object.
*
* @returns {Array} An empty array.
*/
async _refreshUI() {
PlacesObservers.notifyListeners([new PlacesPurgeCaches()]);
return [];
},
/**
* Checks integrity and tries to fix the database through a reindex.
*
* @return {Promise} resolves if database is sane or is made sane.
* @resolves to an array of logs for this task.
* @rejects if we're unable to fix corruption or unable to check status.
*/
async checkIntegrity() {
let logs = [];
async function check(dbName) {
try {
await integrity(dbName);
logs.push(`The ${dbName} database is sane`);
} catch (ex) {
PlacesDBUtils.clearPendingTasks();
if (ex.result == Cr.NS_ERROR_FILE_CORRUPTED) {
logs.push(`The ${dbName} database is corrupt`);
Services.prefs.setCharPref(
"places.database.replaceDatabaseOnStartup",
dbName
);
throw new Error(
`Unable to fix corruption, ${dbName} will be replaced on next startup`
);
}
throw new Error(`Unable to check ${dbName} integrity: ${ex}`);
}
}
await check("places.sqlite");
await check("favicons.sqlite");
return logs;
},
/**
* Checks data coherence and tries to fix most common errors.
*
* @return {Promise} resolves when coherence is checked.
* @resolves to an array of logs for this task.
* @rejects if database is not coherent.
*/
async checkCoherence() {
let logs = [];
let stmts = await PlacesDBUtils._getCoherenceStatements();
let coherenceCheck = true;
await lazy.PlacesUtils.withConnectionWrapper(
"PlacesDBUtils: coherence check:",
db =>
db.executeTransaction(async () => {
for (let { query, params } of stmts) {
try {
await db.execute(query, params || null);
} catch (ex) {
console.error(ex);
coherenceCheck = false;
}
}
})
);
if (coherenceCheck) {
logs.push("The database is coherent");
} else {
PlacesDBUtils.clearPendingTasks();
throw new Error("Unable to complete the coherence check");
}
return logs;
},
/**
* Runs incremental vacuum on databases supporting it.
*
* @return {Promise} resolves when done.
* @resolves to an array of logs for this task.
* @rejects if we were unable to vacuum.
*/
async incrementalVacuum() {
let logs = [];
return lazy.PlacesUtils.withConnectionWrapper(
"PlacesDBUtils: incrementalVacuum",
async db => {
let count = (
await db.execute("PRAGMA favicons.freelist_count")
)[0].getResultByIndex(0);
if (count < 10) {
logs.push(
`The favicons database has only ${count} free pages, not vacuuming.`
);
} else {
logs.push(
`The favicons database has ${count} free pages, vacuuming.`
);
await db.execute("PRAGMA favicons.incremental_vacuum");
count = (
await db.execute("PRAGMA favicons.freelist_count")
)[0].getResultByIndex(0);
logs.push(
`The database has been vacuumed and has now ${count} free pages.`
);
}
return logs;
}
).catch(ex => {
PlacesDBUtils.clearPendingTasks();
throw new Error(
"Unable to incrementally vacuum the favicons database " + ex
);
});
},
/**
* Expire orphan previews that don't have a Places entry anymore.
*
* @return {Promise} resolves when done.
* @resolves to an array of logs for this task.
*/
async deleteOrphanPreviews() {
let logs = [];
try {
let deleted = await lazy.PlacesPreviews.deleteOrphans();
if (deleted) {
logs.push(`Orphan previews deleted.`);
}
} catch (ex) {
throw new Error("Unable to delete orphan previews " + ex);
}
return logs;
},
async _getCoherenceStatements() {
let cleanupStatements = [
// MOZ_PLACES
// L.1 remove duplicate URLs.
// This task uses a temp table of potential dupes, and a trigger to remove
// them. It runs first because it relies on subsequent tasks to clean up
// orphaned foreign key references. The task works like this: first, we
// insert all rows with the same hash into the temp table. This lets
// SQLite use the `url_hash` index for scanning `moz_places`. Hashes
// aren't unique, so two different URLs might have the same hash. To find
// the actual dupes, we use a unique constraint on the URL in the temp
// table. If that fails, we bump the dupe count. Then, we delete all dupes
// from the table. This fires the cleanup trigger, which updates all
// foreign key references to point to one of the duplicate Places, then
// deletes the others.
{
query: `CREATE TEMP TABLE IF NOT EXISTS moz_places_dupes_temp(
id INTEGER PRIMARY KEY
, hash INTEGER NOT NULL
, url TEXT UNIQUE NOT NULL
, count INTEGER NOT NULL DEFAULT 0
)`,
},
{
query: `CREATE TEMP TRIGGER IF NOT EXISTS moz_places_remove_dupes_temp_trigger
AFTER DELETE ON moz_places_dupes_temp
FOR EACH ROW
BEGIN
/* Reassign history visits. */
UPDATE moz_historyvisits SET
place_id = OLD.id
WHERE place_id IN (SELECT id FROM moz_places
WHERE id <> OLD.id AND
url_hash = OLD.hash AND
url = OLD.url);
/* Merge autocomplete history entries. */
INSERT INTO moz_inputhistory(place_id, input, use_count)
SELECT OLD.id, a.input, a.use_count
FROM moz_inputhistory a
JOIN moz_places h ON h.id = a.place_id
WHERE h.id <> OLD.id AND
h.url_hash = OLD.hash AND
h.url = OLD.url
ON CONFLICT(place_id, input) DO UPDATE SET
place_id = excluded.place_id,
use_count = use_count + excluded.use_count;
/* Merge page annos, ignoring annos with the same name that are
already set on the destination. */
INSERT OR IGNORE INTO moz_annos(id, place_id, anno_attribute_id,
content, flags, expiration, type,
dateAdded, lastModified)
SELECT (SELECT k.id FROM moz_annos k
WHERE k.place_id = OLD.id AND
k.anno_attribute_id = a.anno_attribute_id), OLD.id,
a.anno_attribute_id, a.content, a.flags, a.expiration, a.type,
a.dateAdded, a.lastModified
FROM moz_annos a
JOIN moz_places h ON h.id = a.place_id
WHERE h.id <> OLD.id AND
url_hash = OLD.hash AND
url = OLD.url;
/* Reassign bookmarks, and bump the Sync change counter just in case
we have new keywords. */
UPDATE moz_bookmarks SET
fk = OLD.id,
syncChangeCounter = syncChangeCounter + 1
WHERE fk IN (SELECT id FROM moz_places
WHERE url_hash = OLD.hash AND
url = OLD.url);
/* Reassign keywords. */
UPDATE moz_keywords SET
place_id = OLD.id
WHERE place_id IN (SELECT id FROM moz_places
WHERE id <> OLD.id AND
url_hash = OLD.hash AND
url = OLD.url);
/* Now that we've updated foreign key references, drop the
conflicting source. */
DELETE FROM moz_places
WHERE id <> OLD.id AND
url_hash = OLD.hash AND
url = OLD.url;
/* Recalculate frecency for the destination. */
UPDATE moz_places SET recalc_frecency = 1, recalc_alt_frecency = 1
WHERE id = OLD.id;
END`,
},
{
query: `INSERT INTO moz_places_dupes_temp(id, hash, url, count)
SELECT h.id, h.url_hash, h.url, 1
FROM moz_places h
JOIN (SELECT url_hash FROM moz_places
GROUP BY url_hash
HAVING count(*) > 1) d ON d.url_hash = h.url_hash
ON CONFLICT(url) DO UPDATE SET
count = count + 1`,
},
{ query: `DELETE FROM moz_places_dupes_temp WHERE count > 1` },
{ query: `DROP TABLE moz_places_dupes_temp` },
// MOZ_ANNO_ATTRIBUTES
// A.1 remove obsolete annotations from moz_annos.
// The 'weave0' idiom exploits character ordering (0 follows /) to
// efficiently select all annos with a 'weave/' prefix.
{
query: `DELETE FROM moz_annos
WHERE type = 4 OR anno_attribute_id IN (
SELECT id FROM moz_anno_attributes
WHERE name = 'downloads/destinationFileName' OR
name BETWEEN 'weave/' AND 'weave0'
)`,
},
// A.3 remove unused attributes.
{
query: `DELETE FROM moz_anno_attributes WHERE id IN (
SELECT id FROM moz_anno_attributes n
WHERE NOT EXISTS
(SELECT id FROM moz_annos WHERE anno_attribute_id = n.id LIMIT 1)
)`,
},
// MOZ_ANNOS
// B.1 remove annos with an invalid attribute
{
query: `DELETE FROM moz_annos WHERE id IN (
SELECT id FROM moz_annos a
WHERE NOT EXISTS
(SELECT id FROM moz_anno_attributes
WHERE id = a.anno_attribute_id LIMIT 1)
)`,
},
// B.2 remove orphan annos
{
query: `DELETE FROM moz_annos WHERE id IN (
SELECT id FROM moz_annos a
WHERE NOT EXISTS
(SELECT id FROM moz_places WHERE id = a.place_id LIMIT 1)
)`,
},
// D.1 remove items that are not uri bookmarks from tag containers
{
query: `DELETE FROM moz_bookmarks WHERE guid NOT IN (
:rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */
) AND id IN (
SELECT b.id FROM moz_bookmarks b
WHERE b.parent IN
(SELECT id FROM moz_bookmarks WHERE parent = :tags_folder)
AND b.type <> :bookmark_type
)`,
params: {
tags_folder: lazy.PlacesUtils.tagsFolderId,
bookmark_type: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK,
rootGuid: lazy.PlacesUtils.bookmarks.rootGuid,
menuGuid: lazy.PlacesUtils.bookmarks.menuGuid,
toolbarGuid: lazy.PlacesUtils.bookmarks.toolbarGuid,
unfiledGuid: lazy.PlacesUtils.bookmarks.unfiledGuid,
tagsGuid: lazy.PlacesUtils.bookmarks.tagsGuid,
},
},
// D.2 remove empty tags
{
query: `DELETE FROM moz_bookmarks WHERE guid NOT IN (
:rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */
) AND id IN (
SELECT b.id FROM moz_bookmarks b
WHERE b.id IN
(SELECT id FROM moz_bookmarks WHERE parent = :tags_folder)
AND NOT EXISTS
(SELECT id from moz_bookmarks WHERE parent = b.id LIMIT 1)
)`,
params: {
tags_folder: lazy.PlacesUtils.tagsFolderId,
rootGuid: lazy.PlacesUtils.bookmarks.rootGuid,
menuGuid: lazy.PlacesUtils.bookmarks.menuGuid,
toolbarGuid: lazy.PlacesUtils.bookmarks.toolbarGuid,
unfiledGuid: lazy.PlacesUtils.bookmarks.unfiledGuid,
tagsGuid: lazy.PlacesUtils.bookmarks.tagsGuid,
},
},
// D.3 move orphan items to unsorted folder
{
query: `UPDATE moz_bookmarks SET
parent = (SELECT id FROM moz_bookmarks WHERE guid = :unfiledGuid)
WHERE guid NOT IN (
:rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */
) AND id IN (
SELECT b.id FROM moz_bookmarks b
WHERE NOT EXISTS
(SELECT id FROM moz_bookmarks WHERE id = b.parent LIMIT 1)
)`,
params: {
rootGuid: lazy.PlacesUtils.bookmarks.rootGuid,
menuGuid: lazy.PlacesUtils.bookmarks.menuGuid,
toolbarGuid: lazy.PlacesUtils.bookmarks.toolbarGuid,
unfiledGuid: lazy.PlacesUtils.bookmarks.unfiledGuid,
tagsGuid: lazy.PlacesUtils.bookmarks.tagsGuid,
},
},
// D.4 Insert tombstones for any synced items with the wrong type.
// Sync doesn't support changing the type of an existing item while
// keeping its GUID. To avoid confusing other clients, we insert
// tombstones for all synced items with the wrong type, so that we
// can reupload them with the correct type and a new GUID.
{
query: `INSERT OR IGNORE INTO moz_bookmarks_deleted(guid, dateRemoved)
SELECT guid, :dateRemoved
FROM moz_bookmarks
WHERE syncStatus <> :syncStatus AND
((type IN (:folder_type, :separator_type) AND
fk NOTNULL) OR
(type = :bookmark_type AND
fk IS NULL) OR
type IS NULL)`,
params: {
dateRemoved: lazy.PlacesUtils.toPRTime(new Date()),
syncStatus: lazy.PlacesUtils.bookmarks.SYNC_STATUS.NEW,
bookmark_type: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK,
folder_type: lazy.PlacesUtils.bookmarks.TYPE_FOLDER,
separator_type: lazy.PlacesUtils.bookmarks.TYPE_SEPARATOR,
},
},
// D.5 fix wrong item types
// Folders and separators should not have an fk.
// If they have a valid fk, convert them to bookmarks, and give them new
// GUIDs. If the item has children, we'll move them to the unfiled root
// in D.8. If the `fk` doesn't exist in `moz_places`, we'll remove the
// item in D.9.
{
query: `UPDATE moz_bookmarks
SET guid = GENERATE_GUID(),
type = :bookmark_type
WHERE guid NOT IN (
:rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */
) AND id IN (
SELECT id FROM moz_bookmarks b
WHERE type IN (:folder_type, :separator_type)
AND fk NOTNULL
)`,
params: {
bookmark_type: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK,
folder_type: lazy.PlacesUtils.bookmarks.TYPE_FOLDER,
separator_type: lazy.PlacesUtils.bookmarks.TYPE_SEPARATOR,
rootGuid: lazy.PlacesUtils.bookmarks.rootGuid,
menuGuid: lazy.PlacesUtils.bookmarks.menuGuid,
toolbarGuid: lazy.PlacesUtils.bookmarks.toolbarGuid,
unfiledGuid: lazy.PlacesUtils.bookmarks.unfiledGuid,
tagsGuid: lazy.PlacesUtils.bookmarks.tagsGuid,
},
},
// D.6 fix wrong item types
// Bookmarks should have an fk, if they don't have any, convert them to
// folders.
{
query: `UPDATE moz_bookmarks
SET guid = GENERATE_GUID(),
type = :folder_type
WHERE guid NOT IN (
:rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */
) AND id IN (
SELECT id FROM moz_bookmarks b
WHERE type = :bookmark_type
AND fk IS NULL
)`,
params: {
bookmark_type: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK,
folder_type: lazy.PlacesUtils.bookmarks.TYPE_FOLDER,
rootGuid: lazy.PlacesUtils.bookmarks.rootGuid,
menuGuid: lazy.PlacesUtils.bookmarks.menuGuid,
toolbarGuid: lazy.PlacesUtils.bookmarks.toolbarGuid,
unfiledGuid: lazy.PlacesUtils.bookmarks.unfiledGuid,
tagsGuid: lazy.PlacesUtils.bookmarks.tagsGuid,
},
},
// D.7 fix wrong item types
// `moz_bookmarks.type` doesn't have a NOT NULL constraint, so it's
// possible for an item to not have a type (bug 1586427).
{
query: `UPDATE moz_bookmarks
SET guid = GENERATE_GUID(),
type = CASE WHEN fk NOT NULL THEN :bookmark_type ELSE :folder_type END
WHERE guid NOT IN (
:rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */
) AND type IS NULL`,
params: {
bookmark_type: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK,
folder_type: lazy.PlacesUtils.bookmarks.TYPE_FOLDER,
rootGuid: lazy.PlacesUtils.bookmarks.rootGuid,
menuGuid: lazy.PlacesUtils.bookmarks.menuGuid,
toolbarGuid: lazy.PlacesUtils.bookmarks.toolbarGuid,
unfiledGuid: lazy.PlacesUtils.bookmarks.unfiledGuid,
tagsGuid: lazy.PlacesUtils.bookmarks.tagsGuid,
},
},
// D.8 fix wrong parents
// Items cannot have separators or other bookmarks
// as parent, if they have bad parent move them to unsorted bookmarks.
{
query: `UPDATE moz_bookmarks SET
parent = (SELECT id FROM moz_bookmarks WHERE guid = :unfiledGuid)
WHERE guid NOT IN (
:rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */
) AND id IN (
SELECT id FROM moz_bookmarks b
WHERE EXISTS
(SELECT id FROM moz_bookmarks WHERE id = b.parent
AND type IN (:bookmark_type, :separator_type)
LIMIT 1)
)`,
params: {
bookmark_type: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK,
separator_type: lazy.PlacesUtils.bookmarks.TYPE_SEPARATOR,
rootGuid: lazy.PlacesUtils.bookmarks.rootGuid,
menuGuid: lazy.PlacesUtils.bookmarks.menuGuid,
toolbarGuid: lazy.PlacesUtils.bookmarks.toolbarGuid,
unfiledGuid: lazy.PlacesUtils.bookmarks.unfiledGuid,
tagsGuid: lazy.PlacesUtils.bookmarks.tagsGuid,
},
},
// D.9 remove items without a valid place
// We've already converted folders with an `fk` to bookmarks in D.5,
// and bookmarks without an `fk` to folders in D.6. However, the `fk`
// might not reference an existing `moz_places.id`, even if it's
// NOT NULL. This statement takes care of those.
{
query: `DELETE FROM moz_bookmarks AS b
WHERE b.guid NOT IN (
:rootGuid, :menuGuid, :toolbarGuid, :unfiledGuid, :tagsGuid /* skip roots */
) AND b.fk NOT NULL
AND b.type = :bookmark_type
AND NOT EXISTS (SELECT 1 FROM moz_places h WHERE h.id = b.fk)`,
params: {
bookmark_type: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK,
rootGuid: lazy.PlacesUtils.bookmarks.rootGuid,
menuGuid: lazy.PlacesUtils.bookmarks.menuGuid,
toolbarGuid: lazy.PlacesUtils.bookmarks.toolbarGuid,
unfiledGuid: lazy.PlacesUtils.bookmarks.unfiledGuid,
tagsGuid: lazy.PlacesUtils.bookmarks.tagsGuid,
},
},
// D.10 fix non-consecutive positions.
{
query: `
WITH positions(item_id, pos, seq) AS (
SELECT id, position AS pos,
(row_number() OVER (PARTITION BY parent ORDER BY position)) - 1 AS seq
FROM moz_bookmarks
)
UPDATE moz_bookmarks
SET position = seq
FROM positions
WHERE item_id = moz_bookmarks.id AND seq <> pos`,
},
// D.12 Fix empty-named tags.
// Tags were allowed to have empty names due to a UI bug. Fix them by
// replacing their title with "(notitle)", and bumping the change counter
// for all bookmarks with the fixed tags.
{
query: `UPDATE moz_bookmarks SET syncChangeCounter = syncChangeCounter + 1
WHERE fk IN (SELECT b.fk FROM moz_bookmarks b
JOIN moz_bookmarks p ON p.id = b.parent
WHERE length(p.title) = 0 AND p.type = :folder_type AND
p.parent = :tags_folder)`,
params: {
folder_type: lazy.PlacesUtils.bookmarks.TYPE_FOLDER,
tags_folder: lazy.PlacesUtils.tagsFolderId,
},
},
{
query: `UPDATE moz_bookmarks SET title = :empty_title
WHERE length(title) = 0 AND type = :folder_type
AND parent = :tags_folder`,
params: {
empty_title: "(notitle)",
folder_type: lazy.PlacesUtils.bookmarks.TYPE_FOLDER,
tags_folder: lazy.PlacesUtils.tagsFolderId,
},
},
// MOZ_ICONS
// E.1 remove orphan icon entries.
{
query: `DELETE FROM moz_pages_w_icons WHERE page_url_hash NOT IN (
SELECT url_hash FROM moz_places
)`,
},
// Remove icons whose origin is not in moz_origins, unless referenced.
{
query: `DELETE FROM moz_icons WHERE id IN (
SELECT id FROM moz_icons WHERE root = 0
UNION ALL
SELECT id FROM moz_icons
WHERE root = 1
AND get_host_and_port(icon_url) NOT IN (SELECT host FROM moz_origins)
AND fixup_url(get_host_and_port(icon_url)) NOT IN (SELECT host FROM moz_origins)
EXCEPT
SELECT icon_id FROM moz_icons_to_pages
)`,
},
// MOZ_HISTORYVISITS
// F.1 remove orphan visits
{
query: `DELETE FROM moz_historyvisits WHERE id IN (
SELECT id FROM moz_historyvisits v
WHERE NOT EXISTS
(SELECT id FROM moz_places WHERE id = v.place_id LIMIT 1)
)`,
},
// MOZ_INPUTHISTORY
// G.1 remove orphan input history
{
query: `DELETE FROM moz_inputhistory WHERE place_id IN (
SELECT place_id FROM moz_inputhistory i
WHERE NOT EXISTS
(SELECT id FROM moz_places WHERE id = i.place_id LIMIT 1)
)`,
},
// MOZ_KEYWORDS
// I.1 remove unused keywords
{
query: `DELETE FROM moz_keywords WHERE id IN (
SELECT id FROM moz_keywords k
WHERE NOT EXISTS
(SELECT 1 FROM moz_places h WHERE k.place_id = h.id)
)`,
},
// MOZ_PLACES
// L.2 recalculate visit_count and last_visit_date
{
query: `UPDATE moz_places
SET visit_count = (SELECT count(*) FROM moz_historyvisits
WHERE place_id = moz_places.id AND visit_type NOT IN (0,4,7,8,9)),
last_visit_date = (SELECT MAX(visit_date) FROM moz_historyvisits
WHERE place_id = moz_places.id)
WHERE id IN (
SELECT h.id FROM moz_places h
WHERE visit_count <> (SELECT count(*) FROM moz_historyvisits v
WHERE v.place_id = h.id AND visit_type NOT IN (0,4,7,8,9))
OR last_visit_date IS NOT
(SELECT MAX(visit_date) FROM moz_historyvisits v WHERE v.place_id = h.id)
)`,
},
// L.3 recalculate hidden for redirects.
{
query: `UPDATE moz_places
SET hidden = 1
WHERE id IN (
SELECT h.id FROM moz_places h
JOIN moz_historyvisits src ON src.place_id = h.id
JOIN moz_historyvisits dst ON dst.from_visit = src.id AND dst.visit_type IN (5,6)
LEFT JOIN moz_bookmarks on fk = h.id AND fk ISNULL
GROUP BY src.place_id HAVING count(*) = visit_count
)`,
},
// L.4 recalculate foreign_count.
{
query: `UPDATE moz_places SET foreign_count =
(SELECT count(*) FROM moz_bookmarks WHERE fk = moz_places.id ) +
(SELECT count(*) FROM moz_keywords WHERE place_id = moz_places.id )`,
},
// L.5 recalculate missing hashes.
{
query: `UPDATE moz_places SET url_hash = hash(url) WHERE url_hash = 0`,
},
// L.6 fix invalid Place GUIDs.
{
query: `UPDATE moz_places
SET guid = GENERATE_GUID()
WHERE guid IS NULL OR
NOT IS_VALID_GUID(guid)`,
},
// MOZ_BOOKMARKS
// S.1 fix invalid GUIDs for synced bookmarks.
// This requires multiple related statements.
// First, we insert tombstones for all synced bookmarks with invalid
// GUIDs, so that we can delete them on the server. Second, we add a
// temporary trigger to bump the change counter for the parents of any
// items we update, since Sync stores the list of child GUIDs on the
// parent. Finally, we assign new GUIDs for all items with missing and
// invalid GUIDs, bump their change counters, and reset their sync
// statuses to NEW so that they're considered for deduping.
{
query: `INSERT OR IGNORE INTO moz_bookmarks_deleted(guid, dateRemoved)
SELECT guid, :dateRemoved
FROM moz_bookmarks
WHERE syncStatus <> :syncStatus AND
guid NOT NULL AND
NOT IS_VALID_GUID(guid)`,
params: {
dateRemoved: lazy.PlacesUtils.toPRTime(new Date()),
syncStatus: lazy.PlacesUtils.bookmarks.SYNC_STATUS.NEW,
},
},
{
query: `UPDATE moz_bookmarks
SET guid = GENERATE_GUID(),
syncStatus = :syncStatus
WHERE guid IS NULL OR
NOT IS_VALID_GUID(guid)`,
params: {
syncStatus: lazy.PlacesUtils.bookmarks.SYNC_STATUS.NEW,
},
},
// S.2 drop tombstones for bookmarks that aren't deleted.
{
query: `DELETE FROM moz_bookmarks_deleted
WHERE guid IN (SELECT guid FROM moz_bookmarks)`,
},
// S.3 set missing added and last modified dates.
{
query: `UPDATE moz_bookmarks
SET dateAdded = COALESCE(NULLIF(dateAdded, 0), NULLIF(lastModified, 0), NULLIF((
SELECT MIN(visit_date) FROM moz_historyvisits
WHERE place_id = fk
), 0), STRFTIME('%s', 'now', 'localtime', 'utc') * 1000000),
lastModified = COALESCE(NULLIF(lastModified, 0), NULLIF(dateAdded, 0), NULLIF((
SELECT MAX(visit_date) FROM moz_historyvisits
WHERE place_id = fk
), 0), STRFTIME('%s', 'now', 'localtime', 'utc') * 1000000)
WHERE NULLIF(dateAdded, 0) IS NULL OR
NULLIF(lastModified, 0) IS NULL`,
},
// S.4 reset added dates that are ahead of last modified dates.
{
query: `UPDATE moz_bookmarks
SET dateAdded = lastModified
WHERE dateAdded > lastModified`,
},
];
// Create triggers for updating Sync metadata. The "sync change" trigger
// bumps the parent's change counter when we update a GUID or move an item
// to a different folder, since Sync stores the list of child GUIDs on the
// parent. The "sync tombstone" trigger inserts tombstones for deleted
// synced bookmarks.
cleanupStatements.unshift({
query: `CREATE TEMP TRIGGER IF NOT EXISTS moz_bm_sync_change_temp_trigger
AFTER UPDATE OF guid, parent, position ON moz_bookmarks
FOR EACH ROW
BEGIN
UPDATE moz_bookmarks
SET syncChangeCounter = syncChangeCounter + 1
WHERE id IN (OLD.parent, NEW.parent, NEW.id);
END`,
});
cleanupStatements.unshift({
query: `CREATE TEMP TRIGGER IF NOT EXISTS moz_bm_sync_tombstone_temp_trigger
AFTER DELETE ON moz_bookmarks
FOR EACH ROW WHEN OLD.guid NOT NULL AND
OLD.syncStatus <> 1
BEGIN
UPDATE moz_bookmarks
SET syncChangeCounter = syncChangeCounter + 1
WHERE id = OLD.parent;
INSERT INTO moz_bookmarks_deleted(guid, dateRemoved)
VALUES(OLD.guid, STRFTIME('%s', 'now', 'localtime', 'utc') * 1000000);
END`,
});
cleanupStatements.push({
query: `DROP TRIGGER moz_bm_sync_change_temp_trigger`,
});
cleanupStatements.push({
query: `DROP TRIGGER moz_bm_sync_tombstone_temp_trigger`,
});
return cleanupStatements;
},
/**
* Tries to vacuum the database.
*
* Note: although this function isn't actually async, we keep it async to
* allow us to maintain a simple, consistent API for the tasks within this object.
*
* @return {Promise} resolves when database is vacuumed.
* @resolves to an array of logs for this task.
* @rejects if we are unable to vacuum database.
*/
async vacuum() {
let logs = [];
let placesDbPath = PathUtils.join(PathUtils.profileDir, "places.sqlite");
let info = await IOUtils.stat(placesDbPath);
logs.push(`Initial database size is ${parseInt(info.size / 1024)}KiB`);
return lazy.PlacesUtils.withConnectionWrapper(
"PlacesDBUtils: vacuum",
async db => {
await db.execute("VACUUM");
logs.push("The database has been vacuumed");
info = await IOUtils.stat(placesDbPath);
logs.push(`Final database size is ${parseInt(info.size / 1024)}KiB`);
return logs;
}
).catch(() => {
PlacesDBUtils.clearPendingTasks();
throw new Error("Unable to vacuum database");
});
},
/**
* Forces a full expiration on the database.
*
* Note: although this function isn't actually async, we keep it async to
* allow us to maintain a simple, consistent API for the tasks within this object.
*
* @return {Promise} resolves when the database in cleaned up.
* @resolves to an array of logs for this task.
*/
async expire() {
let logs = [];
let expiration = Cc["@mozilla.org/places/expiration;1"].getService(
Ci.nsIObserver
);
let returnPromise = new Promise(res => {
let observer = (subject, topic) => {
Services.obs.removeObserver(observer, topic);
logs.push("Database cleaned up");
res(logs);
};
Services.obs.addObserver(
observer,
lazy.PlacesUtils.TOPIC_EXPIRATION_FINISHED
);
});
// Force an orphans expiration step.
expiration.observe(null, "places-debug-start-expiration", 0);
return returnPromise;
},
/**
* Collects statistical data on the database.
*
* @return {Promise} resolves when statistics are collected.
* @resolves to an array of logs for this task.
* @rejects if we are unable to collect stats for some reason.
*/
async stats() {
let logs = [];
let placesDbPath = PathUtils.join(PathUtils.profileDir, "places.sqlite");
let info = await IOUtils.stat(placesDbPath);
logs.push(`Places.sqlite size is ${parseInt(info.size / 1024)}KiB`);
let faviconsDbPath = PathUtils.join(
PathUtils.profileDir,
"favicons.sqlite"
);
info = await IOUtils.stat(faviconsDbPath);
logs.push(`Favicons.sqlite size is ${parseInt(info.size / 1024)}KiB`);
// Execute each step async.
let pragmas = [
"user_version",
"page_size",
"cache_size",
"journal_mode",
"synchronous",
].map(p => `pragma_${p}`);
let pragmaQuery = `SELECT * FROM ${pragmas.join(", ")}`;
await lazy.PlacesUtils.withConnectionWrapper(
"PlacesDBUtils: pragma for stats",
async db => {
let row = (await db.execute(pragmaQuery))[0];
for (let i = 0; i != pragmas.length; i++) {
logs.push(`${pragmas[i]} is ${row.getResultByIndex(i)}`);
}
}
).catch(() => {
logs.push("Could not set pragma for stat collection");
});
// Get maximum number of unique URIs.
try {
let limitURIs = await Cc["@mozilla.org/places/expiration;1"]
.getService(Ci.nsISupports)
.wrappedJSObject.getPagesLimit();
logs.push(
"History can store a maximum of " + limitURIs + " unique pages"
);
} catch (ex) {}
let query = "SELECT name FROM sqlite_master WHERE type = :type";
let params = {};
let _getTableCount = async tableName => {
let db = await lazy.PlacesUtils.promiseDBConnection();
let rows = await db.execute(`SELECT count(*) FROM ${tableName}`);
logs.push(
`Table ${tableName} has ${rows[0].getResultByIndex(0)} records`
);
};
try {
params.type = "table";
let db = await lazy.PlacesUtils.promiseDBConnection();
await db.execute(query, params, r =>
_getTableCount(r.getResultByIndex(0))
);
} catch (ex) {
throw new Error("Unable to collect stats.");
}
let details = await PlacesDBUtils.getEntitiesStats();
logs.push(
`Pages sequentiality: ${details.get("moz_places").sequentialityPerc}`
);
let entities = Array.from(details.keys()).sort((a, b) => {
return details.get(a).sizePerc - details.get(b).sizePerc;
});
for (let key of entities) {
let info = details.get(key);
logs.push(
`${key}: ${info.sizeBytes / 1024}KiB (${info.sizePerc}%), ${
info.efficiencyPerc
}% eff.`
);
}
return logs;
},
/**
* Collects telemetry data and reports it to Telemetry.
*
* Note: although this function isn't actually async, we keep it async to
* allow us to maintain a simple, consistent API for the tasks within this object.
*
*/
async telemetry() {
// This will be populated with one integer property for each probe result,
// using the histogram name as key.
let probeValues = {};
// The following array contains an ordered list of entries that are
// processed to collect telemetry data. Each entry has these properties:
//
// histogram: Name of the telemetry histogram to update.
// query: This is optional. If present, contains a database command
// that will be executed asynchronously, and whose result will
// be added to the telemetry histogram.
// callback: This is optional. If present, contains a function that must
// return the value that will be added to the telemetry
// histogram. If a query is also present, its result is passed
// as the first argument of the function. If the function
// raises an exception, no data is added to the histogram.
//
// Since all queries are executed in order by the database backend, the
// callbacks can also use the result of previous queries stored in the
// probeValues object.
let probes = [
{
histogram: "PLACES_PAGES_COUNT",
query: "SELECT count(*) FROM moz_places",
},
{
histogram: "PLACES_BOOKMARKS_COUNT",
query: `SELECT count(*) FROM moz_bookmarks b
JOIN moz_bookmarks t ON t.id = b.parent
AND t.parent <> :tags_folder
WHERE b.type = :type_bookmark`,
params: {
tags_folder: lazy.PlacesUtils.tagsFolderId,
type_bookmark: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK,
},
},
{
histogram: "PLACES_TAGS_COUNT",
query: `SELECT count(*) FROM moz_bookmarks
WHERE parent = :tags_folder`,
params: {
tags_folder: lazy.PlacesUtils.tagsFolderId,
},
},
{
histogram: "PLACES_KEYWORDS_COUNT",
query: "SELECT count(*) FROM moz_keywords",
},
{
histogram: "PLACES_SORTED_BOOKMARKS_PERC",
query: `SELECT IFNULL(ROUND((
SELECT count(*) FROM moz_bookmarks b
JOIN moz_bookmarks p ON p.id = b.parent
JOIN moz_bookmarks g ON g.id = p.parent
WHERE g.guid <> :root_guid
AND g.guid <> :tags_guid
AND b.type = :type_bookmark
) * 100 / (
SELECT count(*) FROM moz_bookmarks b
JOIN moz_bookmarks p ON p.id = b.parent
JOIN moz_bookmarks g ON g.id = p.parent
AND g.guid <> :tags_guid
WHERE b.type = :type_bookmark
)), 0)`,
params: {
root_guid: lazy.PlacesUtils.bookmarks.rootGuid,
tags_guid: lazy.PlacesUtils.bookmarks.tagsGuid,
type_bookmark: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK,
},
},
{
histogram: "PLACES_TAGGED_BOOKMARKS_PERC",
query: `SELECT IFNULL(ROUND((
SELECT count(*) FROM moz_bookmarks b
JOIN moz_bookmarks t ON t.id = b.parent
AND t.parent = :tags_folder
) * 100 / (
SELECT count(*) FROM moz_bookmarks b
JOIN moz_bookmarks t ON t.id = b.parent
AND t.parent <> :tags_folder
WHERE b.type = :type_bookmark
)), 0)`,
params: {
tags_folder: lazy.PlacesUtils.tagsFolderId,
type_bookmark: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK,
},
},
{
histogram: "PLACES_DATABASE_FILESIZE_MB",
async callback() {
let placesDbPath = PathUtils.join(
PathUtils.profileDir,
"places.sqlite"
);
let info = await IOUtils.stat(placesDbPath);
return parseInt(info.size / BYTES_PER_MEBIBYTE);
},
},
{
histogram: "PLACES_DATABASE_FAVICONS_FILESIZE_MB",
async callback() {
let faviconsDbPath = PathUtils.join(
PathUtils.profileDir,
"favicons.sqlite"
);
let info = await IOUtils.stat(faviconsDbPath);
return parseInt(info.size / BYTES_PER_MEBIBYTE);
},
},
{
histogram: "PLACES_ANNOS_PAGES_COUNT",
query: "SELECT count(*) FROM moz_annos",
},
{
histogram: "PLACES_MAINTENANCE_DAYSFROMLAST",
callback() {
try {
let lastMaintenance = Services.prefs.getIntPref(
"places.database.lastMaintenance"
);
let nowSeconds = parseInt(Date.now() / 1000);
return parseInt((nowSeconds - lastMaintenance) / 86400);
} catch (ex) {
return 60;
}
},
},
{
quantity: Glean.places.pagesNeedFrecencyRecalculation,
query: "SELECT count(*) FROM moz_places WHERE recalc_frecency = 1",
},
{
quantity: Glean.places.previousdayVisits,
query: `SELECT COUNT(*) from moz_places
WHERE hidden=0 AND last_visit_date < (strftime('%s', 'now', 'start of day') * 1000000)
AND last_visit_date > (strftime('%s', 'now', 'start of day', '-1 day') * 1000000)
AND last_visit_date IS NOT NULL;`,
},
];
for (let probe of probes) {
let val;
if ("query" in probe) {
let db = await lazy.PlacesUtils.promiseDBConnection();
val = (
await db.execute(probe.query, probe.params || {})
)[0].getResultByIndex(0);
}
// Report the result of the probe through Telemetry.
// The resulting promise cannot reject.
if ("callback" in probe) {
val = await probe.callback(val);
}
probeValues[probe.histogram || probe.scalar] = val;
if (probe.histogram) {
Services.telemetry.getHistogramById(probe.histogram).add(val);
} else if (probe.quantity) {
probe.quantity.set(val);
} else {
throw new Error("Unknwon telemetry probe type");
}
}
},
/**
* Remove old and useless places.sqlite.corrupt files.
*
* @resolves to an array of logs for this task.
*
*/
async removeOldCorruptDBs() {
let logs = [];
logs.push(
"> Cleanup profile from places.sqlite.corrupt files older than " +
CORRUPT_DB_RETAIN_DAYS +
" days."
);
let re = /places\.sqlite(-\d)?\.corrupt$/;
let currentTime = Date.now();
let children = await IOUtils.getChildren(PathUtils.profileDir);
try {
for (let entry of children) {
let fileInfo = await IOUtils.stat(entry);
let lastModificationDate;
if (fileInfo.type == "regular" && re.test(entry)) {
lastModificationDate = fileInfo.lastModified;
try {
// Convert milliseconds to days.
let days = Math.ceil(
(currentTime - lastModificationDate) / MS_PER_DAY
);
if (days >= CORRUPT_DB_RETAIN_DAYS || days < 0) {
await IOUtils.remove(entry);
}
} catch (error) {
logs.push("Could not remove file: " + entry, error);
}
}
}
} catch (error) {
logs.push("removeOldCorruptDBs failed", error);
}
return logs;
},
/**
* Gets detailed statistics about database entities like tables and indices.
* @returns {Map} a Map by table name, containing an object with the following
* properties:
* - efficiencyPerc: percentage filling of pages, an high
* efficiency means most pages are filled up almost completely.
* This value is not particularly useful with a low number of
* pages.
* - sizeBytes: size of the entity in bytes
* - pages: number of pages of the entity
* - sizePerc: percentage of the total database size
* - sequentialityPerc: percentage of sequential pages, this is
* a global value of the database, thus it's the same for every
* entity, and it can be used to evaluate fragmentation and the
* need for vacuum.
*/
async getEntitiesStats() {
let db = await lazy.PlacesUtils.promiseDBConnection();
let rows = await db.execute(`
/* do not warn (bug no): no need for index */
SELECT name,
round((pgsize - unused) * 100.0 / pgsize, 1) as efficiency_perc,
pgsize as size_bytes, pageno as pages,
round(pgsize * 100.0 / (SELECT sum(pgsize) FROM dbstat WHERE aggregate = TRUE), 1) as size_per
c,
round((
WITH s(row, pageno) AS (
SELECT row_number() OVER (ORDER BY path), pageno FROM dbstat ORDER BY path
)
SELECT sum(s1.pageno+1==s2.pageno)*100.0/count(*)
FROM s AS s1, s AS s2
WHERE s1.row+1=s2.row
), 1) AS sequentiality_perc
FROM dbstat
WHERE aggregate = TRUE
`);
let entitiesByName = new Map();
for (let row of rows) {
let details = {
efficiencyPerc: row.getResultByName("efficiency_perc"),
pages: row.getResultByName("pages"),
sizeBytes: row.getResultByName("size_bytes"),
sizePerc: row.getResultByName("size_perc"),
sequentialityPerc: row.getResultByName("sequentiality_perc"),
};
entitiesByName.set(row.getResultByName("name"), details);
}
return entitiesByName;
},
/**
* Gets detailed statistics about database entities and their respective row
* counts.
* @returns {Array} An array that augments each object returned by
* {@link getEntitiesStats} with the following extra properties:
* - entity: name of the entity
* - count: row count of the entity
*/
async getEntitiesStatsAndCounts() {
let stats = await PlacesDBUtils.getEntitiesStats();
let data = [];
let db = await lazy.PlacesUtils.promiseDBConnection();
for (let [entity, value] of stats) {
let count = "-";
try {
if (
entity.startsWith("moz_") &&
!entity.endsWith("index") &&
entity != "moz_places_visitcount" /* bug in index name */
) {
count = (
await db.execute(`SELECT count(*) FROM ${entity}`)
)[0].getResultByIndex(0);
}
} catch (ex) {
console.error(ex);
}
data.push(Object.assign(value, { entity, count }));
}
return data;
},
/**
* Runs a list of tasks, returning a Map when done.
*
* @param tasks
* Array of tasks to be executed, in form of pointers to methods in
* this module.
* @return {Promise}
* A promise that resolves with a Map[taskName(String) -> Object].
* The Object has the following properties:
* - succeeded: boolean
* - logs: an array of strings containing the messages logged by the task
*/
async runTasks(tasks) {
if (!this._registeredShutdownObserver) {
this._registeredShutdownObserver = true;
lazy.PlacesUtils.registerShutdownFunction(() => {
this._isShuttingDown = true;
});
}
PlacesDBUtils._clearTaskQueue = false;
let tasksMap = new Map();
for (let task of tasks) {
if (PlacesDBUtils._isShuttingDown) {
tasksMap.set(task.name, {
succeeded: false,
logs: ["Shutting down, will not schedule the task."],
});
continue;
}
if (PlacesDBUtils._clearTaskQueue) {
tasksMap.set(task.name, {
succeeded: false,
logs: ["The task queue was cleared by an error in another task."],
});
continue;
}
let result = await task()
.then((logs = [`${task.name} complete`]) => ({ succeeded: true, logs }))
.catch(err => ({ succeeded: false, logs: [err.message] }));
tasksMap.set(task.name, result);
}
return tasksMap;
},
};
async function integrity(dbName) {
async function check(db) {
let row;
await db.execute("PRAGMA integrity_check", null, (r, cancel) => {
row = r;
cancel();
});
return row.getResultByIndex(0) === "ok";
}
// Create a new connection for this check, so we can operate independently
// from a broken Places service.
// openConnection returns an exception with .result == Cr.NS_ERROR_FILE_CORRUPTED,
// we should do the same everywhere we want maintenance to try replacing the
// database on next startup.
let path = PathUtils.join(PathUtils.profileDir, dbName);
let db = await lazy.Sqlite.openConnection({ path });
try {
if (await check(db)) {
return;
}
// We stopped due to an integrity corruption, try to fix it if possible.
// First, try to reindex, this often fixes simple indices problems.
try {
await db.execute("REINDEX");
} catch (ex) {
throw new Components.Exception(
"Impossible to reindex database",
Cr.NS_ERROR_FILE_CORRUPTED
);
}
// Check again.
if (!(await check(db))) {
throw new Components.Exception(
"The database is still corrupt",
Cr.NS_ERROR_FILE_CORRUPTED
);
}
} finally {
await db.close();
}
}
export function PlacesDBUtilsIdleMaintenance() {}
PlacesDBUtilsIdleMaintenance.prototype = {
observe(subject, topic) {
switch (topic) {
case "idle-daily": {
// Once a week run places.sqlite maintenance tasks.
let lastMaintenance = Services.prefs.getIntPref(
"places.database.lastMaintenance",
0
);
let nowSeconds = parseInt(Date.now() / 1000);
if (lastMaintenance < nowSeconds - MAINTENANCE_INTERVAL_SECONDS) {
PlacesDBUtils.maintenanceOnIdle();
}
break;
}
default:
throw new Error("Trying to handle an unknown category.");
}
},
classID: Components.ID("d38926e0-29c1-11eb-8588-0800200c9a66"),
QueryInterface: ChromeUtils.generateQI(["nsIObserver"]),
};
