Quelle  store.rs   Sprache: unbekannt

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

use std::{
    collections::{hash_map::Entry, BTreeSet, HashMap, HashSet},
    path::{Path, PathBuf},
    sync::Arc,
};

use error_support::{breadcrumb, handle_error};
use once_cell::sync::OnceCell;
use parking_lot::Mutex;
use remote_settings::{self, RemoteSettingsConfig, RemoteSettingsServer};

use serde::de::DeserializeOwned;

use crate::{
    config::{SuggestGlobalConfig, SuggestProviderConfig},
    db::{ConnectionType, IngestedRecord, Sqlite3Extension, SuggestDao, SuggestDb},
    error::Error,
    geoname::{Geoname, GeonameMatch, GeonameType},
    metrics::{MetricsContext, SuggestIngestionMetrics, SuggestQueryMetrics},
    provider::{SuggestionProvider, SuggestionProviderConstraints, DEFAULT_INGEST_PROVIDERS},
    rs::{
        Client, Collection, DownloadedExposureRecord, Record, RemoteSettingsClient,
        SuggestAttachment, SuggestRecord, SuggestRecordId, SuggestRecordType,
    },
    suggestion::AmpSuggestionType,
    QueryWithMetricsResult, Result, SuggestApiResult, Suggestion, SuggestionQuery,
};

/// Builder for [SuggestStore]
///
/// Using a builder is preferred to calling the constructor directly since it's harder to confuse
/// the data_path and cache_path strings.
#[derive(uniffi::Object)]
pub struct SuggestStoreBuilder(Mutex<SuggestStoreBuilderInner>);

#[derive(Default)]
struct SuggestStoreBuilderInner {
    data_path: Option<String>,
    remote_settings_server: Option<RemoteSettingsServer>,
    remote_settings_bucket_name: Option<String>,
    extensions_to_load: Vec<Sqlite3Extension>,
}

impl Default for SuggestStoreBuilder {
    fn default() -> Self {
        Self::new()
    }
}

#[uniffi::export]
impl SuggestStoreBuilder {
    #[uniffi::constructor]
    pub fn new() -> SuggestStoreBuilder {
        Self(Mutex::new(SuggestStoreBuilderInner::default()))
    }

    pub fn data_path(self: Arc<Self>, path: String) -> Arc<Self> {
        self.0.lock().data_path = Some(path);
        self
    }

    /// Deprecated: this is no longer used by the suggest component.
    pub fn cache_path(self: Arc<Self>, _path: String) -> Arc<Self> {
        // We used to use this, but we're not using it anymore, just ignore the call
        self
    }

    pub fn remote_settings_server(self: Arc<Self>, server: RemoteSettingsServer) -> Arc<Self> {
        self.0.lock().remote_settings_server = Some(server);
        self
    }

    pub fn remote_settings_bucket_name(self: Arc<Self>, bucket_name: String) -> Arc<Self> {
        self.0.lock().remote_settings_bucket_name = Some(bucket_name);
        self
    }

    /// Add an sqlite3 extension to load
    ///
    /// library_name should be the name of the library without any extension, for example `libmozsqlite3`.
    /// entrypoint should be the entry point, for example `sqlite3_fts5_init`.  If `null` (the default)
    /// entry point will be used (see https://sqlite.org/loadext.html for details).
    pub fn load_extension(
        self: Arc<Self>,
        library: String,
        entry_point: Option<String>,
    ) -> Arc<Self> {
        self.0.lock().extensions_to_load.push(Sqlite3Extension {
            library,
            entry_point,
        });
        self
    }

    #[handle_error(Error)]
    pub fn build(&self) -> SuggestApiResult<Arc<SuggestStore>> {
        let inner = self.0.lock();
        let extensions_to_load = inner.extensions_to_load.clone();
        let data_path = inner
            .data_path
            .clone()
            .ok_or_else(|| Error::SuggestStoreBuilder("data_path not specified".to_owned()))?;

        let client = RemoteSettingsClient::new(
            inner.remote_settings_server.clone(),
            inner.remote_settings_bucket_name.clone(),
            None,
        )?;

        Ok(Arc::new(SuggestStore {
            inner: SuggestStoreInner::new(data_path, extensions_to_load, client),
        }))
    }
}

/// What should be interrupted when [SuggestStore::interrupt] is called?
#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, uniffi::Enum)]
pub enum InterruptKind {
    /// Interrupt read operations like [SuggestStore::query]
    Read,
    /// Interrupt write operations.  This mostly means [SuggestStore::ingest], but
    /// [SuggestStore::dismiss_suggestion] may also be interrupted.
    Write,
    /// Interrupt both read and write operations,
    ReadWrite,
}

/// The store is the entry point to the Suggest component. It incrementally
/// downloads suggestions from the Remote Settings service, stores them in a
/// local database, and returns them in response to user queries.
///
/// Your application should create a single store, and manage it as a singleton.
/// The store is thread-safe, and supports concurrent queries and ingests. We
/// expect that your application will call [`SuggestStore::query()`] to show
/// suggestions as the user types into the address bar, and periodically call
/// [`SuggestStore::ingest()`] in the background to update the database with
/// new suggestions from Remote Settings.
///
/// For responsiveness, we recommend always calling `query()` on a worker
/// thread. When the user types new input into the address bar, call
/// [`SuggestStore::interrupt()`] on the main thread to cancel the query
/// for the old input, and unblock the worker thread for the new query.
///
/// The store keeps track of the state needed to support incremental ingestion,
/// but doesn't schedule the ingestion work itself, or decide how many
/// suggestions to ingest at once. This is for two reasons:
///
/// 1. The primitives for scheduling background work vary between platforms, and
///    aren't available to the lower-level Rust layer. You might use an idle
///    timer on Desktop, `WorkManager` on Android, or `BGTaskScheduler` on iOS.
/// 2. Ingestion constraints can change, depending on the platform and the needs
///    of your application. A mobile device on a metered connection might want
///    to request a small subset of the Suggest data and download the rest
///    later, while a desktop on a fast link might download the entire dataset
///    on the first launch.
#[derive(uniffi::Object)]
pub struct SuggestStore {
    inner: SuggestStoreInner<RemoteSettingsClient>,
}

#[uniffi::export]
impl SuggestStore {
    /// Creates a Suggest store.
    #[handle_error(Error)]
    #[uniffi::constructor(default(settings_config = None))]
    pub fn new(
        path: &str,
        settings_config: Option<RemoteSettingsConfig>,
    ) -> SuggestApiResult<Self> {
        let client = match settings_config {
            Some(settings_config) => RemoteSettingsClient::new(
                settings_config.server,
                settings_config.bucket_name,
                settings_config.server_url,
                // Note: collection name is ignored, since we fetch from multiple collections
                // (fakespot-suggest-products and quicksuggest).  No consumer sets it to a
                // non-default value anyways.
            )?,
            None => RemoteSettingsClient::new(None, None, None)?,
        };

        Ok(Self {
            inner: SuggestStoreInner::new(path.to_owned(), vec![], client),
        })
    }

    /// Queries the database for suggestions.
    #[handle_error(Error)]
    pub fn query(&self, query: SuggestionQuery) -> SuggestApiResult<Vec<Suggestion>> {
        Ok(self.inner.query(query)?.suggestions)
    }

    /// Queries the database for suggestions.
    #[handle_error(Error)]
    pub fn query_with_metrics(
        &self,
        query: SuggestionQuery,
    ) -> SuggestApiResult<QueryWithMetricsResult> {
        self.inner.query(query)
    }

    /// Dismiss a suggestion
    ///
    /// Dismissed suggestions will not be returned again
    ///
    /// In the case of AMP suggestions this should be the raw URL.
    #[handle_error(Error)]
    pub fn dismiss_suggestion(&self, suggestion_url: String) -> SuggestApiResult<()> {
        self.inner.dismiss_suggestion(suggestion_url)
    }

    /// Clear dismissed suggestions
    #[handle_error(Error)]
    pub fn clear_dismissed_suggestions(&self) -> SuggestApiResult<()> {
        self.inner.clear_dismissed_suggestions()
    }

    /// Interrupts any ongoing queries.
    ///
    /// This should be called when the user types new input into the address
    /// bar, to ensure that they see fresh suggestions as they type. This
    /// method does not interrupt any ongoing ingests.
    #[uniffi::method(default(kind = None))]
    pub fn interrupt(&self, kind: Option<InterruptKind>) {
        self.inner.interrupt(kind)
    }

    /// Ingests new suggestions from Remote Settings.
    #[handle_error(Error)]
    pub fn ingest(
        &self,
        constraints: SuggestIngestionConstraints,
    ) -> SuggestApiResult<SuggestIngestionMetrics> {
        self.inner.ingest(constraints)
    }

    /// Removes all content from the database.
    #[handle_error(Error)]
    pub fn clear(&self) -> SuggestApiResult<()> {
        self.inner.clear()
    }

    /// Returns global Suggest configuration data.
    #[handle_error(Error)]
    pub fn fetch_global_config(&self) -> SuggestApiResult<SuggestGlobalConfig> {
        self.inner.fetch_global_config()
    }

    /// Returns per-provider Suggest configuration data.
    #[handle_error(Error)]
    pub fn fetch_provider_config(
        &self,
        provider: SuggestionProvider,
    ) -> SuggestApiResult<Option<SuggestProviderConfig>> {
        self.inner.fetch_provider_config(provider)
    }

    /// Fetches geonames stored in the database. A geoname represents a
    /// geographic place.
    ///
    /// `query` is a string that will be matched directly against geoname names.
    /// It is not a query string in the usual Suggest sense. `match_name_prefix`
    /// determines whether prefix matching is performed on names excluding
    /// abbreviations and airport codes. When `true`, names that start with
    /// `query` will match. When false, names that equal `query` will match.
    ///
    /// `geoname_type` restricts returned geonames to a [`GeonameType`].
    ///
    /// `filter` restricts returned geonames to certain cities or regions.
    /// Cities can be restricted to regions by including the regions in
    /// `filter`, and regions can be restricted to those containing certain
    /// cities by including the cities in `filter`. This is especially useful
    /// since city and region names are not unique. `filter` is disjunctive: If
    /// any item in `filter` matches a geoname, the geoname will be filtered in.
    ///
    /// The query can match a single geoname in more than one way. For example,
    /// it can match both a full name and an abbreviation. The returned vec of
    /// [`GeonameMatch`] values will include all matches for a geoname, one
    /// match per `match_type` per geoname. In other words, a matched geoname
    /// can map to more than one `GeonameMatch`.
    #[handle_error(Error)]
    pub fn fetch_geonames(
        &self,
        query: &str,
        match_name_prefix: bool,
        geoname_type: Option<GeonameType>,
        filter: Option<Vec<Geoname>>,
    ) -> SuggestApiResult<Vec<GeonameMatch>> {
        self.inner
            .fetch_geonames(query, match_name_prefix, geoname_type, filter)
    }
}

impl SuggestStore {
    pub fn force_reingest(&self) {
        self.inner.force_reingest()
    }
}

#[cfg(feature = "benchmark_api")]
impl SuggestStore {
    /// Creates a WAL checkpoint. This will cause changes in the write-ahead log
    /// to be written to the DB. See:
    /// https://sqlite.org/pragma.html#pragma_wal_checkpoint
    pub fn checkpoint(&self) {
        self.inner.checkpoint();
    }
}

/// Constraints limit which suggestions to ingest from Remote Settings.
#[derive(Clone, Default, Debug, uniffi::Record)]
pub struct SuggestIngestionConstraints {
    #[uniffi(default = None)]
    pub providers: Option<Vec<SuggestionProvider>>,
    #[uniffi(default = None)]
    pub provider_constraints: Option<SuggestionProviderConstraints>,
    /// Only run ingestion if the table `suggestions` is empty
    ///
    // This is indented to handle periodic updates.  Consumers can schedule an ingest with
    // `empty_only=true` on startup and a regular ingest with `empty_only=false` to run on a long periodic schedule (maybe
    // once a day). This allows ingestion to normally be run at a slow, periodic rate.  However, if
    // there is a schema upgrade that causes the database to be thrown away, then the
    // `empty_only=true` ingestion that runs on startup will repopulate it.
    #[uniffi(default = false)]
    pub empty_only: bool,
}

impl SuggestIngestionConstraints {
    pub fn all_providers() -> Self {
        Self {
            providers: Some(vec![
                SuggestionProvider::Amp,
                SuggestionProvider::Wikipedia,
                SuggestionProvider::Amo,
                SuggestionProvider::Pocket,
                SuggestionProvider::Yelp,
                SuggestionProvider::Mdn,
                SuggestionProvider::Weather,
                SuggestionProvider::AmpMobile,
                SuggestionProvider::Fakespot,
                SuggestionProvider::Exposure,
            ]),
            ..Self::default()
        }
    }

    fn matches_exposure_record(&self, record: &DownloadedExposureRecord) -> bool {
        match self
            .provider_constraints
            .as_ref()
            .and_then(|c| c.exposure_suggestion_types.as_ref())
        {
            None => false,
            Some(suggestion_types) => suggestion_types
                .iter()
                .any(|t| *t == record.suggestion_type),
        }
    }

    fn amp_matching_uses_fts(&self) -> bool {
        self.provider_constraints
            .as_ref()
            .and_then(|c| c.amp_alternative_matching.as_ref())
            .map(|constraints| constraints.uses_fts())
            .unwrap_or(false)
    }
}

/// The implementation of the store. This is generic over the Remote Settings
/// client, and is split out from the concrete [`SuggestStore`] for testing
/// with a mock client.
pub(crate) struct SuggestStoreInner<S> {
    /// Path to the persistent SQL database.
    ///
    /// This stores things that should persist when the user clears their cache.
    /// It's not currently used because not all consumers pass this in yet.
    #[allow(unused)]
    data_path: PathBuf,
    dbs: OnceCell<SuggestStoreDbs>,
    extensions_to_load: Vec<Sqlite3Extension>,
    settings_client: S,
}

impl<S> SuggestStoreInner<S> {
    pub fn new(
        data_path: impl Into<PathBuf>,
        extensions_to_load: Vec<Sqlite3Extension>,
        settings_client: S,
    ) -> Self {
        Self {
            data_path: data_path.into(),
            extensions_to_load,
            dbs: OnceCell::new(),
            settings_client,
        }
    }

    /// Returns this store's database connections, initializing them if
    /// they're not already open.
    fn dbs(&self) -> Result<&SuggestStoreDbs> {
        self.dbs
            .get_or_try_init(|| SuggestStoreDbs::open(&self.data_path, &self.extensions_to_load))
    }

    fn query(&self, query: SuggestionQuery) -> Result<QueryWithMetricsResult> {
        let mut metrics = SuggestQueryMetrics::default();
        let mut suggestions = vec![];

        let unique_providers = query.providers.iter().collect::<HashSet<_>>();
        let reader = &self.dbs()?.reader;
        for provider in unique_providers {
            let new_suggestions = metrics.measure_query(provider.to_string(), || {
                reader.read(|dao| match provider {
                    SuggestionProvider::Amp => {
                        dao.fetch_amp_suggestions(&query, AmpSuggestionType::Desktop)
                    }
                    SuggestionProvider::AmpMobile => {
                        dao.fetch_amp_suggestions(&query, AmpSuggestionType::Mobile)
                    }
                    SuggestionProvider::Wikipedia => dao.fetch_wikipedia_suggestions(&query),
                    SuggestionProvider::Amo => dao.fetch_amo_suggestions(&query),
                    SuggestionProvider::Pocket => dao.fetch_pocket_suggestions(&query),
                    SuggestionProvider::Yelp => dao.fetch_yelp_suggestions(&query),
                    SuggestionProvider::Mdn => dao.fetch_mdn_suggestions(&query),
                    SuggestionProvider::Weather => dao.fetch_weather_suggestions(&query),
                    SuggestionProvider::Fakespot => dao.fetch_fakespot_suggestions(&query),
                    SuggestionProvider::Exposure => dao.fetch_exposure_suggestions(&query),
                })
            })?;
            suggestions.extend(new_suggestions);
        }

        // Note: it's important that this is a stable sort to keep the intra-provider order stable.
        // For example, we can return multiple fakespot-suggestions all with `score=0.245`.  In
        // that case, they must be in the same order that `fetch_fakespot_suggestions` returned
        // them in.
        suggestions.sort();
        if let Some(limit) = query.limit.and_then(|limit| usize::try_from(limit).ok()) {
            suggestions.truncate(limit);
        }
        Ok(QueryWithMetricsResult {
            suggestions,
            query_times: metrics.times,
        })
    }

    fn dismiss_suggestion(&self, suggestion_url: String) -> Result<()> {
        self.dbs()?
            .writer
            .write(|dao| dao.insert_dismissal(&suggestion_url))
    }

    fn clear_dismissed_suggestions(&self) -> Result<()> {
        self.dbs()?.writer.write(|dao| dao.clear_dismissals())?;
        Ok(())
    }

    fn interrupt(&self, kind: Option<InterruptKind>) {
        if let Some(dbs) = self.dbs.get() {
            // Only interrupt if the databases are already open.
            match kind.unwrap_or(InterruptKind::Read) {
                InterruptKind::Read => {
                    dbs.reader.interrupt_handle.interrupt();
                }
                InterruptKind::Write => {
                    dbs.writer.interrupt_handle.interrupt();
                }
                InterruptKind::ReadWrite => {
                    dbs.reader.interrupt_handle.interrupt();
                    dbs.writer.interrupt_handle.interrupt();
                }
            }
        }
    }

    fn clear(&self) -> Result<()> {
        self.dbs()?.writer.write(|dao| dao.clear())
    }

    pub fn fetch_global_config(&self) -> Result<SuggestGlobalConfig> {
        self.dbs()?.reader.read(|dao| dao.get_global_config())
    }

    pub fn fetch_provider_config(
        &self,
        provider: SuggestionProvider,
    ) -> Result<Option<SuggestProviderConfig>> {
        self.dbs()?
            .reader
            .read(|dao| dao.get_provider_config(provider))
    }

    // Cause the next ingestion to re-ingest all data
    pub fn force_reingest(&self) {
        let writer = &self.dbs().unwrap().writer;
        writer.write(|dao| dao.force_reingest()).unwrap();
    }

    fn fetch_geonames(
        &self,
        query: &str,
        match_name_prefix: bool,
        geoname_type: Option<GeonameType>,
        filter: Option<Vec<Geoname>>,
    ) -> Result<Vec<GeonameMatch>> {
        self.dbs()?.reader.read(|dao| {
            dao.fetch_geonames(
                query,
                match_name_prefix,
                geoname_type,
                filter.as_ref().map(|f| f.iter().collect()),
            )
        })
    }
}

impl<S> SuggestStoreInner<S>
where
    S: Client,
{
    pub fn ingest(
        &self,
        constraints: SuggestIngestionConstraints,
    ) -> Result<SuggestIngestionMetrics> {
        breadcrumb!("Ingestion starting");
        let writer = &self.dbs()?.writer;
        let mut metrics = SuggestIngestionMetrics::default();
        if constraints.empty_only && !writer.read(|dao| dao.suggestions_table_empty())? {
            return Ok(metrics);
        }

        // Figure out which record types we're ingesting and group them by
        // collection. A record type may be used by multiple providers, but we
        // want to ingest each one at most once.
        let mut record_types_by_collection = HashMap::<Collection, BTreeSet<_>>::new();
        for p in constraints
            .providers
            .as_ref()
            .unwrap_or(&DEFAULT_INGEST_PROVIDERS.to_vec())
            .iter()
        {
            for t in p.record_types() {
                record_types_by_collection
                    .entry(t.collection())
                    .or_default()
                    .insert(t);
            }
        }

        // Always ingest these record types.
        for rt in [SuggestRecordType::Icon, SuggestRecordType::GlobalConfig] {
            record_types_by_collection
                .entry(rt.collection())
                .or_default()
                .insert(rt);
        }

        // Create a single write scope for all DB operations
        let mut write_scope = writer.write_scope()?;

        // Read the previously ingested records.  We use this to calculate what's changed
        let ingested_records = write_scope.read(|dao| dao.get_ingested_records())?;

        // For each collection, fetch all records
        for (collection, record_types) in record_types_by_collection {
            breadcrumb!("Ingesting collection {}", collection.name());
            let records =
                write_scope.write(|dao| self.settings_client.get_records(collection, dao))?;

            // For each record type in that collection, calculate the changes and pass them to
            // [Self::ingest_records]
            for record_type in record_types {
                breadcrumb!("Ingesting record_type: {record_type}");
                metrics.measure_ingest(record_type.to_string(), |context| {
                    let changes = RecordChanges::new(
                        records.iter().filter(|r| r.record_type() == record_type),
                        ingested_records.iter().filter(|i| {
                            i.record_type == record_type.as_str()
                                && i.collection == collection.name()
                        }),
                    );
                    write_scope.write(|dao| {
                        self.process_changes(dao, collection, changes, &constraints, context)
                    })
                })?;
                write_scope.err_if_interrupted()?;
            }
        }
        breadcrumb!("Ingestion complete");

        Ok(metrics)
    }

    fn process_changes(
        &self,
        dao: &mut SuggestDao,
        collection: Collection,
        changes: RecordChanges<'_>,
        constraints: &SuggestIngestionConstraints,
        context: &mut MetricsContext,
    ) -> Result<()> {
        for record in &changes.new {
            log::trace!("Ingesting record ID: {}", record.id.as_str());
            self.process_record(dao, record, constraints, context)?;
        }
        for record in &changes.updated {
            // Drop any data that we previously ingested from this record.
            // Suggestions in particular don't have a stable identifier, and
            // determining which suggestions in the record actually changed is
            // more complicated than dropping and re-ingesting all of them.
            log::trace!("Reingesting updated record ID: {}", record.id.as_str());
            dao.delete_record_data(&record.id)?;
            self.process_record(dao, record, constraints, context)?;
        }
        for record in &changes.unchanged {
            if self.should_reprocess_record(dao, record, constraints)? {
                log::trace!("Reingesting unchanged record ID: {}", record.id.as_str());
                self.process_record(dao, record, constraints, context)?;
            } else {
                log::trace!("Skipping unchanged record ID: {}", record.id.as_str());
            }
        }
        for record in &changes.deleted {
            log::trace!("Deleting record ID: {:?}", record.id);
            dao.delete_record_data(&record.id)?;
        }
        dao.update_ingested_records(
            collection.name(),
            &changes.new,
            &changes.updated,
            &changes.deleted,
        )?;
        Ok(())
    }

    fn process_record(
        &self,
        dao: &mut SuggestDao,
        record: &Record,
        constraints: &SuggestIngestionConstraints,
        context: &mut MetricsContext,
    ) -> Result<()> {
        match &record.payload {
            SuggestRecord::AmpWikipedia => {
                self.download_attachment(dao, record, context, |dao, record_id, suggestions| {
                    dao.insert_amp_wikipedia_suggestions(
                        record_id,
                        suggestions,
                        constraints.amp_matching_uses_fts(),
                    )
                })?;
            }
            SuggestRecord::AmpMobile => {
                self.download_attachment(dao, record, context, |dao, record_id, suggestions| {
                    dao.insert_amp_mobile_suggestions(record_id, suggestions)
                })?;
            }
            SuggestRecord::Icon => {
                let (Some(icon_id), Some(attachment)) =
                    (record.id.as_icon_id(), record.attachment.as_ref())
                else {
                    // An icon record should have an icon ID and an
                    // attachment. Icons that don't have these are
                    // malformed, so skip to the next record.
                    return Ok(());
                };
                let data = context
                    .measure_download(|| self.settings_client.download_attachment(record))?;
                dao.put_icon(icon_id, &data, &attachment.mimetype)?;
            }
            SuggestRecord::Amo => {
                self.download_attachment(dao, record, context, |dao, record_id, suggestions| {
                    dao.insert_amo_suggestions(record_id, suggestions)
                })?;
            }
            SuggestRecord::Pocket => {
                self.download_attachment(dao, record, context, |dao, record_id, suggestions| {
                    dao.insert_pocket_suggestions(record_id, suggestions)
                })?;
            }
            SuggestRecord::Yelp => {
                self.download_attachment(dao, record, context, |dao, record_id, suggestions| {
                    match suggestions.first() {
                        Some(suggestion) => dao.insert_yelp_suggestions(record_id, suggestion),
                        None => Ok(()),
                    }
                })?;
            }
            SuggestRecord::Mdn => {
                self.download_attachment(dao, record, context, |dao, record_id, suggestions| {
                    dao.insert_mdn_suggestions(record_id, suggestions)
                })?;
            }
            SuggestRecord::Weather => self.process_weather_record(dao, record, context)?,
            SuggestRecord::GlobalConfig(config) => {
                dao.put_global_config(&SuggestGlobalConfig::from(config))?
            }
            SuggestRecord::Fakespot => {
                self.download_attachment(dao, record, context, |dao, record_id, suggestions| {
                    dao.insert_fakespot_suggestions(record_id, suggestions)
                })?;
            }
            SuggestRecord::Exposure(r) => {
                if constraints.matches_exposure_record(r) {
                    self.download_attachment(
                        dao,
                        record,
                        context,
                        |dao, record_id, suggestions| {
                            dao.insert_exposure_suggestions(
                                record_id,
                                &r.suggestion_type,
                                suggestions,
                            )
                        },
                    )?;
                }
            }
            SuggestRecord::Geonames => self.process_geoname_record(dao, record, context)?,
        }
        Ok(())
    }

    pub(crate) fn download_attachment<T>(
        &self,
        dao: &mut SuggestDao,
        record: &Record,
        context: &mut MetricsContext,
        ingestion_handler: impl FnOnce(&mut SuggestDao<'_>, &SuggestRecordId, &[T]) -> Result<()>,
    ) -> Result<()>
    where
        T: DeserializeOwned,
    {
        if record.attachment.is_none() {
            return Ok(());
        };

        let attachment_data =
            context.measure_download(|| self.settings_client.download_attachment(record))?;
        match serde_json::from_slice::<SuggestAttachment<T>>(&attachment_data) {
            Ok(attachment) => ingestion_handler(dao, &record.id, attachment.suggestions()),
            // If the attachment doesn't match our expected schema, just skip it.  It's possible
            // that we're using an older version.  If so, we'll get the data when we re-ingest
            // after updating the schema.
            Err(_) => Ok(()),
        }
    }

    fn should_reprocess_record(
        &self,
        dao: &mut SuggestDao,
        record: &Record,
        constraints: &SuggestIngestionConstraints,
    ) -> Result<bool> {
        match &record.payload {
            SuggestRecord::Exposure(r) => {
                // Even though the record was previously ingested, its
                // suggestion wouldn't have been if it never matched the
                // provider constraints of any ingest. Return true if the
                // suggestion is not ingested and the provider constraints of
                // the current ingest do match the suggestion.
                Ok(!dao.is_exposure_suggestion_ingested(&record.id)?
                    && constraints.matches_exposure_record(r))
            }
            SuggestRecord::AmpWikipedia => {
                Ok(constraints.amp_matching_uses_fts()
                    && !dao.is_amp_fts_data_ingested(&record.id)?)
            }
            _ => Ok(false),
        }
    }
}

/// Tracks changes in suggest records since the last ingestion
struct RecordChanges<'a> {
    new: Vec<&'a Record>,
    updated: Vec<&'a Record>,
    deleted: Vec<&'a IngestedRecord>,
    unchanged: Vec<&'a Record>,
}

impl<'a> RecordChanges<'a> {
    fn new(
        current: impl Iterator<Item = &'a Record>,
        previously_ingested: impl Iterator<Item = &'a IngestedRecord>,
    ) -> Self {
        let mut ingested_map: HashMap<&str, &IngestedRecord> =
            previously_ingested.map(|i| (i.id.as_str(), i)).collect();
        // Iterate through current, finding new/updated records.
        // Remove existing records from ingested_map.
        let mut new = vec![];
        let mut updated = vec![];
        let mut unchanged = vec![];
        for r in current {
            match ingested_map.entry(r.id.as_str()) {
                Entry::Vacant(_) => new.push(r),
                Entry::Occupied(e) => {
                    if e.remove().last_modified != r.last_modified {
                        updated.push(r);
                    } else {
                        unchanged.push(r);
                    }
                }
            }
        }
        // Anything left in ingested_map is a deleted record
        let deleted = ingested_map.into_values().collect();
        Self {
            new,
            deleted,
            updated,
            unchanged,
        }
    }
}

#[cfg(feature = "benchmark_api")]
impl<S> SuggestStoreInner<S>
where
    S: Client,
{
    pub fn into_settings_client(self) -> S {
        self.settings_client
    }

    pub fn ensure_db_initialized(&self) {
        self.dbs().unwrap();
    }

    fn checkpoint(&self) {
        let conn = self.dbs().unwrap().writer.conn.lock();
        conn.pragma_update(None, "wal_checkpoint", "TRUNCATE")
            .expect("Error performing checkpoint");
    }

    pub fn ingest_records_by_type(&self, ingest_record_type: SuggestRecordType) {
        let writer = &self.dbs().unwrap().writer;
        let mut context = MetricsContext::default();
        let ingested_records = writer.read(|dao| dao.get_ingested_records()).unwrap();
        let records = writer
            .write(|dao| {
                self.settings_client
                    .get_records(ingest_record_type.collection(), dao)
            })
            .unwrap();

        let changes = RecordChanges::new(
            records
                .iter()
                .filter(|r| r.record_type() == ingest_record_type),
            ingested_records
                .iter()
                .filter(|i| i.record_type == ingest_record_type.as_str()),
        );
        writer
            .write(|dao| {
                self.process_changes(
                    dao,
                    ingest_record_type.collection(),
                    changes,
                    &SuggestIngestionConstraints::default(),
                    &mut context,
                )
            })
            .unwrap();
    }

    pub fn table_row_counts(&self) -> Vec<(String, u32)> {
        use sql_support::ConnExt;

        // Note: since this is just used for debugging, use unwrap to simplify the error handling.
        let reader = &self.dbs().unwrap().reader;
        let conn = reader.conn.lock();
        let table_names: Vec<String> = conn
            .query_rows_and_then(
                "SELECT name FROM sqlite_master where type = 'table'",
                (),
                |row| row.get(0),
            )
            .unwrap();
        let mut table_names_with_counts: Vec<(String, u32)> = table_names
            .into_iter()
            .map(|name| {
                let count: u32 = conn
                    .query_one(&format!("SELECT COUNT(*) FROM {name}"))
                    .unwrap();
                (name, count)
            })
            .collect();
        table_names_with_counts.sort_by(|a, b| (b.1.cmp(&a.1)));
        table_names_with_counts
    }

    pub fn db_size(&self) -> usize {
        use sql_support::ConnExt;

        let reader = &self.dbs().unwrap().reader;
        let conn = reader.conn.lock();
        conn.query_one("SELECT page_size * page_count FROM pragma_page_count(), pragma_page_size()")
            .unwrap()
    }
}

/// Holds a store's open connections to the Suggest database.
struct SuggestStoreDbs {
    /// A read-write connection used to update the database with new data.
    writer: SuggestDb,
    /// A read-only connection used to query the database.
    reader: SuggestDb,
}

impl SuggestStoreDbs {
    fn open(path: &Path, extensions_to_load: &[Sqlite3Extension]) -> Result<Self> {
        // Order is important here: the writer must be opened first, so that it
        // can set up the database and run any migrations.
        let writer = SuggestDb::open(path, extensions_to_load, ConnectionType::ReadWrite)?;
        let reader = SuggestDb::open(path, extensions_to_load, ConnectionType::ReadOnly)?;
        Ok(Self { writer, reader })
    }
}

#[cfg(test)]
pub(crate) mod tests {
    use super::*;

    use std::sync::atomic::{AtomicUsize, Ordering};

    use crate::{
        provider::AmpMatchingStrategy, suggestion::FtsMatchInfo, testing::*, SuggestionProvider,
    };

    // Extra methods for the tests
    impl SuggestIngestionConstraints {
        fn amp_with_fts() -> Self {
            Self {
                providers: Some(vec![SuggestionProvider::Amp]),
                provider_constraints: Some(SuggestionProviderConstraints {
                    amp_alternative_matching: Some(AmpMatchingStrategy::FtsAgainstFullKeywords),
                    ..SuggestionProviderConstraints::default()
                }),
                ..Self::default()
            }
        }
        fn amp_without_fts() -> Self {
            Self {
                providers: Some(vec![SuggestionProvider::Amp]),
                ..Self::default()
            }
        }
    }

    /// In-memory Suggest store for testing
    pub(crate) struct TestStore {
        pub inner: SuggestStoreInner<MockRemoteSettingsClient>,
    }

    impl TestStore {
        pub fn new(client: MockRemoteSettingsClient) -> Self {
            static COUNTER: AtomicUsize = AtomicUsize::new(0);
            let db_path = format!(
                "file:test_store_data_{}?mode=memory&cache=shared",
                COUNTER.fetch_add(1, Ordering::Relaxed),
            );
            Self {
                inner: SuggestStoreInner::new(db_path, vec![], client),
            }
        }

        pub fn client_mut(&mut self) -> &mut MockRemoteSettingsClient {
            &mut self.inner.settings_client
        }

        pub fn read<T>(&self, op: impl FnOnce(&SuggestDao) -> Result<T>) -> Result<T> {
            self.inner.dbs().unwrap().reader.read(op)
        }

        pub fn count_rows(&self, table_name: &str) -> u64 {
            let sql = format!("SELECT count(*) FROM {table_name}");
            self.read(|dao| Ok(dao.conn.query_one(&sql)?))
                .unwrap_or_else(|e| panic!("SQL error in count: {e}"))
        }

        pub fn ingest(&self, constraints: SuggestIngestionConstraints) {
            self.inner.ingest(constraints).unwrap();
        }

        pub fn fetch_suggestions(&self, query: SuggestionQuery) -> Vec<Suggestion> {
            self.inner.query(query).unwrap().suggestions
        }

        pub fn fetch_global_config(&self) -> SuggestGlobalConfig {
            self.inner
                .fetch_global_config()
                .expect("Error fetching global config")
        }

        pub fn fetch_provider_config(
            &self,
            provider: SuggestionProvider,
        ) -> Option<SuggestProviderConfig> {
            self.inner
                .fetch_provider_config(provider)
                .expect("Error fetching provider config")
        }

        pub fn fetch_geonames(
            &self,
            query: &str,
            match_name_prefix: bool,
            geoname_type: Option<GeonameType>,
            filter: Option<Vec<Geoname>>,
        ) -> Vec<GeonameMatch> {
            self.inner
                .fetch_geonames(query, match_name_prefix, geoname_type, filter)
                .expect("Error fetching geonames")
        }
    }

    /// Tests that `SuggestStore` is usable with UniFFI, which requires exposed
    /// interfaces to be `Send` and `Sync`.
    #[test]
    fn is_thread_safe() {
        before_each();

        fn is_send_sync<T: Send + Sync>() {}
        is_send_sync::<SuggestStore>();
    }

    /// Tests ingesting suggestions into an empty database.
    #[test]
    fn ingest_suggestions() -> anyhow::Result<()> {
        before_each();

        let store = TestStore::new(
            MockRemoteSettingsClient::default()
                .with_record("data", "1234", json![los_pollos_amp()])
                .with_icon(los_pollos_icon()),
        );
        store.ingest(SuggestIngestionConstraints::all_providers());
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::amp("lo")),
            vec![los_pollos_suggestion("los", None)],
        );
        Ok(())
    }

    /// Tests ingesting suggestions into an empty database.
    #[test]
    fn ingest_empty_only() -> anyhow::Result<()> {
        before_each();

        let mut store = TestStore::new(MockRemoteSettingsClient::default().with_record(
            "data",
            "1234",
            json![los_pollos_amp()],
        ));
        // suggestions_table_empty returns true before the ingestion is complete
        assert!(store.read(|dao| dao.suggestions_table_empty())?);
        // This ingestion should run, since the DB is empty
        store.ingest(SuggestIngestionConstraints {
            empty_only: true,
            ..SuggestIngestionConstraints::all_providers()
        });
        // suggestions_table_empty returns false after the ingestion is complete
        assert!(!store.read(|dao| dao.suggestions_table_empty())?);

        // This ingestion should not run since the DB is no longer empty
        store.client_mut().update_record(
            "data",
            "1234",
            json!([los_pollos_amp(), good_place_eats_amp()]),
        );
        store.ingest(SuggestIngestionConstraints {
            empty_only: true,
            ..SuggestIngestionConstraints::all_providers()
        });
        // "la" should not match the good place eats suggestion, since that should not have been
        // ingested.
        assert_eq!(store.fetch_suggestions(SuggestionQuery::amp("la")), vec![]);

        Ok(())
    }

    /// Tests ingesting suggestions with icons.
    #[test]
    fn ingest_amp_icons() -> anyhow::Result<()> {
        before_each();

        let store = TestStore::new(
            MockRemoteSettingsClient::default()
                .with_record(
                    "data",
                    "1234",
                    json!([los_pollos_amp(), good_place_eats_amp()]),
                )
                .with_icon(los_pollos_icon())
                .with_icon(good_place_eats_icon()),
        );
        // This ingestion should run, since the DB is empty
        store.ingest(SuggestIngestionConstraints::all_providers());

        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::amp("lo")),
            vec![los_pollos_suggestion("los", None)]
        );
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::amp("la")),
            vec![good_place_eats_suggestion("lasagna", None)]
        );

        Ok(())
    }

    #[test]
    fn ingest_full_keywords() -> anyhow::Result<()> {
        before_each();

        let store = TestStore::new(MockRemoteSettingsClient::default()
            .with_record("data", "1234", json!([
                // AMP attachment with full keyword data
                los_pollos_amp().merge(json!({
                    "keywords": ["lo", "los", "los p", "los pollos", "los pollos h", "los pollos hermanos"],
                    "full_keywords": [
                        // Full keyword for the first 4 keywords
                        ("los pollos", 4),
                        // Full keyword for the next 2 keywords
                        ("los pollos hermanos (restaurant)", 2),
                    ],
                })),
                // AMP attachment without full keyword data
                good_place_eats_amp(),
                // Wikipedia attachment with full keyword data.  We should ignore the full
                // keyword data for Wikipedia suggestions
                california_wiki(),
                // california_wiki().merge(json!({
                //     "keywords": ["cal", "cali", "california"],
                //     "full_keywords": [("california institute of technology", 3)],
                // })),
            ]))
            .with_record("amp-mobile-suggestions", "2468", json!([
                // Amp mobile attachment with full keyword data
                a1a_amp_mobile().merge(json!({
                    "keywords": ["a1a", "ca", "car", "car wash"],
                    "full_keywords": [
                        ("A1A Car Wash", 1),
                        ("car wash", 3),
                    ],
                })),
            ]))
            .with_icon(los_pollos_icon())
            .with_icon(good_place_eats_icon())
            .with_icon(california_icon())
        );
        store.ingest(SuggestIngestionConstraints::all_providers());

        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::amp("lo")),
            // This keyword comes from the provided full_keywords list
            vec![los_pollos_suggestion("los pollos", None)],
        );

        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::amp("la")),
            // Good place eats did not have full keywords, so this one is calculated with the
            // keywords.rs code
            vec![good_place_eats_suggestion("lasagna", None)],
        );

        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::wikipedia("cal")),
            // Even though this had a full_keywords field, we should ignore it since it's a
            // wikipedia suggestion and use the keywords.rs code instead
            vec![california_suggestion("california")],
        );

        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::amp_mobile("a1a")),
            // This keyword comes from the provided full_keywords list.
            vec![a1a_suggestion("A1A Car Wash", None)],
        );

        Ok(())
    }

    #[test]
    fn amp_no_keyword_expansion() -> anyhow::Result<()> {
        before_each();

        let store = TestStore::new(
            MockRemoteSettingsClient::default()
                // Setup the keywords such that:
                //   * There's a `chicken` keyword, which is not a substring of any full
                //     keywords (i.e. it was the result of keyword expansion).
                //   * There's a `los pollos ` keyword with an extra space
                .with_record(
                    "data",
                    "1234",
                    los_pollos_amp().merge(json!({
                        "keywords": ["los", "los pollos", "los pollos ", "los pollos hermanos", "chicken"],
                        "full_keywords": [("los pollos", 3), ("los pollos hermanos", 2)],
                    }))
                )
                .with_icon(los_pollos_icon()),
        );
        store.ingest(SuggestIngestionConstraints::all_providers());
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery {
                provider_constraints: Some(SuggestionProviderConstraints {
                    amp_alternative_matching: Some(AmpMatchingStrategy::NoKeywordExpansion),
                    ..SuggestionProviderConstraints::default()
                }),
                // Should not match, because `chicken` is not a substring of a full keyword.
                // i.e. it was added because of keyword expansion.
                ..SuggestionQuery::amp("chicken")
            }),
            vec![],
        );
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery {
                provider_constraints: Some(SuggestionProviderConstraints {
                    amp_alternative_matching: Some(AmpMatchingStrategy::NoKeywordExpansion),
                    ..SuggestionProviderConstraints::default()
                }),
                // Should match, even though "los pollos " technically is not a substring
                // because there's an extra space.  The reason these keywords are in the DB is
                // because we want to keep showing the current suggestion when the user types
                // the space key.
                ..SuggestionQuery::amp("los pollos ")
            }),
            vec![los_pollos_suggestion("los pollos", None)],
        );
        Ok(())
    }

    #[test]
    fn amp_fts_against_full_keywords() -> anyhow::Result<()> {
        before_each();

        let store = TestStore::new(
            MockRemoteSettingsClient::default()
                // Make sure there's full keywords to match against
                .with_record(
                    "data",
                    "1234",
                    los_pollos_amp().merge(json!({
                        "keywords": ["los", "los pollos", "los pollos ", "los pollos hermanos"],
                        "full_keywords": [("los pollos", 3), ("los pollos hermanos", 1)],
                    })),
                )
                .with_icon(los_pollos_icon()),
        );
        store.ingest(SuggestIngestionConstraints::amp_with_fts());
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery {
                provider_constraints: Some(SuggestionProviderConstraints {
                    amp_alternative_matching: Some(AmpMatchingStrategy::FtsAgainstFullKeywords),
                    ..SuggestionProviderConstraints::default()
                }),
                // "Hermanos" should match, even though it's not listed in the keywords,
                // because this strategy uses an FTS match against the full keyword list.
                ..SuggestionQuery::amp("hermanos")
            }),
            vec![los_pollos_suggestion(
                "hermanos",
                Some(FtsMatchInfo {
                    prefix: false,
                    stemming: false,
                })
            )],
        );
        Ok(())
    }

    #[test]
    fn amp_fts_against_title() -> anyhow::Result<()> {
        before_each();

        let store = TestStore::new(
            MockRemoteSettingsClient::default()
                .with_record("data", "1234", los_pollos_amp())
                .with_icon(los_pollos_icon()),
        );
        store.ingest(SuggestIngestionConstraints::amp_with_fts());
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery {
                provider_constraints: Some(SuggestionProviderConstraints {
                    amp_alternative_matching: Some(AmpMatchingStrategy::FtsAgainstTitle),
                    ..SuggestionProviderConstraints::default()
                }),
                // "Albuquerque" should match, even though it's not listed in the keywords,
                // because this strategy uses an FTS match against the title
                ..SuggestionQuery::amp("albuquerque")
            }),
            vec![los_pollos_suggestion(
                "albuquerque",
                Some(FtsMatchInfo {
                    prefix: false,
                    stemming: false,
                })
            )],
        );
        Ok(())
    }

    /// Tests ingesting a data attachment containing a single suggestion,
    /// instead of an array of suggestions.
    #[test]
    fn ingest_one_suggestion_in_data_attachment() -> anyhow::Result<()> {
        before_each();

        let store = TestStore::new(
            MockRemoteSettingsClient::default()
                // This record contains just one JSON object, rather than an array of them
                .with_record("data", "1234", los_pollos_amp())
                .with_icon(los_pollos_icon()),
        );
        store.ingest(SuggestIngestionConstraints::all_providers());
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::amp("lo")),
            vec![los_pollos_suggestion("los", None)],
        );

        Ok(())
    }

    /// Tests re-ingesting suggestions from an updated attachment.
    #[test]
    fn reingest_amp_suggestions() -> anyhow::Result<()> {
        before_each();

        let mut store = TestStore::new(MockRemoteSettingsClient::default().with_record(
            "data",
            "1234",
            json!([los_pollos_amp(), good_place_eats_amp()]),
        ));
        // Ingest once
        store.ingest(SuggestIngestionConstraints::all_providers());
        // Update the snapshot with new suggestions: Los pollos has a new name and Good place eats
        // is now serving Penne
        store.client_mut().update_record(
            "data",
            "1234",
            json!([
                los_pollos_amp().merge(json!({
                    "title": "Los Pollos Hermanos - Now Serving at 14 Locations!",
                })),
                good_place_eats_amp().merge(json!({
                    "keywords": ["pe", "pen", "penne", "penne for your thoughts"],
                    "title": "Penne for Your Thoughts",
                    "url": "https://penne.biz",
                }))
            ]),
        );
        store.ingest(SuggestIngestionConstraints::all_providers());

        assert!(matches!(
            store.fetch_suggestions(SuggestionQuery::amp("lo")).as_slice(),
            [ Suggestion::Amp { title, .. } ] if title == "Los Pollos Hermanos - Now Serving at 14 Locations!",
        ));

        assert_eq!(store.fetch_suggestions(SuggestionQuery::amp("la")), vec![]);
        assert!(matches!(
            store.fetch_suggestions(SuggestionQuery::amp("pe")).as_slice(),
            [ Suggestion::Amp { title, url, .. } ] if title == "Penne for Your Thoughts" && url == "https://penne.biz"
        ));

        Ok(())
    }

    #[test]
    fn reingest_amp_after_fts_constraint_changes() -> anyhow::Result<()> {
        before_each();

        // Ingest with FTS enabled, this will populate the FTS table
        let store = TestStore::new(
            MockRemoteSettingsClient::default()
                .with_record(
                    "data",
                    "data-1",
                    json!([los_pollos_amp().merge(json!({
                        "keywords": ["los", "los pollos", "los pollos ", "los pollos hermanos"],
                        "full_keywords": [("los pollos", 3), ("los pollos hermanos", 1)],
                    }))]),
                )
                .with_icon(los_pollos_icon()),
        );
        // Ingest without FTS
        store.ingest(SuggestIngestionConstraints::amp_without_fts());
        // Ingest again with FTS
        store.ingest(SuggestIngestionConstraints::amp_with_fts());

        assert_eq!(
            store.fetch_suggestions(SuggestionQuery {
                provider_constraints: Some(SuggestionProviderConstraints {
                    amp_alternative_matching: Some(AmpMatchingStrategy::FtsAgainstFullKeywords),
                    ..SuggestionProviderConstraints::default()
                }),
                // "Hermanos" should match, even though it's not listed in the keywords,
                // because this strategy uses an FTS match against the full keyword list.
                ..SuggestionQuery::amp("hermanos")
            }),
            vec![los_pollos_suggestion(
                "hermanos",
                Some(FtsMatchInfo {
                    prefix: false,
                    stemming: false,
                }),
            )],
        );
        Ok(())
    }

    /// Tests re-ingesting icons from an updated attachment.
    #[test]
    fn reingest_icons() -> anyhow::Result<()> {
        before_each();

        let mut store = TestStore::new(
            MockRemoteSettingsClient::default()
                .with_record(
                    "data",
                    "1234",
                    json!([los_pollos_amp(), good_place_eats_amp()]),
                )
                .with_icon(los_pollos_icon())
                .with_icon(good_place_eats_icon()),
        );
        // This ingestion should run, since the DB is empty
        store.ingest(SuggestIngestionConstraints::all_providers());

        // Reingest with updated icon data
        //  - Los pollos gets new data and a new id
        //  - Good place eats gets new data only
        store
            .client_mut()
            .update_record(
                "data",
                "1234",
                json!([
                    los_pollos_amp().merge(json!({"icon": "1000"})),
                    good_place_eats_amp()
                ]),
            )
            .delete_icon(los_pollos_icon())
            .add_icon(MockIcon {
                id: "1000",
                data: "new-los-pollos-icon",
                ..los_pollos_icon()
            })
            .update_icon(MockIcon {
                data: "new-good-place-eats-icon",
                ..good_place_eats_icon()
            });
        store.ingest(SuggestIngestionConstraints::all_providers());

        assert!(matches!(
            store.fetch_suggestions(SuggestionQuery::amp("lo")).as_slice(),
            [ Suggestion::Amp { icon, .. } ] if *icon == Some("new-los-pollos-icon".as_bytes().to_vec())
        ));

        assert!(matches!(
            store.fetch_suggestions(SuggestionQuery::amp("la")).as_slice(),
            [ Suggestion::Amp { icon, .. } ] if *icon == Some("new-good-place-eats-icon".as_bytes().to_vec())
        ));

        Ok(())
    }

    /// Tests re-ingesting AMO suggestions from an updated attachment.
    #[test]
    fn reingest_amo_suggestions() -> anyhow::Result<()> {
        before_each();

        let mut store = TestStore::new(
            MockRemoteSettingsClient::default()
                .with_record("amo-suggestions", "data-1", json!([relay_amo()]))
                .with_record(
                    "amo-suggestions",
                    "data-2",
                    json!([dark_mode_amo(), foxy_guestures_amo()]),
                ),
        );

        store.ingest(SuggestIngestionConstraints::all_providers());

        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::amo("masking e")),
            vec![relay_suggestion()],
        );
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::amo("night")),
            vec![dark_mode_suggestion()],
        );
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::amo("grammar")),
            vec![foxy_guestures_suggestion()],
        );

        // Update the snapshot with new suggestions: update the second, drop the
        // third, and add the fourth.
        store
            .client_mut()
            .update_record("amo-suggestions", "data-1", json!([relay_amo()]))
            .update_record(
                "amo-suggestions",
                "data-2",
                json!([
                    dark_mode_amo().merge(json!({"title": "Updated second suggestion"})),
                    new_tab_override_amo(),
                ]),
            );
        store.ingest(SuggestIngestionConstraints::all_providers());

        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::amo("masking e")),
            vec![relay_suggestion()],
        );
        assert!(matches!(
            store.fetch_suggestions(SuggestionQuery::amo("night")).as_slice(),
            [Suggestion::Amo { title, .. } ] if title == "Updated second suggestion"
        ));
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::amo("grammar")),
            vec![],
        );
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::amo("image search")),
            vec![new_tab_override_suggestion()],
        );

        Ok(())
    }

    /// Tests ingestion when previously-ingested suggestions/icons have been deleted.
    #[test]
    fn ingest_with_deletions() -> anyhow::Result<()> {
        before_each();

        let mut store = TestStore::new(
            MockRemoteSettingsClient::default()
                .with_record("data", "data-1", json!([los_pollos_amp()]))
                .with_record("data", "data-2", json!([good_place_eats_amp()]))
                .with_icon(los_pollos_icon())
                .with_icon(good_place_eats_icon()),
        );
        store.ingest(SuggestIngestionConstraints::all_providers());
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::amp("lo")),
            vec![los_pollos_suggestion("los", None)],
        );
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::amp("la")),
            vec![good_place_eats_suggestion("lasagna", None)],
        );
        // Re-ingest without los-pollos and good place eat's icon.  The suggest store should
        // recognize that they're missing and delete them.
        store
            .client_mut()
            .delete_record("quicksuggest", "data-1")
            .delete_icon(good_place_eats_icon());
        store.ingest(SuggestIngestionConstraints::all_providers());

        assert_eq!(store.fetch_suggestions(SuggestionQuery::amp("lo")), vec![]);
        assert!(matches!(
            store.fetch_suggestions(SuggestionQuery::amp("la")).as_slice(),
            [
                Suggestion::Amp { icon, icon_mimetype, .. }
            ] if icon.is_none() && icon_mimetype.is_none(),
        ));
        Ok(())
    }

    /// Tests clearing the store.
    #[test]
    fn clear() -> anyhow::Result<()> {
        before_each();

        let store = TestStore::new(
            MockRemoteSettingsClient::default()
                .with_record("data", "data-1", json!([los_pollos_amp()]))
                .with_record("data", "data-2", json!([good_place_eats_amp()]))
                .with_icon(los_pollos_icon())
                .with_icon(good_place_eats_icon()),
        );
        store.ingest(SuggestIngestionConstraints::all_providers());
        assert!(store.count_rows("suggestions") > 0);
        assert!(store.count_rows("keywords") > 0);
        assert!(store.count_rows("icons") > 0);

        store.inner.clear()?;
        assert!(store.count_rows("suggestions") == 0);
        assert!(store.count_rows("keywords") == 0);
        assert!(store.count_rows("icons") == 0);

        Ok(())
    }

    /// Tests querying suggestions.
    #[test]
    fn query() -> anyhow::Result<()> {
        before_each();

        let store = TestStore::new(
            MockRemoteSettingsClient::default()
                .with_record(
                    "data",
                    "data-1",
                    json!([
                        good_place_eats_amp(),
                        california_wiki(),
                        caltech_wiki(),
                        multimatch_wiki(),
                    ]),
                )
                .with_record(
                    "amo-suggestions",
                    "data-2",
                    json!([relay_amo(), multimatch_amo(),]),
                )
                .with_record(
                    "pocket-suggestions",
                    "data-3",
                    json!([burnout_pocket(), multimatch_pocket(),]),
                )
                .with_record("yelp-suggestions", "data-4", json!([ramen_yelp(),]))
                .with_record("mdn-suggestions", "data-5", json!([array_mdn(),]))
                .with_icon(good_place_eats_icon())
                .with_icon(california_icon())
                .with_icon(caltech_icon())
                .with_icon(yelp_favicon())
                .with_icon(multimatch_wiki_icon()),
        );

        store.ingest(SuggestIngestionConstraints::all_providers());

        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::all_providers("")),
            vec![]
        );
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::all_providers("la")),
            vec![good_place_eats_suggestion("lasagna", None),]
        );
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::all_providers("multimatch")),
            vec![
                multimatch_pocket_suggestion(true),
                multimatch_amo_suggestion(),
                multimatch_wiki_suggestion(),
            ]
        );
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::all_providers("MultiMatch")),
            vec![
                multimatch_pocket_suggestion(true),
                multimatch_amo_suggestion(),
                multimatch_wiki_suggestion(),
            ]
        );
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::all_providers("multimatch").limit(2)),
            vec![
                multimatch_pocket_suggestion(true),
                multimatch_amo_suggestion(),
            ],
        );
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::amp("la")),
            vec![good_place_eats_suggestion("lasagna", None)],
        );
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::all_providers_except(
                "la",
                SuggestionProvider::Amp
            )),
            vec![],
        );
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::with_providers("la", vec![])),
            vec![],
        );
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::with_providers(
                "cal",
                vec![
                    SuggestionProvider::Amp,
                    SuggestionProvider::Amo,
                    SuggestionProvider::Pocket,
                ]
            )),
            vec![],
        );
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::wikipedia("cal")),
            vec![
                california_suggestion("california"),
                caltech_suggestion("california"),
            ],
        );
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::wikipedia("cal").limit(1)),
            vec![california_suggestion("california"),],
        );
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::with_providers("cal", vec![])),
            vec![],
        );
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::amo("spam")),
            vec![relay_suggestion()],
        );
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::amo("masking")),
            vec![relay_suggestion()],
        );
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::amo("masking e")),
            vec![relay_suggestion()],
        );
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::amo("masking s")),
            vec![],
        );
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::with_providers(
                "soft",
                vec![SuggestionProvider::Amp, SuggestionProvider::Wikipedia]
            )),
            vec![],
        );
        assert_eq!(
            store.fetch_suggestions(SuggestionQuery::pocket("soft")),
            vec![burnout_suggestion(false),],
--> --------------------

--> maximum size reached

--> --------------------