From 08c8487646e0c55cc5f4d52aeff2054433eb28ab Mon Sep 17 00:00:00 2001 From: Dani Palou Date: Fri, 4 Oct 2019 16:02:03 +0200 Subject: [PATCH] MOBILE-2491 filter: Add memory caches to speed up filter --- src/classes/site.ts | 31 ++-- src/core/filter/providers/delegate.ts | 7 +- src/core/filter/providers/filter.ts | 114 ++++++++++++- src/core/filter/providers/helper.ts | 155 +++++++++++++----- .../settings/pages/space-usage/space-usage.ts | 11 +- src/providers/events.ts | 2 + 6 files changed, 264 insertions(+), 56 deletions(-) diff --git a/src/classes/site.ts b/src/classes/site.ts index 84e7bdaaf..d4988f66a 100644 --- a/src/classes/site.ts +++ b/src/classes/site.ts @@ -1036,15 +1036,7 @@ export class CoreSite { preSets.omitExpires = preSets.omitExpires || preSets.forceOffline || !this.appProvider.isOnline(); if (!preSets.omitExpires) { - let expirationDelay = this.UPDATE_FREQUENCIES[preSets.updateFrequency] || - this.UPDATE_FREQUENCIES[CoreSite.FREQUENCY_USUALLY]; - - if (this.appProvider.isNetworkAccessLimited()) { - // Not WiFi, increase the expiration delay a 50% to decrease the data usage in this case. - expirationDelay *= 1.5; - } - - expirationTime = entry.expirationTime + expirationDelay; + expirationTime = entry.expirationTime + this.getExpirationDelay(preSets.updateFrequency); if (now > expirationTime) { this.logger.debug('Cached element found, but it is expired'); @@ -1165,7 +1157,9 @@ export class CoreSite { this.logger.debug('Invalidate all the cache for site: ' + this.id); - return this.db.updateRecords(CoreSite.WS_CACHE_TABLE, { expirationTime: 0 }); + return this.db.updateRecords(CoreSite.WS_CACHE_TABLE, { expirationTime: 0 }).finally(() => { + this.eventsProvider.trigger(CoreEventsProvider.WS_CACHE_INVALIDATED, {}, this.getId()); + }); } /** @@ -1875,4 +1869,21 @@ export class CoreSite { setLocalSiteConfig(name: string, value: number | string): Promise { return this.db.insertRecord(CoreSite.CONFIG_TABLE, { name: name, value: value }); } + + /** + * Get a certain cache expiration delay. + * + * @param updateFrequency The update frequency of the entry. + * @return {number} Expiration delay. + */ + getExpirationDelay(updateFrequency?: number): number { + let expirationDelay = this.UPDATE_FREQUENCIES[updateFrequency] || this.UPDATE_FREQUENCIES[CoreSite.FREQUENCY_USUALLY]; + + if (this.appProvider.isNetworkAccessLimited()) { + // Not WiFi, increase the expiration delay a 50% to decrease the data usage in this case. + expirationDelay *= 1.5; + } + + return expirationDelay; + } } diff --git a/src/core/filter/providers/delegate.ts b/src/core/filter/providers/delegate.ts index 3fa27c298..11e081c0b 100644 --- a/src/core/filter/providers/delegate.ts +++ b/src/core/filter/providers/delegate.ts @@ -96,7 +96,12 @@ export class CoreFilterDelegate extends CoreDelegate { } promise = promise.then((text) => { - return this.executeFunctionOnEnabled(filter.filter, 'filter', [text, filter, options, siteId]); + return Promise.resolve(this.executeFunctionOnEnabled(filter.filter, 'filter', [text, filter, options, siteId])) + .catch((error) => { + this.logger.error('Error applying filter' + filter.filter, error); + + return text; + }); }); }); diff --git a/src/core/filter/providers/filter.ts b/src/core/filter/providers/filter.ts index 8092d01c4..24d01f83c 100644 --- a/src/core/filter/providers/filter.ts +++ b/src/core/filter/providers/filter.ts @@ -13,6 +13,8 @@ // limitations under the License. import { Injectable } from '@angular/core'; +import { CoreAppProvider } from '@providers/app'; +import { CoreEventsProvider } from '@providers/events'; import { CoreLoggerProvider } from '@providers/logger'; import { CoreSitesProvider } from '@providers/sites'; import { CoreSite } from '@classes/site'; @@ -30,11 +32,36 @@ export class CoreFilterProvider { protected logger; + /** + * Store the contexts in memory to speed up the process, it can take a lot of time otherwise. + */ + protected contextsCache: { + [siteId: string]: { + [contextlevel: string]: { + [instanceid: number]: { + filters: CoreFilterFilter[], + time: number + } + } + } + } = {}; + constructor(logger: CoreLoggerProvider, + eventsProvider: CoreEventsProvider, private sitesProvider: CoreSitesProvider, private textUtils: CoreTextUtilsProvider, - private filterDelegate: CoreFilterDelegate) { + private filterDelegate: CoreFilterDelegate, + private appProvider: CoreAppProvider) { + this.logger = logger.getInstance('CoreFilterProvider'); + + eventsProvider.on(CoreEventsProvider.WS_CACHE_INVALIDATED, (data) => { + delete this.contextsCache[data.siteId]; + }); + + eventsProvider.on(CoreEventsProvider.SITE_STORAGE_DELETED, (data) => { + delete this.contextsCache[data.siteId]; + }); } /** @@ -148,9 +175,19 @@ export class CoreFilterProvider { * @return Promise resolved with the filters classified by context. */ getAvailableInContexts(contexts: {contextlevel: string, instanceid: number}[], siteId?: string) - : Promise<{[contextlevel: string]: {[instanceid: number]: CoreFilterFilter[]}}> { + : Promise { return this.sitesProvider.getSite(siteId).then((site) => { + siteId = site.getId(); + + const result = this.getFromMemoryCache(contexts, site); + + if (result) { + return result; + } + + this.contextsCache[siteId] = this.contextsCache[siteId] || {}; + let hasSystemContext = false, hasSiteHomeContext = false; @@ -194,7 +231,7 @@ export class CoreFilterProvider { return site.read('core_filters_get_available_in_context', data, preSets) .then((result: CoreFilterGetAvailableInContextResult) => { - const classified: {[contextlevel: string]: {[instanceid: number]: CoreFilterFilter[]}} = {}; + const classified: CoreFilterClassifiedFilters = {}; // Initialize all contexts. contexts.forEach((context) => { @@ -205,6 +242,7 @@ export class CoreFilterProvider { if (contexts.length == 1 && !hasSystemContext) { // Only 1 context, no need to iterate over the filters. classified[contexts[0].contextlevel][contexts[0].instanceid] = result.filters; + this.storeInMemoryCache(classified, siteId); return classified; } @@ -228,6 +266,8 @@ export class CoreFilterProvider { classified[filter.contextlevel][filter.instanceid].push(filter); }); + this.storeInMemoryCache(classified, siteId); + return classified; }); }); @@ -247,6 +287,45 @@ export class CoreFilterProvider { }); } + /** + * Get contexts filters from the memory cache. + * + * @param contexts Contexts to get. + * @param site Site. + * @return The filters classified by context and instance. + */ + protected getFromMemoryCache(contexts: {contextlevel: string, instanceid: number}[], site: CoreSite) + : CoreFilterClassifiedFilters { + + if (this.contextsCache[site.getId()]) { + // Check if we have the contexts in the memory cache. + const siteContexts = this.contextsCache[site.getId()], + isOnline = this.appProvider.isOnline(), + result: CoreFilterClassifiedFilters = {}; + let allFound = true; + + for (let i = 0; i < contexts.length; i++) { + const context = contexts[i], + cachedCtxt = siteContexts[context.contextlevel] && siteContexts[context.contextlevel][context.instanceid]; + + // Check the context isn't "expired". The time stored in this cache will not match the one in the site cache. + if (cachedCtxt && (!isOnline || + Date.now() <= cachedCtxt.time + site.getExpirationDelay(CoreSite.FREQUENCY_RARELY))) { + + result[context.contextlevel] = result[context.contextlevel] || {}; + result[context.contextlevel][context.instanceid] = cachedCtxt.filters; + } else { + allFound = false; + break; + } + } + + if (allFound) { + return result; + } + } + } + /** * Invalidates all available in context WS calls. * @@ -283,6 +362,26 @@ export class CoreFilterProvider { invalidateAvailableInContext(contextLevel: string, instanceId: number, siteId?: string): Promise { return this.invalidateAvailableInContexts([{contextlevel: contextLevel, instanceid: instanceId}], siteId); } + + /** + * Store filters in the memory cache. + * + * @param filters Filters to store, classified by contextlevel and instanceid + * @param siteId Site ID. + */ + protected storeInMemoryCache(filters: CoreFilterClassifiedFilters, siteId: string): void { + + for (const contextLevel in filters) { + this.contextsCache[siteId][contextLevel] = this.contextsCache[siteId][contextLevel] || {}; + + for (const instanceId in filters[contextLevel]) { + this.contextsCache[siteId][contextLevel][instanceId] = { + filters: filters[contextLevel][instanceId], + time: Date.now() + }; + } + } + } } /** @@ -319,3 +418,12 @@ export type CoreFilterFormatTextOptions = { wsNotFiltered?: boolean; // If true it means the WS didn't filter the text for some reason. courseId?: number; // Course ID the text belongs to. It can be used to improve performance. }; + +/** + * Filters classified by context and instance. + */ +export type CoreFilterClassifiedFilters = { + [contextlevel: string]: { + [instanceid: number]: CoreFilterFilter[] + } +}; diff --git a/src/core/filter/providers/helper.ts b/src/core/filter/providers/helper.ts index 06c4a7b83..bb01bdbaf 100644 --- a/src/core/filter/providers/helper.ts +++ b/src/core/filter/providers/helper.ts @@ -13,10 +13,12 @@ // limitations under the License. import { Injectable } from '@angular/core'; +import { CoreAppProvider } from '@providers/app'; +import { CoreEventsProvider } from '@providers/events'; import { CoreLoggerProvider } from '@providers/logger'; import { CoreSitesProvider } from '@providers/sites'; import { CoreFilterDelegate } from './delegate'; -import { CoreFilterProvider, CoreFilterFilter, CoreFilterFormatTextOptions } from './filter'; +import { CoreFilterProvider, CoreFilterFilter, CoreFilterFormatTextOptions, CoreFilterClassifiedFilters } from './filter'; import { CoreCourseProvider } from '@core/course/providers/course'; import { CoreSite } from '@classes/site'; @@ -28,12 +30,36 @@ export class CoreFilterHelperProvider { protected logger; + /** + * When a module context is requested, we request all the modules in a course to decrease WS calls. If there are a lot of + * modules, checking the cache of all contexts can be really slow, so we use this memory cache to speed up the process. + */ + protected moduleContextsCache: { + [siteId: string]: { + [courseId: number]: { + contexts: CoreFilterClassifiedFilters, + time: number + } + } + } = {}; + constructor(logger: CoreLoggerProvider, + eventsProvider: CoreEventsProvider, + private appProvider: CoreAppProvider, private sitesProvider: CoreSitesProvider, private filterDelegate: CoreFilterDelegate, private courseProvider: CoreCourseProvider, private filterProvider: CoreFilterProvider) { + this.logger = logger.getInstance('CoreFilterHelperProvider'); + + eventsProvider.on(CoreEventsProvider.WS_CACHE_INVALIDATED, (data) => { + delete this.moduleContextsCache[data.siteId]; + }); + + eventsProvider.on(CoreEventsProvider.SITE_STORAGE_DELETED, (data) => { + delete this.moduleContextsCache[data.siteId]; + }); } /** @@ -79,55 +105,64 @@ export class CoreFilterHelperProvider { getFilters(contextLevel: string, instanceId: number, options?: CoreFilterFormatTextOptions, siteId?: string) : Promise { - let site: CoreSite; - options.contextLevel = contextLevel; options.instanceId = instanceId; options.filter = false; - return this.sitesProvider.getSite(siteId).then((s) => { - site = s; + return this.sitesProvider.getSite(siteId).then((site) => { + siteId = site.getId(); - return this.filterProvider.canGetAvailableInContext(siteId); - }).then((canGet) => { - if (!canGet) { - options.filter = true; - - // We cannot check which filters are available, apply them all. - return this.filterDelegate.getEnabledFilters(contextLevel, instanceId); - } - - let promise: Promise; - - if (instanceId == site.getSiteHomeId() && (contextLevel == 'system' || contextLevel == 'course')) { - // No need to check the site filters because we're requesting the same context, so we'd do the same twice. - promise = Promise.resolve(true); - } else { - // Check if site has any filter to treat. - promise = this.siteHasFiltersToTreat(options, siteId); - } - - return promise.then((hasFilters) => { - if (hasFilters) { + return this.filterProvider.canGetAvailableInContext(siteId).then((canGet) => { + if (!canGet) { options.filter = true; - if (contextLevel == 'module' && options.courseId) { - // Get all the modules filters with a single call to decrease the number of WS calls. - return this.getCourseModulesContexts(options.courseId, site.getId()).then((contexts) => { - - return this.filterProvider.getAvailableInContexts(contexts, site.getId()).then((filters) => { - return filters[contextLevel][instanceId] || []; - }); - }); - } - - return this.filterProvider.getAvailableInContext(contextLevel, instanceId, siteId); + // We cannot check which filters are available, apply them all. + return this.filterDelegate.getEnabledFilters(contextLevel, instanceId); } - return []; - }).catch(() => { - return []; + let promise: Promise; + + if (instanceId == site.getSiteHomeId() && (contextLevel == 'system' || contextLevel == 'course')) { + // No need to check the site filters because we're requesting the same context, so we'd do the same twice. + promise = Promise.resolve(true); + } else { + // Check if site has any filter to treat. + promise = this.siteHasFiltersToTreat(options, siteId); + } + + return promise.then((hasFilters) => { + if (hasFilters) { + options.filter = true; + + if (contextLevel == 'module' && options.courseId) { + // Get all the modules filters with a single call to decrease the number of WS calls. + // Check the memory cache first to speed up the process. + + const result = this.getFromMemoryCache(options.courseId, contextLevel, instanceId, site); + if (result) { + return result; + } + + return this.getCourseModulesContexts(options.courseId, siteId).then((contexts) => { + + return this.filterProvider.getAvailableInContexts(contexts, siteId).then((filters) => { + this.storeInMemoryCache(options.courseId, filters, siteId); + + return filters[contextLevel][instanceId] || []; + }); + }); + } + + return this.filterProvider.getAvailableInContext(contextLevel, instanceId, siteId); + } + + return []; + }); }); + }).catch((error) => { + this.logger.error('Error getting filters, return an empty array', error, contextLevel, instanceId); + + return []; }); } @@ -149,6 +184,32 @@ export class CoreFilterHelperProvider { }); } + /** + * Get module context filters from the memory cache. + * + * @param courseId Course the module belongs to. + * @param contextLevel Context level. + * @param instanceId Instance ID. + * @param site Site. + * @return The filters, undefined if not found. + */ + protected getFromMemoryCache(courseId: number, contextLevel: string, instanceId: number, site: CoreSite): CoreFilterFilter[] { + + const siteId = site.getId(); + + // Check if we have the context in the memory cache. + if (this.moduleContextsCache[siteId] && this.moduleContextsCache[siteId][courseId]) { + const cachedCourse = this.moduleContextsCache[siteId][courseId]; + + if (!this.appProvider.isOnline() || + Date.now() <= cachedCourse.time + site.getExpirationDelay(CoreSite.FREQUENCY_RARELY)) { + + // We can use cache, return the filters if found. + return cachedCourse.contexts[contextLevel] && cachedCourse.contexts[contextLevel][instanceId]; + } + } + } + /** * Check if site has available any filter that should be treated by the app. * @@ -168,4 +229,18 @@ export class CoreFilterHelperProvider { }); }); } + + /** + * Store filters in the memory cache. + * + * @param contexts Filters to store, classified by contextlevel and instanceid + * @param siteId Site ID. + */ + protected storeInMemoryCache(courseId: number, contexts: CoreFilterClassifiedFilters, siteId: string): void { + this.moduleContextsCache[siteId] = this.moduleContextsCache[siteId] || {}; + this.moduleContextsCache[siteId][courseId] = { + contexts: contexts, + time: Date.now() + }; + } } diff --git a/src/core/settings/pages/space-usage/space-usage.ts b/src/core/settings/pages/space-usage/space-usage.ts index 8ce241df8..44a24784f 100644 --- a/src/core/settings/pages/space-usage/space-usage.ts +++ b/src/core/settings/pages/space-usage/space-usage.ts @@ -16,6 +16,7 @@ import { Component, } from '@angular/core'; import { IonicPage } from 'ionic-angular'; import { TranslateService } from '@ngx-translate/core'; import { CoreAppProvider } from '@providers/app'; +import { CoreEventsProvider } from '@providers/events'; import { CoreFilepoolProvider } from '@providers/filepool'; import { CoreSitesProvider } from '@providers/sites'; import { CoreDomUtilsProvider } from '@providers/utils/dom'; @@ -39,8 +40,12 @@ export class CoreSettingsSpaceUsagePage { totalEntries = 0; constructor(private filePoolProvider: CoreFilepoolProvider, - private sitesProvider: CoreSitesProvider, private filterHelper: CoreFilterHelperProvider, - private translate: TranslateService, private domUtils: CoreDomUtilsProvider, appProvider: CoreAppProvider, + private eventsProvider: CoreEventsProvider, + private sitesProvider: CoreSitesProvider, + private filterHelper: CoreFilterHelperProvider, + private translate: TranslateService, + private domUtils: CoreDomUtilsProvider, + appProvider: CoreAppProvider, private courseProvider: CoreCourseProvider) { this.currentSiteId = this.sitesProvider.getCurrentSiteId(); } @@ -196,6 +201,8 @@ export class CoreSettingsSpaceUsagePage { }); } }).finally(() => { + this.eventsProvider.trigger(CoreEventsProvider.SITE_STORAGE_DELETED, {}, site.getId()); + this.calcSiteClearRows(site).then((rows) => { siteData.cacheEntries = rows; }); diff --git a/src/providers/events.ts b/src/providers/events.ts index 2371b7fc5..5d1557ed8 100644 --- a/src/providers/events.ts +++ b/src/providers/events.ts @@ -62,6 +62,8 @@ export class CoreEventsProvider { static SEND_ON_ENTER_CHANGED = 'send_on_enter_changed'; static MAIN_MENU_OPEN = 'main_menu_open'; static SELECT_COURSE_TAB = 'select_course_tab'; + static WS_CACHE_INVALIDATED = 'ws_cache_invalidated'; + static SITE_STORAGE_DELETED = 'site_storage_deleted'; protected logger; protected observables: { [s: string]: Subject } = {};