From e63a40d0925f8c31dfa2aafea107d7698ede1612 Mon Sep 17 00:00:00 2001 From: Dani Palou Date: Mon, 13 Jun 2022 08:27:59 +0200 Subject: [PATCH] MOBILE-4086 core: Split wscache table into several tables --- src/core/classes/site.ts | 125 ++++++++++++++++------------ src/core/services/database/sites.ts | 122 ++++++++++++++++----------- upgrade.txt | 1 + 3 files changed, 144 insertions(+), 104 deletions(-) diff --git a/src/core/classes/site.ts b/src/core/classes/site.ts index 082a4f0f2..58c994959 100644 --- a/src/core/classes/site.ts +++ b/src/core/classes/site.ts @@ -48,6 +48,15 @@ import { CoreDatabaseTable } from './database/database-table'; import { CoreDatabaseCachingStrategy } from './database/database-table-proxy'; import { CoreSilentError } from './errors/silenterror'; import { CorePromisedValue } from '@classes/promised-value'; +import { + CONFIG_TABLE, + CoreSiteConfigDBRecord, + CoreSiteLastViewedDBRecord, + CoreSiteWSCacheRecord, + LAST_VIEWED_TABLE, + WSGroups, + WS_CACHE_TABLES_PREFIX, +} from '@services/database/sites'; /** * QR Code type enumeration. @@ -81,11 +90,6 @@ export class CoreSite { static readonly FREQUENCY_SOMETIMES = 2; static readonly FREQUENCY_RARELY = 3; - // Variables for the database. - static readonly WS_CACHE_TABLE = 'wscache_2'; - static readonly CONFIG_TABLE = 'core_site_config'; - static readonly LAST_VIEWED_TABLE = 'core_site_last_viewed'; - static readonly MINIMUM_MOODLE_VERSION = '3.5'; // Versions of Moodle releases. @@ -111,7 +115,7 @@ export class CoreSite { // Rest of variables. protected logger: CoreLogger; protected db?: SQLiteDB; - protected cacheTable: AsyncInstance>; + protected cacheTables: Record>>; protected configTable: AsyncInstance>; protected lastViewedTable: AsyncInstance>; protected cleanUnicode = false; @@ -147,18 +151,25 @@ export class CoreSite { ) { this.logger = CoreLogger.getInstance('CoreSite'); this.siteUrl = CoreUrlUtils.removeUrlParams(this.siteUrl); // Make sure the URL doesn't have params. - this.cacheTable = asyncInstance(() => CoreSites.getSiteTable(CoreSite.WS_CACHE_TABLE, { - siteId: this.getId(), - database: this.getDb(), - config: { cachingStrategy: CoreDatabaseCachingStrategy.None }, - })); - this.configTable = asyncInstance(() => CoreSites.getSiteTable(CoreSite.CONFIG_TABLE, { + + this.cacheTables = Object.values(WSGroups).reduce((tables, group) => { + tables[group] = asyncInstance(() => CoreSites.getSiteTable(WS_CACHE_TABLES_PREFIX + group, { + siteId: this.getId(), + database: this.getDb(), + config: { cachingStrategy: CoreDatabaseCachingStrategy.None }, + })); + + return tables; + }, >>> {}); + + this.configTable = asyncInstance(() => CoreSites.getSiteTable(CONFIG_TABLE, { siteId: this.getId(), database: this.getDb(), config: { cachingStrategy: CoreDatabaseCachingStrategy.Eager }, primaryKeyColumns: ['name'], })); - this.lastViewedTable = asyncInstance(() => CoreSites.getSiteTable(CoreSite.LAST_VIEWED_TABLE, { + + this.lastViewedTable = asyncInstance(() => CoreSites.getSiteTable(LAST_VIEWED_TABLE, { siteId: this.getId(), database: this.getDb(), config: { cachingStrategy: CoreDatabaseCachingStrategy.Eager }, @@ -588,7 +599,7 @@ export class CoreSite { const cacheId = this.getCacheId(method, data); // Check for an ongoing identical request if we're not ignoring cache. - if (preSets.getFromCache && this.ongoingRequests[cacheId]) { + if (preSets.getFromCache && this.ongoingRequests[cacheId] !== undefined) { const response = await this.ongoingRequests[cacheId]; // Clone the data, this may prevent errors if in the callback the object is modified. @@ -967,14 +978,15 @@ export class CoreSite { } const id = this.getCacheId(method, data); + const group = this.getWSGroupFromWSName(method); let entry: CoreSiteWSCacheRecord | undefined; if (preSets.getCacheUsingCacheKey || (emergency && preSets.getEmergencyCacheUsingCacheKey)) { - const entries = await this.cacheTable.getMany({ key: preSets.cacheKey }); + const entries = await this.cacheTables[group].getMany({ key: preSets.cacheKey }); if (!entries.length) { // Cache key not found, get by params sent. - entry = await this.cacheTable.getOneByPrimaryKey({ id }); + entry = await this.cacheTables[group].getOneByPrimaryKey({ id }); } else { if (entries.length > 1) { // More than one entry found. Search the one with same ID as this call. @@ -986,7 +998,7 @@ export class CoreSite { } } } else { - entry = await this.cacheTable.getOneByPrimaryKey({ id }); + entry = await this.cacheTables[group].getOneByPrimaryKey({ id }); } if (entry === undefined) { @@ -1022,6 +1034,25 @@ export class CoreSite { throw new CoreError('Cache entry not valid.'); } + /** + * Get WS group based on a WS name. + * + * @return WS group. + */ + protected getWSGroupFromWSName(name: string): WSGroups { + if (name.startsWith('mod_')) { + return WSGroups.MOD; + } else if (name.startsWith('tool_')) { + return WSGroups.TOOL; + } else if (name.startsWith('block_') || name.startsWith('core_block_')) { + return WSGroups.BLOCK; + } else if (name.startsWith('core_')) { + return WSGroups.CORE; + } else { + return WSGroups.OTHER; + } + } + /** * Gets the size of cached data for a specific component or component instance. * @@ -1037,7 +1068,7 @@ export class CoreSite { extraClause = ' AND componentId = ?'; } - return this.cacheTable.reduce( + const sizes = await Promise.all(Object.values(this.cacheTables).map(table => table.reduce( { sql: 'SUM(length(data))', js: (size, record) => size + record.data.length, @@ -1048,7 +1079,9 @@ export class CoreSite { sqlParams: params, js: record => record.component === component && (params.length === 1 || record.componentId === componentId), }, - ); + ))); + + return sizes.reduce((totalSize, size) => totalSize + size, 0); } /** @@ -1070,6 +1103,7 @@ export class CoreSite { // Since 3.7, the expiration time contains the time the entry is modified instead of the expiration time. // We decided to reuse this field to prevent modifying the database table. const id = this.getCacheId(method, data); + const group = this.getWSGroupFromWSName(method); const entry = { id, data: JSON.stringify(response), @@ -1087,7 +1121,7 @@ export class CoreSite { } } - await this.cacheTable.insert(entry); + await this.cacheTables[group].insert(entry); } /** @@ -1102,11 +1136,12 @@ export class CoreSite { // eslint-disable-next-line @typescript-eslint/no-explicit-any protected async deleteFromCache(method: string, data: any, preSets: CoreSiteWSPreSets, allCacheKey?: boolean): Promise { const id = this.getCacheId(method, data); + const group = this.getWSGroupFromWSName(method); if (allCacheKey) { - await this.cacheTable.delete({ key: preSets.cacheKey }); + await this.cacheTables[group].delete({ key: preSets.cacheKey }); } else { - await this.cacheTable.deleteByPrimaryKey({ id }); + await this.cacheTables[group].deleteByPrimaryKey({ id }); } } @@ -1129,7 +1164,7 @@ export class CoreSite { params['componentId'] = componentId; } - await this.cacheTable.delete(params); + await Promise.all(Object.values(this.cacheTables).map(table => table.delete(params))); } /* @@ -1164,7 +1199,7 @@ export class CoreSite { this.logger.debug('Invalidate all the cache for site: ' + this.id); try { - await this.cacheTable.update({ expirationTime: 0 }); + await Promise.all(Object.values(this.cacheTables).map(table => table.update({ expirationTime: 0 }))); } finally { CoreEvents.trigger(CoreEvents.WS_CACHE_INVALIDATED, {}, this.getId()); } @@ -1183,7 +1218,7 @@ export class CoreSite { this.logger.debug('Invalidate cache for key: ' + key); - await this.cacheTable.update({ expirationTime: 0 }, { key }); + await Promise.all(Object.values(this.cacheTables).map(table => table.update({ expirationTime: 0 }, { key }))); } /** @@ -1217,11 +1252,11 @@ export class CoreSite { this.logger.debug('Invalidate cache for key starting with: ' + key); - await this.cacheTable.updateWhere({ expirationTime: 0 }, { + await Promise.all(Object.values(this.cacheTables).map(table => table.updateWhere({ expirationTime: 0 }, { sql: 'key LIKE ?', sqlParams: [key + '%'], js: record => !!record.key?.startsWith(key), - }); + }))); } /** @@ -1289,18 +1324,20 @@ export class CoreSite { } /** - * Gets an approximation of the cache table usage of the site. + * Gets an approximation of the cache tables usage of the site. * - * Currently this is just the total length of the data fields in the cache table. + * Currently this is just the total length of the data fields in the cache tables. * - * @return Promise resolved with the total size of all data in the cache table (bytes) + * @return Promise resolved with the total size of all data in the cache tables (bytes) */ async getCacheUsage(): Promise { - return this.cacheTable.reduce({ + const sizes = await Promise.all(Object.values(this.cacheTables).map(table => table.reduce({ sql: 'SUM(length(data))', js: (size, record) => size + record.data.length, jsInitialValue: 0, - }); + }))); + + return sizes.reduce((totalSize, size) => totalSize + size, 0); } /** @@ -1416,7 +1453,7 @@ export class CoreSite { } // Check for an ongoing identical request if we're not ignoring cache. - if (cachePreSets.getFromCache && this.ongoingRequests[cacheId]) { + if (cachePreSets.getFromCache && this.ongoingRequests[cacheId] !== undefined) { const response = await this.ongoingRequests[cacheId]; return response; @@ -2455,28 +2492,6 @@ export type CoreSiteCallExternalFunctionsResult = { }[]; }; -export type CoreSiteConfigDBRecord = { - name: string; - value: string | number; -}; - -export type CoreSiteWSCacheRecord = { - id: string; - data: string; - expirationTime: number; - key?: string; - component?: string; - componentId?: number; -}; - -export type CoreSiteLastViewedDBRecord = { - component: string; - id: number; - value: string; - timeaccess: number; - data?: string; -}; - /** * Options for storeLastViewed. */ diff --git a/src/core/services/database/sites.ts b/src/core/services/database/sites.ts index 80aadcd48..ffc6b9f17 100644 --- a/src/core/services/database/sites.ts +++ b/src/core/services/database/sites.ts @@ -15,7 +15,6 @@ import { CoreAppSchema } from '@services/app'; import { CoreSiteSchema } from '@services/sites'; import { SQLiteDB, SQLiteDBTableSchema } from '@classes/sqlitedb'; -import { CoreSite } from '@classes/site'; /** * Database variables for CoreSites service. @@ -23,6 +22,22 @@ import { CoreSite } from '@classes/site'; export const SITES_TABLE_NAME = 'sites_2'; export const SCHEMA_VERSIONS_TABLE_NAME = 'schema_versions'; +/** + * Database variables for CoreSite class. + */ +export enum WSGroups { + CORE = 'core', + BLOCK = 'block', + MOD = 'mod', + TOOL = 'tool', + OTHER = 'other', +} +export const WS_CACHE_OLD_TABLE = 'wscache_2'; +export const WS_CACHE_TABLES_PREFIX = 'wscache_'; +export const CONFIG_TABLE = 'core_site_config'; +export const LAST_VIEWED_TABLE = 'core_site_last_viewed'; +export const WS_CACHE_TABLES = Object.values(WSGroups).map(group => WS_CACHE_TABLES_PREFIX + group); + // Schema to register in App DB. export const APP_SCHEMA: CoreAppSchema = { name: 'CoreSitesProvider', @@ -78,41 +93,40 @@ export const APP_SCHEMA: CoreAppSchema = { // Schema to register for Site DB. export const SITE_SCHEMA: CoreSiteSchema = { name: 'CoreSitesProvider', - version: 3, - canBeCleared: [CoreSite.WS_CACHE_TABLE], - tables: [ + version: 4, + canBeCleared: [...WS_CACHE_TABLES, WS_CACHE_OLD_TABLE], + tables: WS_CACHE_TABLES.concat(WS_CACHE_OLD_TABLE).map(name => ({ + name: name, + columns: [ + { + name: 'id', + type: 'TEXT', + primaryKey: true, + }, + { + name: 'data', + type: 'TEXT', + }, + { + name: 'key', + type: 'TEXT', + }, + { + name: 'expirationTime', + type: 'INTEGER', + }, + { + name: 'component', + type: 'TEXT', + }, + { + name: 'componentId', + type: 'INTEGER', + }, + ], + })).concat([ { - name: CoreSite.WS_CACHE_TABLE, - columns: [ - { - name: 'id', - type: 'TEXT', - primaryKey: true, - }, - { - name: 'data', - type: 'TEXT', - }, - { - name: 'key', - type: 'TEXT', - }, - { - name: 'expirationTime', - type: 'INTEGER', - }, - { - name: 'component', - type: 'TEXT', - }, - { - name: 'componentId', - type: 'INTEGER', - }, - ], - }, - { - name: CoreSite.CONFIG_TABLE, + name: CONFIG_TABLE, columns: [ { name: 'name', @@ -126,7 +140,7 @@ export const SITE_SCHEMA: CoreSiteSchema = { ], }, { - name: CoreSite.LAST_VIEWED_TABLE, + name: LAST_VIEWED_TABLE, columns: [ { name: 'component', @@ -152,19 +166,7 @@ export const SITE_SCHEMA: CoreSiteSchema = { ], primaryKeys: ['component', 'id'], }, - ], - async migrate(db: SQLiteDB, oldVersion: number): Promise { - if (oldVersion < 2) { - await db.migrateTable('wscache', CoreSite.WS_CACHE_TABLE, (record) => ({ - id: record.id, - data: record.data, - key: record.key, - expirationTime: record.expirationTime, - component: null, - componentId: null, - })); - } - }, + ]), }; // Table for site DB to include the schema versions. It's not part of SITE_SCHEMA because it needs to be created first. @@ -198,3 +200,25 @@ export type SchemaVersionsDBEntry = { name: string; version: number; }; + +export type CoreSiteConfigDBRecord = { + name: string; + value: string | number; +}; + +export type CoreSiteWSCacheRecord = { + id: string; + data: string; + expirationTime: number; + key?: string; + component?: string; + componentId?: number; +}; + +export type CoreSiteLastViewedDBRecord = { + component: string; + id: number; + value: string; + timeaccess: number; + data?: string; +}; diff --git a/upgrade.txt b/upgrade.txt index 7576f9804..88775bda3 100644 --- a/upgrade.txt +++ b/upgrade.txt @@ -5,6 +5,7 @@ information provided here is intended especially for developers. - Zoom levels changed from "normal / low / high" to " none / medium / high". - --addon-messages-* CSS3 variables have been renamed to --core-messages-* +- The database constants in CoreSite (WS_CACHE_TABLE, CONFIG_TABLE, LAST_VIEWED_TABLE) and the DBRecord types have been moved to src/core/services/database. === 4.0.0 ===