EVOgeek
/
Vmeda.Online
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MOBILE-3981 core: Optimize filepool tables

main
Noel De Martin 2022-02-08 14:38:35 +01:00
parent 26482ea355
commit a626930e72
9 changed files with 489 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
src/core/classes/database/database-table-proxy.ts
View File

@ -17,7 +17,13 @@ import { asyncInstance } from '@/core/utils/async-instance';
import { SQLiteDB, SQLiteDBRecordValues } from '@classes/sqlitedb';
import { CoreConfig, CoreConfigProvider } from '@services/config';
import { CoreEventObserver, CoreEvents } from '@singletons/events';
import { CoreDatabaseReducer, CoreDatabaseTable, CoreDatabaseConditions, GetDBRecordPrimaryKey } from './database-table';
import {
CoreDatabaseReducer,
CoreDatabaseTable,
CoreDatabaseConditions,
GetDBRecordPrimaryKey,
CoreDatabaseQueryOptions,
} from './database-table';
import { CoreDebugDatabaseTable } from './debug-database-table';
import { CoreEagerDatabaseTable } from './eager-database-table';
import { CoreLazyDatabaseTable } from './lazy-database-table';
@ -67,15 +73,25 @@ export class CoreDatabaseTableProxy<
/**
* @inheritdoc
*/
async getMany(conditions?: Partial<DBRecord>): Promise<DBRecord[]> {
return this.target.getMany(conditions);
async getMany(conditions?: Partial<DBRecord>, options?: Partial<CoreDatabaseQueryOptions<DBRecord>>): Promise<DBRecord[]> {
return this.target.getMany(conditions, options);
}
/**
* @inheritdoc
*/
async getOne(conditions: Partial<DBRecord>): Promise<DBRecord> {
return this.target.getOne(conditions);
getManyWhere(conditions: CoreDatabaseConditions<DBRecord>): Promise<DBRecord[]> {
return this.target.getManyWhere(conditions);
}
/**
* @inheritdoc
*/
async getOne(
conditions?: Partial<DBRecord>,
options?: Partial<Omit<CoreDatabaseQueryOptions<DBRecord>, 'offset' | 'limit'>>,
): Promise<DBRecord> {
return this.target.getOne(conditions, options);
}
/**
@ -92,6 +108,20 @@ export class CoreDatabaseTableProxy<
return this.target.reduce<T>(reducer, conditions);
}
/**
* @inheritdoc
*/
hasAny(conditions?: Partial<DBRecord>): Promise<boolean> {
return this.target.hasAny(conditions);
}
/**
* @inheritdoc
*/
count(conditions?: Partial<DBRecord>): Promise<number> {
return this.target.count(conditions);
}
/**
* @inheritdoc
*/

172
src/core/classes/database/database-table.ts
View File

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
import { CoreError } from '@classes/errors/error';
import { SQLiteDB, SQLiteDBRecordValue, SQLiteDBRecordValues } from '@classes/sqlitedb';
/**
@ -78,22 +79,57 @@ export class CoreDatabaseTable<
* Get records matching the given conditions.
*
* @param conditions Matching conditions. If this argument is missing, all records in the table will be returned.
* @param options Query options.
* @returns Database records.
*/
getMany(conditions?: Partial<DBRecord>): Promise<DBRecord[]> {
return conditions
? this.database.getRecords(this.tableName, conditions)
: this.database.getAllRecords(this.tableName);
getMany(conditions?: Partial<DBRecord>, options?: Partial<CoreDatabaseQueryOptions<DBRecord>>): Promise<DBRecord[]> {
if (!conditions && !options) {
return this.database.getAllRecords(this.tableName);
}
const sorting = options?.sorting
&& this.normalizedSorting(options.sorting).map(([column, direction]) => `${column} ${direction}`).join(', ');
return this.database.getRecords(this.tableName, conditions, sorting, '*', options?.offset, options?.limit);
}
/**
* Get records matching the given conditions.
*
* This method should be used when it's necessary to apply complex conditions; the simple `getMany`
* method should be favored otherwise for better performance.
*
* @param conditions Matching conditions in SQL and JavaScript.
*/
getManyWhere(conditions: CoreDatabaseConditions<DBRecord>): Promise<DBRecord[]> {
return this.database.getRecordsSelect(this.tableName, conditions.sql, conditions.sqlParams);
}
/**
* Find one record matching the given conditions.
*
* @param conditions Matching conditions.
* @param options Result options.
* @returns Database record.
*/
getOne(conditions: Partial<DBRecord>): Promise<DBRecord> {
return this.database.getRecord<DBRecord>(this.tableName, conditions);
async getOne(
conditions?: Partial<DBRecord>,
options?: Partial<Omit<CoreDatabaseQueryOptions<DBRecord>, 'offset' | 'limit'>>,
): Promise<DBRecord> {
if (!options) {
return this.database.getRecord<DBRecord>(this.tableName, conditions);
}
const records = await this.getMany(conditions, {
...options,
limit: 1,
});
if (records.length === 0) {
throw new CoreError('No records found.');
}
return records[0];
}
/**
@ -121,6 +157,43 @@ export class CoreDatabaseTable<
) as unknown as Promise<T>;
}
/**
* Check whether the table is empty or not.
*
* @returns Whether the table is empty or not.
*/
isEmpty(): Promise<boolean> {
return this.hasAny();
}
/**
* Check whether the table has any record matching the given conditions.
*
* @param conditions Matching conditions. If this argument is missing, this method will return whether the table
* is empty or not.
* @returns Whether the table contains any records matching the given conditions.
*/
async hasAny(conditions?: Partial<DBRecord>): Promise<boolean> {
try {
await this.getOne(conditions);
return true;
} catch (error) {
// Couldn't get a single record.
return false;
}
}
/**
* Count records in table.
*
* @param conditions Matching conditions.
* @returns Number of records matching the given conditions.
*/
count(conditions?: Partial<DBRecord>): Promise<number> {
return this.database.countRecords(this.tableName, conditions);
}
/**
* Insert a new record.
*
@ -208,6 +281,59 @@ export class CoreDatabaseTable<
return !Object.entries(conditions).some(([column, value]) => record[column] !== value);
}
/**
* Sort a list of records with the given order. This method mutates the input array.
*
* @param records Array of records to sort.
* @param sorting Sorting conditions.
* @returns Sorted array. This will be the same reference that was given as an argument.
*/
protected sortRecords(records: DBRecord[], sorting: CoreDatabaseSorting<DBRecord>): DBRecord[] {
const columnsSorting = this.normalizedSorting(sorting);
records.sort((a, b) => {
for (const [column, direction] of columnsSorting) {
const aValue = a[column];
const bValue = b[column];
if (aValue > bValue) {
return direction === 'desc' ? -1 : 1;
}
if (aValue < bValue) {
return direction === 'desc' ? 1 : -1;
}
}
return 0;
});
return records;
}
/**
* Get a normalized array of sorting conditions.
*
* @param sorting Sorting conditions.
* @returns Normalized sorting conditions.
*/
protected normalizedSorting(sorting: CoreDatabaseSorting<DBRecord>): [keyof DBRecord, 'asc' | 'desc'][] {
const sortingArray = Array.isArray(sorting) ? sorting : [sorting];
return sortingArray.reduce((normalizedSorting, columnSorting) => {
normalizedSorting.push(
typeof columnSorting === 'object'
? [
Object.keys(columnSorting)[0] as keyof DBRecord,
Object.values(columnSorting)[0] as 'asc' | 'desc',
]
: [columnSorting, 'asc'],
);
return normalizedSorting;
}, [] as [keyof DBRecord, 'asc' | 'desc'][]);
}
}
/**
@ -238,3 +364,37 @@ export type CoreDatabaseConditions<DBRecord> = {
sqlParams?: SQLiteDBRecordValue[];
js: (record: DBRecord) => boolean;
};
/**
* Sorting conditions for a single column.
*
* This type will accept an object that defines sorting conditions for a single column, but not more.
* For example, `{id: 'desc'}` and `{name: 'asc'}` would be acceptend values, but `{id: 'desc', name: 'asc'}` wouldn't.
*
* @see https://stackoverflow.com/questions/57571664/typescript-type-for-an-object-with-only-one-key-no-union-type-allowed-as-a-key
*/
export type CoreDatabaseColumnSorting<DBRecordColumn extends string | symbol | number> = {
[Column in DBRecordColumn]:
(Record<Column, 'asc' | 'desc'> & Partial<Record<Exclude<DBRecordColumn, Column>, never>>) extends infer ColumnSorting
? { [Column in keyof ColumnSorting]: ColumnSorting[Column] }
: never;
}[DBRecordColumn];
/**
* Sorting conditions to apply to query results.
*
* Columns will be sorted in ascending order by default.
*/
export type CoreDatabaseSorting<DBRecord> =
keyof DBRecord |
CoreDatabaseColumnSorting<keyof DBRecord> |
Array<keyof DBRecord | CoreDatabaseColumnSorting<keyof DBRecord>>;
/**
* Options to configure query results.
*/
export type CoreDatabaseQueryOptions<DBRecord> = {
offset: number;
limit: number;
sorting: CoreDatabaseSorting<DBRecord>;
};

50
src/core/classes/database/debug-database-table.ts
View File

@ -14,7 +14,13 @@
import { SQLiteDBRecordValues } from '@classes/sqlitedb';
import { CoreLogger } from '@singletons/logger';
import { CoreDatabaseTable, CoreDatabaseConditions, GetDBRecordPrimaryKey, CoreDatabaseReducer } from './database-table';
import {
CoreDatabaseTable,
CoreDatabaseConditions,
GetDBRecordPrimaryKey,
CoreDatabaseReducer,
CoreDatabaseQueryOptions,
} from './database-table';
/**
* Database table proxy used to debug runtime operations.
@ -58,19 +64,31 @@ export class CoreDebugDatabaseTable<
/**
* @inheritdoc
*/
getMany(conditions?: Partial<DBRecord>): Promise<DBRecord[]> {
this.logger.log('getMany', conditions);
getMany(conditions?: Partial<DBRecord>, options?: Partial<CoreDatabaseQueryOptions<DBRecord>>): Promise<DBRecord[]> {
this.logger.log('getMany', conditions, options);
return this.target.getMany(conditions);
return this.target.getMany(conditions, options);
}
/**
* @inheritdoc
*/
getOne(conditions: Partial<DBRecord>): Promise<DBRecord> {
this.logger.log('getOne', conditions);
getManyWhere(conditions: CoreDatabaseConditions<DBRecord>): Promise<DBRecord[]> {
this.logger.log('getManyWhere', conditions);
return this.target.getOne(conditions);
return this.target.getManyWhere(conditions);
}
/**
* @inheritdoc
*/
getOne(
conditions?: Partial<DBRecord>,
options?: Partial<Omit<CoreDatabaseQueryOptions<DBRecord>, 'offset' | 'limit'>>,
): Promise<DBRecord> {
this.logger.log('getOne', conditions, options);
return this.target.getOne(conditions, options);
}
/**
@ -91,6 +109,24 @@ export class CoreDebugDatabaseTable<
return this.target.reduce<T>(reducer, conditions);
}
/**
* @inheritdoc
*/
hasAny(conditions?: Partial<DBRecord>): Promise<boolean> {
this.logger.log('hasAny', conditions);
return this.target.hasAny(conditions);
}
/**
* @inheritdoc
*/
count(conditions?: Partial<DBRecord>): Promise<number> {
this.logger.log('count', conditions);
return this.target.count(conditions);
}
/**
* @inheritdoc
*/

61
src/core/classes/database/eager-database-table.ts
View File

@ -14,7 +14,13 @@
import { CoreError } from '@classes/errors/error';
import { SQLiteDBRecordValues } from '@classes/sqlitedb';
import { CoreDatabaseTable, CoreDatabaseConditions, GetDBRecordPrimaryKey, CoreDatabaseReducer } from './database-table';
import {
CoreDatabaseTable,
CoreDatabaseConditions,
GetDBRecordPrimaryKey,
CoreDatabaseReducer,
CoreDatabaseQueryOptions,
} from './database-table';
/**
* Wrapper used to improve performance by caching all the records for faster read operations.
@ -48,21 +54,44 @@ export class CoreEagerDatabaseTable<
/**
* @inheritdoc
*/
async getMany(conditions?: Partial<DBRecord>): Promise<DBRecord[]> {
async getMany(conditions?: Partial<DBRecord>, options?: Partial<CoreDatabaseQueryOptions<DBRecord>>): Promise<DBRecord[]> {
const records = Object.values(this.records);
return conditions
const filteredRecords = conditions
? records.filter(record => this.recordMatches(record, conditions))
: records;
if (options?.sorting) {
this.sortRecords(filteredRecords, options.sorting);
}
return filteredRecords.slice(options?.offset ?? 0, options?.limit);
}
/**
* @inheritdoc
*/
async getOne(conditions: Partial<DBRecord>): Promise<DBRecord> {
const record = Object.values(this.records).find(record => this.recordMatches(record, conditions)) ?? null;
async getManyWhere(conditions: CoreDatabaseConditions<DBRecord>): Promise<DBRecord[]> {
return Object.values(this.records).filter(record => conditions.js(record));
}
if (record === null) {
/**
* @inheritdoc
*/
async getOne(
conditions?: Partial<DBRecord>,
options?: Partial<Omit<CoreDatabaseQueryOptions<DBRecord>, 'offset' | 'limit'>>,
): Promise<DBRecord> {
let record: DBRecord | undefined;
if (options?.sorting) {
record = this.getMany(conditions, { ...options, limit: 1 })[0];
} else if (conditions) {
record = Object.values(this.records).find(record => this.recordMatches(record, conditions));
} else {
record = Object.values(this.records)[0];
}
if (!record) {
throw new CoreError('No records found.');
}
@ -94,6 +123,24 @@ export class CoreEagerDatabaseTable<
);
}
/**
* @inheritdoc
*/
async hasAny(conditions?: Partial<DBRecord>): Promise<boolean> {
return conditions
? Object.values(this.records).some(record => this.recordMatches(record, conditions))
: Object.values(this.records).length > 0;
}
/**
* @inheritdoc
*/
async count(conditions?: Partial<DBRecord>): Promise<number> {
return conditions
? Object.values(this.records).filter(record => this.recordMatches(record, conditions)).length
: Object.values(this.records).length;
}
/**
* @inheritdoc
*/

31
src/core/classes/database/lazy-database-table.ts
View File

@ -14,7 +14,7 @@
import { CoreError } from '@classes/errors/error';
import { SQLiteDBRecordValues } from '@classes/sqlitedb';
import { CoreDatabaseTable, CoreDatabaseConditions, GetDBRecordPrimaryKey } from './database-table';
import { CoreDatabaseTable, CoreDatabaseConditions, GetDBRecordPrimaryKey, CoreDatabaseQueryOptions } from './database-table';
/**
* Wrapper used to improve performance by caching records that are used often for faster read operations.
@ -33,15 +33,13 @@ export class CoreLazyDatabaseTable<
/**
* @inheritdoc
*/
async getOne(conditions: Partial<DBRecord>): Promise<DBRecord> {
let record: DBRecord | null =
Object.values(this.records).find(record => record && this.recordMatches(record, conditions)) ?? null;
async getOne(
conditions?: Partial<DBRecord>,
options?: Partial<Omit<CoreDatabaseQueryOptions<DBRecord>, 'offset' | 'limit'>>,
): Promise<DBRecord> {
const record = await super.getOne(conditions, options);
if (!record) {
record = await super.getOne(conditions);
this.records[this.serializePrimaryKey(this.getPrimaryKeyFromRecord(record))] = record;
}
this.records[this.serializePrimaryKey(this.getPrimaryKeyFromRecord(record))] = record;
return record;
}
@ -75,6 +73,21 @@ export class CoreLazyDatabaseTable<
return record;
}
/**
* @inheritdoc
*/
async hasAny(conditions?: Partial<DBRecord>): Promise<boolean> {
const hasAnyMatching = Object
.values(this.records)
.some(record => record !== null && (!conditions || this.recordMatches(record, conditions)));
if (hasAnyMatching) {
return true;
}
return super.hasAny(conditions);
}
/**
* @inheritdoc
*/

88
src/core/classes/sqlitedb.ts
View File

@ -136,6 +136,50 @@ export interface SQLiteDBForeignKeySchema {
*/
export class SQLiteDB {
/**
* Constructs 'IN()' or '=' sql fragment
*
* @param items A single value or array of values for the expression. It doesn't accept objects.
* @param equal True means we want to equate to the constructed expression.
* @param onEmptyItems This defines the behavior when the array of items provided is empty. Defaults to false,
* meaning return empty. Other values will become part of the returned SQL fragment.
* @return A list containing the constructed sql fragment and an array of parameters.
*/
static getInOrEqual(
items: SQLiteDBRecordValue | SQLiteDBRecordValue[],
equal: boolean = true,
onEmptyItems?: SQLiteDBRecordValue | null,
): SQLiteDBQueryParams {
let sql = '';
let params: SQLiteDBRecordValue[];
// Default behavior, return empty data on empty array.
if (Array.isArray(items) && !items.length && onEmptyItems === undefined) {
return { sql: '', params: [] };
}
// Handle onEmptyItems on empty array of items.
if (Array.isArray(items) && !items.length) {
if (onEmptyItems === null) { // Special case, NULL value.
sql = equal ? ' IS NULL' : ' IS NOT NULL';
return { sql, params: [] };
} else {
items = [onEmptyItems as SQLiteDBRecordValue]; // Rest of cases, prepare items for processing.
}
}
if (!Array.isArray(items) || items.length == 1) {
sql = equal ? '= ?' : '<> ?';
params = Array.isArray(items) ? items : [items];
} else {
sql = (equal ? '' : 'NOT ') + 'IN (' + ',?'.repeat(items.length).substring(1) + ')';
params = items;
}
return { sql, params };
}
db?: SQLiteObject;
promise!: Promise<void>;
@ -564,50 +608,6 @@ export class SQLiteDB {
return record[Object.keys(record)[0]];
}
/**
* Constructs 'IN()' or '=' sql fragment
*
* @param items A single value or array of values for the expression. It doesn't accept objects.
* @param equal True means we want to equate to the constructed expression.
* @param onEmptyItems This defines the behavior when the array of items provided is empty. Defaults to false,
* meaning return empty. Other values will become part of the returned SQL fragment.
* @return A list containing the constructed sql fragment and an array of parameters.
*/
getInOrEqual(
items: SQLiteDBRecordValue | SQLiteDBRecordValue[],
equal: boolean = true,
onEmptyItems?: SQLiteDBRecordValue | null,
): SQLiteDBQueryParams {
let sql = '';
let params: SQLiteDBRecordValue[];
// Default behavior, return empty data on empty array.
if (Array.isArray(items) && !items.length && onEmptyItems === undefined) {
return { sql: '', params: [] };
}
// Handle onEmptyItems on empty array of items.
if (Array.isArray(items) && !items.length) {
if (onEmptyItems === null || onEmptyItems === undefined) { // Special case, NULL value.
sql = equal ? ' IS NULL' : ' IS NOT NULL';
return { sql, params: [] };
} else {
items = [onEmptyItems]; // Rest of cases, prepare items for processing.
}
}
if (!Array.isArray(items) || items.length == 1) {
sql = equal ? '= ?' : '<> ?';
params = Array.isArray(items) ? items : [items];
} else {
sql = (equal ? '' : 'NOT ') + 'IN (' + ',?'.repeat(items.length).substring(1) + ')';
params = items;
}
return { sql, params };
}
/**
* Get the database name.
*

40
src/core/classes/tests/database-table.test.ts
View File

@ -13,20 +13,20 @@
// limitations under the License.
import { mock, mockSingleton } from '@/testing/utils';
import { CoreDatabaseTable } from '@classes/database/database-table';
import { CoreDatabaseSorting, CoreDatabaseTable } from '@classes/database/database-table';
import {
CoreDatabaseCachingStrategy,
CoreDatabaseConfiguration,
CoreDatabaseTableProxy,
} from '@classes/database/database-table-proxy';
import { SQLiteDB, SQLiteDBRecordValues } from '@classes/sqlitedb';
import { SQLiteDB } from '@classes/sqlitedb';
import { CoreConfig } from '@services/config';
interface User extends SQLiteDBRecordValues {
type User = {
id: number;
name: string;
surname: string;
}
};
function userMatches(user: User, conditions: Partial<User>) {
return !Object.entries(conditions).some(([column, value]) => user[column] !== value);
@ -45,7 +45,7 @@ function prepareStubs(config: Partial<CoreDatabaseConfiguration> = {}): [User[],
return record as unknown as T;
},
getRecords: async <T>(_, conditions) => records.filter(record => userMatches(record, conditions)) as unknown as T[],
getAllRecords: async <T>() => records as unknown as T[],
getAllRecords: async <T>() => records.slice(0) as unknown as T[],
deleteRecords: async (_, conditions) => {
const usersToDelete: User[] = [];
@ -81,10 +81,10 @@ async function testFindItems(records: User[], table: CoreDatabaseTable<User>) {
await table.initialize();
await expect(table.getOneByPrimaryKey({ id: 1 })).resolves.toEqual(john);
await expect(table.getOneByPrimaryKey({ id: 2 })).resolves.toEqual(amy);
await expect(table.getOne({ surname: 'Doe', name: 'John' })).resolves.toEqual(john);
await expect(table.getOne({ surname: 'Doe', name: 'Amy' })).resolves.toEqual(amy);
await expect(table.getOneByPrimaryKey({ id: 1 })).resolves.toEqual(john);
await expect(table.getOneByPrimaryKey({ id: 2 })).resolves.toEqual(amy);
}
async function testInsertItems(records: User[], database: SQLiteDB, table: CoreDatabaseTable<User>) {
@ -165,6 +165,32 @@ describe('CoreDatabaseTable with eager caching', () => {
expect(database.getRecord).not.toHaveBeenCalled();
});
it('sorts items', async () => {
// Arrange.
const john = { id: 1, name: 'John', surname: 'Doe' };
const amy = { id: 2, name: 'Amy', surname: 'Doe' };
const jane = { id: 3, name: 'Jane', surname: 'Smith' };
const expectSorting = async (sorting: CoreDatabaseSorting<User>, expectedResults: User[]) => {
const results = await table.getMany({}, { sorting });
expect(results).toEqual(expectedResults);
};
records.push(john);
records.push(amy);
records.push(jane);
await table.initialize();
// Act & Assert.
await expectSorting('name', [amy, jane, john]);
await expectSorting('surname', [john, amy, jane]);
await expectSorting({ name: 'desc' }, [john, jane, amy]);
await expectSorting({ surname: 'desc' }, [jane, john, amy]);
await expectSorting(['name', { surname: 'desc' }], [amy, jane, john]);
await expectSorting([{ surname: 'desc' }, 'name'], [jane, amy, john]);
});
it('inserts items', () => testInsertItems(records, database, table));
it('deletes items', () => testDeleteItems(records, database, table));
it('deletes items by primary key', () => testDeleteItemsByPrimaryKey(records, database, table));

3
src/core/features/h5p/classes/framework.ts
View File

@ -43,6 +43,7 @@ import { CoreH5PContentBeingSaved, CoreH5PLibraryBeingSaved } from './storage';
import { CoreH5PLibraryAddTo, CoreH5PLibraryMetadataSettings } from './validator';
import { CoreH5PMetadata } from './metadata';
import { Translate } from '@singletons';
import { SQLiteDB } from '@classes/sqlitedb';
/**
* Equivalent to Moodle's implementation of H5PFrameworkInterface.
@ -64,7 +65,7 @@ export class CoreH5PFramework {
const db = await CoreSites.getSiteDb(siteId);
const whereAndParams = db.getInOrEqual(libraryIds);
const whereAndParams = SQLiteDB.getInOrEqual(libraryIds);
whereAndParams.sql = 'mainlibraryid ' + whereAndParams.sql;
await db.updateRecordsWhere(CONTENT_TABLE_NAME, { filtered: null }, whereAndParams.sql, whereAndParams.params);

186
src/core/services/filepool.ts
View File

@ -49,7 +49,7 @@ import {
import { CoreFileHelper } from './file-helper';
import { CoreUrl } from '@singletons/url';
import { CoreDatabaseTable } from '@classes/database/database-table';
import { CoreDatabaseCachingStrategy } from '@classes/database/database-table-proxy';
import { CoreDatabaseCachingStrategy, CoreDatabaseTableProxy } from '@classes/database/database-table-proxy';
import { lazyMap, LazyMap } from '../utils/lazy-map';
import { asyncInstance, AsyncInstance } from '../utils/async-instance';
@ -98,14 +98,14 @@ export class CoreFilepoolProvider {
// Variables to prevent downloading packages/files twice at the same time.
protected packagesPromises: { [s: string]: { [s: string]: Promise<void> } } = {};
protected filePromises: { [s: string]: { [s: string]: Promise<string> } } = {};
// Variables for DB.
protected appDB: Promise<SQLiteDB>;
protected resolveAppDB!: (appDB: SQLiteDB) => void;
protected filesTables: LazyMap<AsyncInstance<CoreDatabaseTable<CoreFilepoolFileEntry, 'fileId'>>>;
protected linksTables:
LazyMap<AsyncInstance<CoreDatabaseTable<CoreFilepoolLinksRecord, 'fileId' | 'component' | 'componentId'>>>;
protected packagesTables: LazyMap<AsyncInstance<CoreDatabaseTable<CoreFilepoolPackageEntry>>>;
protected queueTable = asyncInstance<CoreDatabaseTable<CoreFilepoolQueueDBEntry, 'siteId' | 'fileId'>>();
constructor() {
this.appDB = new Promise(resolve => this.resolveAppDB = resolve);
this.logger = CoreLogger.getInstance('CoreFilepoolProvider');
this.filesTables = lazyMap(
siteId => asyncInstance(
@ -116,6 +116,23 @@ export class CoreFilepoolProvider {
}),
),
);
this.linksTables = lazyMap(
siteId => asyncInstance(
() => CoreSites.getSiteTable<CoreFilepoolLinksRecord, 'fileId' | 'component' | 'componentId'>(LINKS_TABLE_NAME, {
siteId,
config: { cachingStrategy: CoreDatabaseCachingStrategy.Lazy },
primaryKeyColumns: ['fileId', 'component', 'componentId'],
}),
),
);
this.packagesTables = lazyMap(
siteId => asyncInstance(
() => CoreSites.getSiteTable<CoreFilepoolPackageEntry, 'id'>(PACKAGES_TABLE_NAME, {
siteId,
config: { cachingStrategy: CoreDatabaseCachingStrategy.Lazy },
}),
),
);
}
/**
@ -154,7 +171,16 @@ export class CoreFilepoolProvider {
// Ignore errors.
}
this.resolveAppDB(CoreApp.getDB());
const queueTable = new CoreDatabaseTableProxy<CoreFilepoolQueueDBEntry, 'siteId' | 'fileId'>(
{ cachingStrategy: CoreDatabaseCachingStrategy.Lazy },
CoreApp.getDB(),
QUEUE_TABLE_NAME,
['siteId','fileId'],
);
await queueTable.initialize();
this.queueTable.setInstance(queueTable);
}
/**
@ -171,16 +197,11 @@ export class CoreFilepoolProvider {
throw new CoreError('Cannot add link because component is invalid.');
}
componentId = this.fixComponentId(componentId);
const db = await CoreSites.getSiteDb(siteId);
const newEntry: CoreFilepoolLinksRecord = {
await this.linksTables[siteId].insert({
fileId,
component,
componentId: componentId || '',
};
await db.insertRecord(LINKS_TABLE_NAME, newEntry);
componentId: this.fixComponentId(componentId) || '',
});
}
/**
@ -301,9 +322,7 @@ export class CoreFilepoolProvider {
): Promise<void> {
this.logger.debug(`Adding ${fileId} to the queue`);
const db = await this.appDB;
await db.insertRecord(QUEUE_TABLE_NAME, {
await this.queueTable.insert({
siteId,
fileId,
url,
@ -431,10 +450,7 @@ export class CoreFilepoolProvider {
// Update only when required.
this.logger.debug(`Updating file ${fileId} which is already in queue`);
const db = await this.appDB;
return db.updateRecords(QUEUE_TABLE_NAME, newData, primaryKey).then(() =>
this.getQueuePromise(siteId, fileId, true, onProgress));
return this.queueTable.update(newData, primaryKey).then(() => this.getQueuePromise(siteId, fileId, true, onProgress));
}
this.logger.debug(`File ${fileId} already in queue and does not require update`);
@ -560,11 +576,10 @@ export class CoreFilepoolProvider {
async clearAllPackagesStatus(siteId: string): Promise<void> {
this.logger.debug('Clear all packages status for site ' + siteId);
const site = await CoreSites.getSite(siteId);
// Get all the packages to be able to "notify" the change in the status.
const entries: CoreFilepoolPackageEntry[] = await site.getDb().getAllRecords(PACKAGES_TABLE_NAME);
const entries = await this.packagesTables[siteId].getMany();
// Delete all the entries.
await site.getDb().deleteRecords(PACKAGES_TABLE_NAME);
await this.packagesTables[siteId].delete();
entries.forEach((entry) => {
if (!entry.component) {
@ -583,15 +598,13 @@ export class CoreFilepoolProvider {
* @return Promise resolved when the filepool is cleared.
*/
async clearFilepool(siteId: string): Promise<void> {
const db = await CoreSites.getSiteDb(siteId);
// Read the data first to be able to notify the deletions.
const filesEntries = await this.filesTables[siteId].getMany();
const filesLinks = await db.getAllRecords<CoreFilepoolLinksRecord>(LINKS_TABLE_NAME);
const filesLinks = await this.linksTables[siteId].getMany();
await Promise.all([
this.filesTables[siteId].delete(),
db.deleteRecords(LINKS_TABLE_NAME),
this.linksTables[siteId].delete(),
]);
// Notify now.
@ -609,14 +622,14 @@ export class CoreFilepoolProvider {
* @return Resolved means yes, rejected means no.
*/
async componentHasFiles(siteId: string, component: string, componentId?: string | number): Promise<void> {
const db = await CoreSites.getSiteDb(siteId);
const conditions = {
component,
componentId: this.fixComponentId(componentId),
};
const count = await db.countRecords(LINKS_TABLE_NAME, conditions);
if (count <= 0) {
const hasAnyLinks = await this.linksTables[siteId].hasAny(conditions);
if (!hasAnyLinks) {
throw new CoreError('Component doesn\'t have files');
}
}
@ -1144,7 +1157,6 @@ export class CoreFilepoolProvider {
return;
}
const db = await CoreSites.getSiteDb(siteId);
const extension = CoreMimetypeUtils.getFileExtension(entry.path);
if (!extension) {
// Files does not have extension. Invalidate file (stale = true).
@ -1170,7 +1182,7 @@ export class CoreFilepoolProvider {
}
// Now update the links.
await db.updateRecords(LINKS_TABLE_NAME, { fileId: entry.fileId }, { fileId });
await this.linksTables[siteId].update({ fileId: entry.fileId }, { fileId });
}
/**
@ -1228,16 +1240,18 @@ export class CoreFilepoolProvider {
* @return Promise resolved with the files.
*/
protected async getComponentFiles(
db: SQLiteDB,
siteId: string | undefined,
component: string,
componentId?: string | number,
): Promise<CoreFilepoolLinksRecord[]> {
siteId = siteId ?? CoreSites.getCurrentSiteId();
const conditions = {
component,
componentId: this.fixComponentId(componentId),
};
const items = await db.getRecords<CoreFilepoolLinksRecord>(LINKS_TABLE_NAME, conditions);
const items = await this.linksTables[siteId].getMany(conditions);
items.forEach((item) => {
item.componentId = this.fixComponentId(item.componentId);
});
@ -1349,8 +1363,7 @@ export class CoreFilepoolProvider {
* @return Promise resolved with the links.
*/
protected async getFileLinks(siteId: string, fileId: string): Promise<CoreFilepoolLinksRecord[]> {
const db = await CoreSites.getSiteDb(siteId);
const items = await db.getRecords<CoreFilepoolLinksRecord>(LINKS_TABLE_NAME, { fileId });
const items = await this.linksTables[siteId].getMany({ fileId });
items.forEach((item) => {
item.componentId = this.fixComponentId(item.componentId);
@ -1421,8 +1434,7 @@ export class CoreFilepoolProvider {
* @return Promise resolved with the files on success.
*/
async getFilesByComponent(siteId: string, component: string, componentId?: string | number): Promise<CoreFilepoolFileEntry[]> {
const db = await CoreSites.getSiteDb(siteId);
const items = await this.getComponentFiles(db, component, componentId);
const items = await this.getComponentFiles(siteId, component, componentId);
const files: CoreFilepoolFileEntry[] = [];
await Promise.all(items.map(async (item) => {
@ -1706,10 +1718,9 @@ export class CoreFilepoolProvider {
async getPackageData(siteId: string, component: string, componentId?: string | number): Promise<CoreFilepoolPackageEntry> {
componentId = this.fixComponentId(componentId);
const site = await CoreSites.getSite(siteId);
const packageId = this.getPackageId(component, componentId);
return site.getDb().getRecord(PACKAGES_TABLE_NAME, { id: packageId });
return this.packagesTables[siteId].getOneByPrimaryKey({ id: packageId });
}
/**
@ -2171,16 +2182,16 @@ export class CoreFilepoolProvider {
* @return Resolved with file object from DB on success, rejected otherwise.
*/
protected async hasFileInQueue(siteId: string, fileId: string): Promise<CoreFilepoolQueueEntry> {
const db = await this.appDB;
const entry = await db.getRecord<CoreFilepoolQueueEntry>(QUEUE_TABLE_NAME, { siteId, fileId });
const entry = await this.queueTable.getOneByPrimaryKey({ siteId, fileId });
if (entry === undefined) {
throw new CoreError('File not found in queue.');
}
// Convert the links to an object.
entry.linksUnserialized = <CoreFilepoolComponentLink[]> CoreTextUtils.parseJSON(entry.links || '[]', []);
return entry;
return {
...entry,
linksUnserialized: CoreTextUtils.parseJSON(entry.links, []),
};
}
/**
@ -2238,8 +2249,7 @@ export class CoreFilepoolProvider {
componentId?: string | number,
onlyUnknown: boolean = true,
): Promise<void> {
const db = await CoreSites.getSiteDb(siteId);
const items = await this.getComponentFiles(db, component, componentId);
const items = await this.getComponentFiles(siteId, component, componentId);
if (!items.length) {
// Nothing to invalidate.
@ -2250,7 +2260,7 @@ export class CoreFilepoolProvider {
const fileIds = items.map((item) => item.fileId);
const whereAndParams = db.getInOrEqual(fileIds);
const whereAndParams = SQLiteDB.getInOrEqual(fileIds);
whereAndParams.sql = 'fileId ' + whereAndParams.sql;
@ -2523,30 +2533,25 @@ export class CoreFilepoolProvider {
* @return Resolved on success. Rejected on failure.
*/
protected async processImportantQueueItem(): Promise<void> {
let items: CoreFilepoolQueueEntry[];
const db = await this.appDB;
try {
items = await db.getRecords<CoreFilepoolQueueEntry>(
QUEUE_TABLE_NAME,
undefined,
'priority DESC, added ASC',
undefined,
0,
1,
);
const item = await this.queueTable.getOne({}, {
sorting: [
{ priority: 'desc' },
{ added: 'asc' },
],
});
if (!item) {
throw CoreFilepoolProvider.ERR_QUEUE_IS_EMPTY;
}
return this.processQueueItem({
...item,
linksUnserialized: CoreTextUtils.parseJSON(item.links, []),
});
} catch (err) {
throw CoreFilepoolProvider.ERR_QUEUE_IS_EMPTY;
}
const item = items.pop();
if (!item) {
throw CoreFilepoolProvider.ERR_QUEUE_IS_EMPTY;
}
// Convert the links to an object.
item.linksUnserialized = <CoreFilepoolComponentLink[]> CoreTextUtils.parseJSON(item.links, []);
return this.processQueueItem(item);
}
/**
@ -2671,9 +2676,7 @@ export class CoreFilepoolProvider {
* @return Resolved on success. Rejected on failure. It is advised to silently ignore failures.
*/
protected async removeFromQueue(siteId: string, fileId: string): Promise<void> {
const db = await this.appDB;
await db.deleteRecords(QUEUE_TABLE_NAME, { siteId, fileId });
await this.queueTable.deleteByPrimaryKey({ siteId, fileId });
}
/**
@ -2684,8 +2687,6 @@ export class CoreFilepoolProvider {
* @return Resolved on success.
*/
protected async removeFileById(siteId: string, fileId: string): Promise<void> {
const db = await CoreSites.getSiteDb(siteId);
// Get the path to the file first since it relies on the file object stored in the pool.
// Don't use getFilePath to prevent performing 2 DB requests.
let path = this.getFilepoolFolderPath(siteId) + '/' + fileId;
@ -2714,7 +2715,7 @@ export class CoreFilepoolProvider {
promises.push(this.filesTables[siteId].delete(conditions));
// Remove links.
promises.push(db.deleteRecords(LINKS_TABLE_NAME, conditions));
promises.push(this.linksTables[siteId].delete(conditions));
// Remove the file.
if (CoreFile.isAvailable()) {
@ -2745,8 +2746,7 @@ export class CoreFilepoolProvider {
* @return Resolved on success.
*/
async removeFilesByComponent(siteId: string, component: string, componentId?: string | number): Promise<void> {
const db = await CoreSites.getSiteDb(siteId);
const items = await this.getComponentFiles(db, component, componentId);
const items = await this.getComponentFiles(siteId, component, componentId);
await Promise.all(items.map((item) => this.removeFileById(siteId, item.fileId)));
}
@ -2795,11 +2795,10 @@ export class CoreFilepoolProvider {
componentId = this.fixComponentId(componentId);
this.logger.debug(`Set previous status for package ${component} ${componentId}`);
const site = await CoreSites.getSite(siteId);
const packageId = this.getPackageId(component, componentId);
// Get current stored data, we'll only update 'status' and 'updated' fields.
const entry = <CoreFilepoolPackageEntry> site.getDb().getRecord(PACKAGES_TABLE_NAME, { id: packageId });
const entry = await this.packagesTables[siteId].getOneByPrimaryKey({ id: packageId });
const newData: CoreFilepoolPackageEntry = {};
if (entry.status == CoreConstants.DOWNLOADING) {
// Going back from downloading to previous status, restore previous download time.
@ -2809,9 +2808,9 @@ export class CoreFilepoolProvider {
newData.updated = Date.now();
this.logger.debug(`Set previous status '${entry.status}' for package ${component} ${componentId}`);
await site.getDb().updateRecords(PACKAGES_TABLE_NAME, newData, { id: packageId });
await this.packagesTables[siteId].update(newData, { id: packageId });
// Success updating, trigger event.
this.triggerPackageStatusChanged(site.getId(), newData.status, component, componentId);
this.triggerPackageStatusChanged(siteId, newData.status, component, componentId);
return newData.status;
}
@ -2900,7 +2899,6 @@ export class CoreFilepoolProvider {
this.logger.debug(`Set status '${status}' for package ${component} ${componentId}`);
componentId = this.fixComponentId(componentId);
const site = await CoreSites.getSite(siteId);
const packageId = this.getPackageId(component, componentId);
let downloadTime: number | undefined;
let previousDownloadTime: number | undefined;
@ -2913,7 +2911,7 @@ export class CoreFilepoolProvider {
let previousStatus: string | undefined;
// Search current status to set it as previous status.
try {
const entry = await site.getDb().getRecord<CoreFilepoolPackageEntry>(PACKAGES_TABLE_NAME, { id: packageId });
const entry = await this.packagesTables[siteId].getOneByPrimaryKey({ id: packageId });
extra = extra ?? entry.extra;
if (downloadTime === undefined) {
@ -2930,7 +2928,12 @@ export class CoreFilepoolProvider {
// No previous status.
}
const packageEntry: CoreFilepoolPackageEntry = {
if (previousStatus === status) {
// The package already has this status, no need to change it.
return;
}
await this.packagesTables[siteId].insert({
id: packageId,
component,
componentId,
@ -2940,14 +2943,7 @@ export class CoreFilepoolProvider {
downloadTime,
previousDownloadTime,
extra,
};
if (previousStatus === status) {
// The package already has this status, no need to change it.
return;
}
await site.getDb().insertRecord(PACKAGES_TABLE_NAME, packageEntry);
});
// Success inserting, trigger event.
this.triggerPackageStatusChanged(siteId, status, component, componentId);
@ -3067,11 +3063,9 @@ export class CoreFilepoolProvider {
async updatePackageDownloadTime(siteId: string, component: string, componentId?: string | number): Promise<void> {
componentId = this.fixComponentId(componentId);
const site = await CoreSites.getSite(siteId);
const packageId = this.getPackageId(component, componentId);
await site.getDb().updateRecords(
PACKAGES_TABLE_NAME,
await this.packagesTables[siteId].update(
{ downloadTime: CoreTimeUtils.timestamp() },
{ id: packageId },
);