From d14540218a9603a1cc29cb461f2aabd2b6dd6913 Mon Sep 17 00:00:00 2001 From: Dani Palou Date: Tue, 19 Jan 2021 10:00:40 +0100 Subject: [PATCH] MOBILE-3659 core: Split calls with too many parameters --- src/core/classes/site.ts | 8 +++ src/core/services/ws.ts | 113 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 115 insertions(+), 6 deletions(-) diff --git a/src/core/classes/site.ts b/src/core/classes/site.ts index cc24d2297..3f1d67d64 100644 --- a/src/core/classes/site.ts +++ b/src/core/classes/site.ts @@ -26,6 +26,7 @@ import { CoreWSAjaxPreSets, CoreWSExternalWarning, CoreWSUploadFileResult, + CoreWSPreSetsSplitRequest, } from '@services/ws'; import { CoreDomUtils } from '@services/utils/dom'; import { CoreTextUtils } from '@services/utils/text'; @@ -516,6 +517,7 @@ export class CoreSite { cleanUnicode: this.cleanUnicode, typeExpected: preSets.typeExpected, responseExpected: preSets.responseExpected, + splitRequest: preSets.splitRequest, }; if (wsPreSets.cleanUnicode && CoreTextUtils.instance.hasUnicodeData(data)) { @@ -2052,6 +2054,12 @@ export type CoreSiteWSPreSets = { * Component id. Optionally included when 'component' is set. */ componentId?: number; + + /** + * Whether to split a request if it has too many parameters. Sending too many parameters to the site + * can cause the request to fail (see PHP's max_input_vars). + */ + splitRequest?: CoreWSPreSetsSplitRequest; }; /** diff --git a/src/core/services/ws.ts b/src/core/services/ws.ts index cf82d0c22..a290219cc 100644 --- a/src/core/services/ws.ts +++ b/src/core/services/ws.ts @@ -72,11 +72,16 @@ export class CoreWSProvider { * * @param method The WebService method to be called. * @param siteUrl Complete site url to perform the call. - * @param ajaxData Arguments to pass to the method. + * @param data Arguments to pass to the method. * @param preSets Extra settings and information. * @return Deferred promise resolved with the response data in success and rejected with the error if it fails. */ - protected addToRetryQueue(method: string, siteUrl: string, data: unknown, preSets: CoreWSPreSets): Promise { + protected addToRetryQueue( + method: string, + siteUrl: string, + data: Record, + preSets: CoreWSPreSets, + ): Promise { const call = { method, siteUrl, @@ -98,7 +103,7 @@ export class CoreWSProvider { * @param preSets Extra settings and information. * @return Promise resolved with the response data in success and rejected if it fails. */ - call(method: string, data: unknown, preSets: CoreWSPreSets): Promise { + call(method: string, data: Record, preSets: CoreWSPreSets): Promise { if (!preSets) { throw new CoreError(Translate.instance.instant('core.unexpectederror')); } else if (!CoreApp.instance.isOnline()) { @@ -493,7 +498,7 @@ export class CoreWSProvider { } /** - * Perform the post call and save the promise while waiting to be resolved. + * Perform the post call. It can be split into several requests. * * @param method The WebService method to be called. * @param siteUrl Complete site url to perform the call. @@ -501,7 +506,12 @@ export class CoreWSProvider { * @param preSets Extra settings and information. * @return Promise resolved with the response data in success and rejected with CoreWSError if it fails. */ - performPost(method: string, siteUrl: string, ajaxData: unknown, preSets: CoreWSPreSets): Promise { + async performPost( + method: string, + siteUrl: string, + ajaxData: Record, + preSets: CoreWSPreSets, + ): Promise { // eslint-disable-next-line @typescript-eslint/no-explicit-any const options: any = {}; @@ -510,6 +520,71 @@ export class CoreWSProvider { options.responseType = 'text'; } + if (!preSets.splitRequest || !ajaxData[preSets.splitRequest.param]) { + return this.performSinglePost(method, siteUrl, ajaxData, preSets, options); + } + + // Split the request into several requests if needed. + const promises: Promise[] = []; + const splitParam = ajaxData[preSets.splitRequest.param]; + + for (let i = 0; i < splitParam.length; i += preSets.splitRequest.maxLength) { + // Limit the array sent. + const limitedData = Object.assign({}, ajaxData); + limitedData[preSets.splitRequest.param] = splitParam.slice(i, i + preSets.splitRequest.maxLength); + + promises.push(this.performSinglePost(method, siteUrl, limitedData, preSets, options)); + } + + const results = await Promise.all(promises); + + // Combine the results. + const firstResult = results.shift(); + + if (preSets.splitRequest.combineCallback) { + return results.reduce(preSets.splitRequest.combineCallback, firstResult); + } + + return results.reduce((previous: T, current: T) => this.combineObjectsArrays(previous, current), firstResult); + } + + /** + * Combine the arrays of two objects, adding them to the first object. + * + * @param object1 First object. + * @param object2 Second object. + * @return First object with items added. + */ + protected combineObjectsArrays(object1: T, object2: T): T { + for (const name in object2) { + const value = object2[name]; + + if (Array.isArray(value)) { + (object1 as Record)[name] = (object1[name] as typeof value).concat(value); + } + } + + return object1; + } + + /** + * Perform a single post request. + * + * @param method The WebService method to be called. + * @param siteUrl Complete site url to perform the call. + * @param ajaxData Arguments to pass to the method. + * @param preSets Extra settings and information. + * @param options Request options. + * @return Promise resolved with the response data in success and rejected with CoreWSError if it fails. + */ + protected performSinglePost( + method: string, + siteUrl: string, + ajaxData: Record, + preSets: CoreWSPreSets, + options: any, // eslint-disable-line @typescript-eslint/no-explicit-any + ): Promise { + // We add the method name to the URL purely to help with debugging. // This duplicates what is in the ajaxData, but that does no harm. // POST variables take precedence over GET. @@ -1089,6 +1164,32 @@ export type CoreWSPreSets = { * Defaults to false. Clean multibyte Unicode chars from data. */ cleanUnicode?: boolean; + + /** + * Whether to split a request if it has too many parameters. Sending too many parameters to the site + * can cause the request to fail (see PHP's max_input_vars). + */ + splitRequest?: CoreWSPreSetsSplitRequest; +}; + +/** + * Options to split a request. + */ +export type CoreWSPreSetsSplitRequest = { + /** + * Name of the parameter used to split the request if too big. Must be an array parameter. + */ + param: string; + + /** + * Max number of entries sent per request. + */ + maxLength: number; + + /** + * Callback to combine the results. If not supplied, arrays in the result will be concatenated. + */ + combineCallback?: (previousValue: unknown, currentValue: unknown, currentIndex: number, array: unknown[]) => unknown; }; /** @@ -1188,7 +1289,7 @@ type AngularHttpRequestOptions = Omit & { type RetryCall = { method: string; siteUrl: string; - data: unknown; + data: Record; preSets: CoreWSPreSets; deferred: PromiseDefer; };