From d7994eea9ac406123ed04542c7c254c9c88880ca Mon Sep 17 00:00:00 2001 From: Dani Palou Date: Tue, 9 Jan 2018 08:50:08 +0100 Subject: [PATCH] MOBILE-2312 uploader: Implement file uploader provider --- src/classes/site.ts | 5 +- src/core/emulator/providers/file-transfer.ts | 2 +- src/core/fileuploader/lang/en.json | 28 + .../fileuploader/providers/fileuploader.ts | 498 ++++++++++++++++++ src/providers/file.ts | 14 +- src/providers/utils/utils.ts | 27 + 6 files changed, 569 insertions(+), 5 deletions(-) create mode 100644 src/core/fileuploader/lang/en.json create mode 100644 src/core/fileuploader/providers/fileuploader.ts diff --git a/src/classes/site.ts b/src/classes/site.ts index f24809ec9..235fb8241 100644 --- a/src/classes/site.ts +++ b/src/classes/site.ts @@ -717,9 +717,10 @@ export class CoreSite { * * @param {string} filePath File path. * @param {CoreWSFileUploadOptions} options File upload options. + * @param {Function} [onProgress] Function to call on progress. * @return {Promise} Promise resolved when uploaded. */ - uploadFile(filePath: string, options: CoreWSFileUploadOptions) : Promise { + uploadFile(filePath: string, options: CoreWSFileUploadOptions, onProgress?: (event: ProgressEvent) => any) : Promise { if (!options.fileArea) { options.fileArea = 'draft'; } @@ -727,7 +728,7 @@ export class CoreSite { return this.wsProvider.uploadFile(filePath, options, { siteUrl: this.siteUrl, wsToken: this.token - }); + }, onProgress); } /** diff --git a/src/core/emulator/providers/file-transfer.ts b/src/core/emulator/providers/file-transfer.ts index 4ce1da533..e6c0e193e 100644 --- a/src/core/emulator/providers/file-transfer.ts +++ b/src/core/emulator/providers/file-transfer.ts @@ -296,7 +296,7 @@ export class FileTransferObjectMock extends FileTransferObject { } } - (xhr).onprogress = (xhr, ev) => { + xhr.onprogress = (ev: ProgressEvent) : any => { if (this.progressListener) { this.progressListener(ev); } diff --git a/src/core/fileuploader/lang/en.json b/src/core/fileuploader/lang/en.json new file mode 100644 index 000000000..340786594 --- /dev/null +++ b/src/core/fileuploader/lang/en.json @@ -0,0 +1,28 @@ +{ + "addfiletext": "Add file", + "audio": "Audio", + "camera": "Camera", + "confirmuploadfile": "You are about to upload {{size}}. Are you sure you want to continue?", + "confirmuploadunknownsize": "It was not possible to calculate the size of the upload. Are you sure you want to continue?", + "errorcapturingaudio": "Error capturing audio.", + "errorcapturingimage": "Error capturing image.", + "errorcapturingvideo": "Error capturing video.", + "errorgettingimagealbum": "Error getting image from album.", + "errormustbeonlinetoupload": "You have to be online to upload files.", + "errornoapp": "You don't have an app installed to perform this action.", + "errorreadingfile": "Error reading file.", + "errorwhileuploading": "An error occurred during the file upload.", + "file": "File", + "fileuploaded": "The file was successfully uploaded.", + "filesofthesetypes": "Accepted file types:", + "invalidfiletype": "{{$a}} filetype cannot be accepted.", + "maxbytesfile": "The file {{$a.file}} is too large. The maximum size you can upload is {{$a.size}}.", + "more": "More", + "photoalbums": "Photo albums", + "readingfile": "Reading file", + "selectafile": "Select a file", + "uploadafile": "Upload a file", + "uploading": "Uploading", + "uploadingperc": "Uploading: {{$a}}%", + "video": "Video" +} \ No newline at end of file diff --git a/src/core/fileuploader/providers/fileuploader.ts b/src/core/fileuploader/providers/fileuploader.ts new file mode 100644 index 000000000..06c3716ba --- /dev/null +++ b/src/core/fileuploader/providers/fileuploader.ts @@ -0,0 +1,498 @@ +// (C) Copyright 2015 Martin Dougiamas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { Injectable } from '@angular/core'; +import { Platform } from 'ionic-angular'; +import { MediaFile } from '@ionic-native/media-capture'; +import { TranslateService } from '@ngx-translate/core'; +import { CoreFileProvider } from '../../../providers/file'; +import { CoreFilepoolProvider } from '../../../providers/filepool'; +import { CoreLoggerProvider } from '../../../providers/logger'; +import { CoreSitesProvider } from '../../../providers/sites'; +import { CoreMimetypeUtilsProvider } from '../../../providers/utils/mimetype'; +import { CoreTextUtilsProvider } from '../../../providers/utils/text'; +import { CoreTimeUtilsProvider } from '../../../providers/utils/time'; +import { CoreUtilsProvider } from '../../../providers/utils/utils'; +import { CoreWSFileUploadOptions } from '../../../providers/ws'; + +/** + * Interface for file upload options. + */ +export interface CoreFileUploaderOptions extends CoreWSFileUploadOptions { + deleteAfterUpload?: boolean; // Whether the file should be deleted after the upload (if success). +}; + +/** + * Service to upload files. + */ +@Injectable() +export class CoreFileUploaderProvider { + public static LIMITED_SIZE_WARNING = 1048576; // 1 MB. + public static WIFI_SIZE_WARNING = 10485760; // 10 MB. + + protected logger; + + constructor(logger: CoreLoggerProvider, private fileProvider: CoreFileProvider, private textUtils: CoreTextUtilsProvider, + private utils: CoreUtilsProvider, private sitesProvider: CoreSitesProvider, private timeUtils: CoreTimeUtilsProvider, + private mimeUtils: CoreMimetypeUtilsProvider, private filepoolProvider: CoreFilepoolProvider, + private platform: Platform, private translate: TranslateService) { + this.logger = logger.getInstance('CoreFileUploaderProvider'); + } + + /** + * Add a dot to the beginning of an extension. + * + * @param {string} extension Extension. + * @return {string} Treated extension. + */ + protected addDot(extension: string) : string { + return '.' + extension; + } + + /** + * Compares two file lists and returns if they are different. + * + * @param {any[]} a First file list. + * @param {any[]} b Second file list. + * @return {boolean} Whether both lists are different. + */ + areFileListDifferent(a: any[], b: any[]) : boolean { + a = a || []; + b = b || []; + if (a.length != b.length) { + return true; + } + + // Currently we are going to compare the order of the files as well. + // This function can be improved comparing more fields or not comparing the order. + for (let i = 0; i < a.length; i++) { + if ((a[i].name || a[i].filename) != (b[i].name || b[i].filename)) { + return true; + } + } + + return false; + } + + /** + * Clear temporary attachments to be uploaded. + * Attachments already saved in an offline store will NOT be deleted. + * + * @param {any[]} files List of files. + */ + clearTmpFiles(files: any[]) : void { + // Delete the local files. + files.forEach((file) => { + if (!file.offline && file.remove) { + // Pass an empty function to prevent missing parameter error. + file.remove(() => {}); + } + }); + } + + /** + * Get the upload options for a file taken with the Camera Cordova plugin. + * + * @param {string} uri File URI. + * @param {boolean} [isFromAlbum] True if the image was taken from album, false if it's a new image taken with camera. + * @return {CoreFileUploaderOptions} Options. + */ + getCameraUploadOptions(uri: string, isFromAlbum?: boolean) : CoreFileUploaderOptions { + let extension = this.mimeUtils.getExtension(uri), + mimetype = this.mimeUtils.getMimeType(extension), + isIOS = this.platform.is('ios'), + options: CoreFileUploaderOptions = { + deleteAfterUpload: !isFromAlbum, + mimeType: mimetype + }; + + if (isIOS && (mimetype == 'image/jpeg' || mimetype == 'image/png')) { + // In iOS, the pictures can have repeated names, even if they come from the album. + options.fileName = 'image_' + this.timeUtils.readableTimestamp() + '.' + extension; + } else { + // Use the same name that the file already has. + options.fileName = this.fileProvider.getFileAndDirectoryFromPath(uri).name; + } + + if (isFromAlbum) { + // If the file was picked from the album, delete it only if it was copied to the app's folder. + options.deleteAfterUpload = this.fileProvider.isFileInAppFolder(uri); + + if (this.platform.is('android')) { + // Picking an image from album in Android adds a timestamp at the end of the file. Delete it. + options.fileName = options.fileName.replace(/(\.[^\.]*)\?[^\.]*$/, '$1'); + } + } + + return options; + } + + /** + * Get the upload options for a file of any type. + * + * @param {string} uri File URI. + * @param {string} name File name. + * @param {string} type File type. + * @param {boolean} [deleteAfterUpload] Whether the file should be deleted after upload. + * @param {string} [fileArea] File area to upload the file to. It defaults to 'draft'. + * @param {number} [itemId] Draft ID to upload the file to, 0 to create new. + * @return {CoreFileUploaderOptions} Options. + */ + getFileUploadOptions(uri: string, name: string, type: string, deleteAfterUpload?: boolean, fileArea?: string, itemId?: number) + : CoreFileUploaderOptions { + let options : CoreFileUploaderOptions = {}; + options.fileName = name; + options.mimeType = type || this.mimeUtils.getMimeType(this.mimeUtils.getFileExtension(options.fileName)); + options.deleteAfterUpload = !!deleteAfterUpload; + options.itemId = itemId || 0; + options.fileArea = fileArea; + + return options; + } + + /** + * Get the upload options for a file taken with the media capture Cordova plugin. + * + * @param {MediaFile} mediaFile File object to upload. + * @return {CoreFileUploaderOptions} Options. + */ + getMediaUploadOptions(mediaFile: MediaFile) : CoreFileUploaderOptions { + let options : CoreFileUploaderOptions = {}, + filename = mediaFile.name, + split; + + // Add a timestamp to the filename to make it unique. + split = filename.split('.'); + split[0] += '_' + this.timeUtils.readableTimestamp(); + filename = split.join('.'); + + options.fileName = filename; + options.deleteAfterUpload = true; + if (mediaFile.type) { + options.mimeType = mediaFile.type; + } else { + options.mimeType = this.mimeUtils.getMimeType(this.mimeUtils.getFileExtension(options.fileName)); + } + return options; + } + + /** + * Get the files stored in a folder, marking them as offline. + * + * @param {string} folderPath Folder where to get the files. + * @return {Promise} Promise resolved with the list of files. + */ + getStoredFiles(folderPath: string) : Promise { + return this.fileProvider.getDirectoryContents(folderPath).then((files) => { + return this.markOfflineFiles(files); + }); + } + + /** + * Get stored files from combined online and offline file object. + * + * @param {{online: any[], offline: number}} filesObject The combined offline and online files object. + * @param {string} folderPath Folder path to get files from. + * @return {Promise} Promise resolved with files. + */ + getStoredFilesFromOfflineFilesObject(filesObject: {online: any[], offline: number}, folderPath: string) : Promise { + let files = []; + + if (filesObject) { + if (filesObject.online && filesObject.online.length > 0) { + files = this.utils.clone(filesObject.online); + } + + if (filesObject.offline > 0) { + return this.getStoredFiles(folderPath).then((offlineFiles) => { + return files.concat(offlineFiles); + }).catch(() => { + // Ignore not found files. + return files; + }); + } + } + return Promise.resolve(files); + } + + /** + * Check if a file's mimetype is invalid based on the list of accepted mimetypes. This function needs either the file's + * mimetype or the file's path/name. + * + * @param {string[]} [mimetypes] List of supported mimetypes. If undefined, all mimetypes supported. + * @param {string} [path] File's path or name. + * @param {string} [mimetype] File's mimetype. + * @return {string} Undefined if file is valid, error message if file is invalid. + */ + isInvalidMimetype(mimetypes?: string[], path?: string, mimetype?: string) : string { + let extension; + + if (mimetypes) { + // Verify that the mimetype of the file is supported. + if (mimetype) { + extension = this.mimeUtils.getExtension(mimetype); + } else { + extension = this.mimeUtils.getFileExtension(path); + mimetype = this.mimeUtils.getMimeType(extension); + } + + if (mimetype && mimetypes.indexOf(mimetype) == -1) { + extension = extension || this.translate.instant('core.unknown'); + return this.translate.instant('core.fileuploader.invalidfiletype', {$a: extension}); + } + } + } + + /** + * Mark files as offline. + * + * @param {any[]} files Files to mark as offline. + * @return {any[]} Files marked as offline. + */ + markOfflineFiles(files: any[]) : any[] { + // Mark the files as pending offline. + files.forEach((file) => { + file.offline = true; + file.filename = file.name; + }); + return files; + } + + /** + * Parse filetypeList to get the list of allowed mimetypes and the data to render information. + * + * @param {string} filetypeList Formatted string list where the mimetypes can be checked. + * @return {{info: any[], mimetypes: string[]}} Mimetypes and the filetypes informations. + */ + prepareFiletypeList(filetypeList: string) : {info: any[], mimetypes: string[]} { + let filetypes = filetypeList.split(/[;, ]+/g), + mimetypes = {}, // Use an object to prevent duplicates. + typesInfo = []; + + filetypes.forEach((filetype) => { + filetype = filetype.trim(); + + if (filetype) { + if (filetype.indexOf('/') != -1) { + // It's a mimetype. + typesInfo.push({ + name: this.mimeUtils.getMimetypeDescription(filetype), + extlist: this.mimeUtils.getExtensions(filetype).map(this.addDot).join(' ') + }); + + mimetypes[filetype] = true; + } else if (filetype.indexOf('.') === 0) { + // It's an extension. + let mimetype = this.mimeUtils.getMimeType(filetype); + typesInfo.push({ + name: mimetype ? this.mimeUtils.getMimetypeDescription(mimetype) : false, + extlist: filetype + }); + + if (mimetype) { + mimetypes[mimetype] = true; + } + } else { + // It's a group. + let groupExtensions = this.mimeUtils.getGroupMimeInfo(filetype, 'extensions'), + groupMimetypes = this.mimeUtils.getGroupMimeInfo(filetype, 'mimetypes'); + + if (groupExtensions.length > 0) { + typesInfo.push({ + name: this.mimeUtils.getTranslatedGroupName(filetype), + extlist: groupExtensions ? groupExtensions.map(this.addDot).join(' ') : '' + }); + + groupMimetypes.forEach((mimetype) => { + if (mimetype) { + mimetypes[mimetype] = true; + } + }); + } else { + // Treat them as extensions. + filetype = this.addDot(filetype); + let mimetype = this.mimeUtils.getMimeType(filetype); + typesInfo.push({ + name: mimetype ? this.mimeUtils.getMimetypeDescription(mimetype) : false, + extlist: filetype + }); + + if (mimetype) { + mimetypes[mimetype] = true; + } + } + } + } + }); + + return { + info: typesInfo, + mimetypes: Object.keys(mimetypes) + }; + } + + /** + * Given a list of files (either online files or local files), store the local files in a local folder + * to be uploaded later. + * + * @param {string} folderPath Path of the folder where to store the files. + * @param {any[]} files List of files. + * @return {Promise<{online: any[], offline: number}>} Promise resolved if success. + */ + storeFilesToUpload(folderPath: string, files: any[]) : Promise<{online: any[], offline: number}> { + let result = { + online: [], + offline: 0 + }; + + if (!files || !files.length) { + return Promise.resolve(result); + } + + // Remove unused files from previous saves. + return this.fileProvider.removeUnusedFiles(folderPath, files).then(() => { + let promises = []; + + files.forEach((file) => { + if (file.filename && !file.name) { + // It's an online file, add it to the result and ignore it. + result.online.push({ + filename: file.filename, + fileurl: file.fileurl + }); + } else if (!file.name) { + // Error. + promises.push(Promise.reject(null)); + } else if (file.fullPath && file.fullPath.indexOf(folderPath) != -1) { + // File already in the submission folder. + result.offline++; + } else { + // Local file, copy it. Use copy instead of move to prevent having a unstable state if + // some copies succeed and others don't. + let destFile = this.textUtils.concatenatePaths(folderPath, file.name); + promises.push(this.fileProvider.copyFile(file.toURL(), destFile)); + result.offline++; + } + }); + + return Promise.all(promises).then(() => { + return result; + }); + }); + } + + /** + * Upload a file. + * + * @param {string} uri File URI. + * @param {CoreFileUploaderOptions} [options] Options for the upload. + * @param {Function} [onProgress] Function to call on progress. + * @param {string} [siteId] Id of the site to upload the file to. If not defined, use current site. + * @return {Promise} Promise resolved when done. + */ + uploadFile(uri: string, options?: CoreFileUploaderOptions, onProgress?: (event: ProgressEvent) => any, + siteId?: string) : Promise { + options = options || {}; + + const deleteAfterUpload = options.deleteAfterUpload, + ftOptions = this.utils.clone(options); + + delete ftOptions.deleteAfterUpload; + + return this.sitesProvider.getSite(siteId).then((site) => { + return site.uploadFile(uri, ftOptions, onProgress); + }).then((result) => { + if (deleteAfterUpload) { + setTimeout(() => { + // Use set timeout, otherwise in Electron the upload threw an error sometimes. + this.fileProvider.removeExternalFile(uri); + }, 500); + } + return result; + }); + } + + /** + * Upload a file to a draft area. If the file is an online file it will be downloaded and then re-uploaded. + * + * @param {any} file Online file or local FileEntry. + * @param {number} [itemId] Draft ID to use. Undefined or 0 to create a new draft ID. + * @param {string} [component] The component to set to the downloaded files. + * @param {string|number} [componentId] An ID to use in conjunction with the component. + * @param {string} [siteId] Site ID. If not defined, current site. + * @return {Promise} Promise resolved with the itemId. + */ + uploadOrReuploadFile(file: any, itemId?: number, component?: string, componentId?: string|number, + siteId?: string) : Promise { + siteId = siteId || this.sitesProvider.getCurrentSiteId(); + + let promise, + fileName; + + if (file.filename && !file.name) { + // It's an online file. We need to download it and re-upload it. + fileName = file.filename; + promise = this.filepoolProvider.downloadUrl(siteId, file.url || file.fileurl, false, component, componentId, + file.timemodified, undefined, undefined, file).then((path) => { + return this.fileProvider.getExternalFile(path); + }); + } else { + // Local file, we already have the file entry. + fileName = file.name; + promise = Promise.resolve(file); + } + + return promise.then((fileEntry) => { + // Now upload the file. + let options = this.getFileUploadOptions(fileEntry.toURL(), fileName, fileEntry.type, true, 'draft', itemId); + return this.uploadFile(fileEntry.toURL(), options, undefined, siteId).then((result) => { + return result.itemid; + }); + }); + } + + /** + * Given a list of files (either online files or local files), upload them to a draft area and return the draft ID. + * Online files will be downloaded and then re-uploaded. + * If there are no files to upload it will return a fake draft ID (1). + * + * @param {any[]} files List of files. + * @param {string} [component] The component to set to the downloaded files. + * @param {string|number} [componentId] An ID to use in conjunction with the component. + * @param {string} [siteId] Site ID. If not defined, current site. + * @return {Promise} Promise resolved with the itemId. + */ + uploadOrReuploadFiles(files: any[], component?: string, componentId?: string|number, siteId?: string) : Promise { + siteId = siteId || this.sitesProvider.getCurrentSiteId(); + + if (!files || !files.length) { + // Return fake draft ID. + return Promise.resolve(1); + } + + // Upload only the first file first to get a draft id. + return this.uploadOrReuploadFile(files[0], 0, component, componentId, siteId).then((itemId) => { + let promises = []; + + for (let i = 1; i < files.length; i++) { + let file = files[i]; + promises.push(this.uploadOrReuploadFile(file, itemId, component, componentId, siteId)); + } + + return Promise.all(promises).then(() => { + return itemId; + }); + }); + } +} diff --git a/src/providers/file.ts b/src/providers/file.ts index bfe7f9c4c..e8c693f8a 100644 --- a/src/providers/file.ts +++ b/src/providers/file.ts @@ -457,7 +457,7 @@ export class CoreFileProvider { // Create file (and parent folders) to prevent errors. return this.createFile(path).then((fileEntry) => { - if (this.isHTMLAPI && this.appProvider.isDesktop() && + if (this.isHTMLAPI && !this.appProvider.isDesktop() && (typeof data == 'string' || data.toString() == '[object ArrayBuffer]')) { // We need to write Blobs. let type = this.mimeUtils.getMimeType(this.mimeUtils.getFileExtension(path)); @@ -832,7 +832,7 @@ export class CoreFileProvider { * @param {string} [defaultExt] Default extension to use if no extension found in the file. * @return {Promise} Promise resolved with the unique file name. */ - getUniqueNameInFolder(dirPath: string, fileName: string, defaultExt: string) : Promise { + getUniqueNameInFolder(dirPath: string, fileName: string, defaultExt?: string) : Promise { // Get existing files in the folder. return this.getDirectoryContents(dirPath).then((entries) => { let files = {}, @@ -923,4 +923,14 @@ export class CoreFileProvider { // Ignore errors, maybe it doesn't exist. }); } + + /** + * Check if a file is inside the app's folder. + * + * @param {string} path The absolute path of the file to check. + * @return {boolean} Whether the file is in the app's folder. + */ + isFileInAppFolder(path: string) : boolean { + return path.indexOf(this.basePath) != -1; + } } diff --git a/src/providers/utils/utils.ts b/src/providers/utils/utils.ts index 9f077a25a..d75334f2d 100644 --- a/src/providers/utils/utils.ts +++ b/src/providers/utils/utils.ts @@ -259,6 +259,33 @@ export class CoreUtilsProvider { } } + /** + * Clone a variable. It should be an object, array or primitive type. + * + * @param {any} source The variable to clone. + * @return {any} Cloned variable. + */ + clone(source: any) : any { + if (Array.isArray(source)) { + // Clone the array and all the entries. + let newArray = []; + for (let i = 0; i < source.length; i++) { + newArray[i] = this.clone(source[i]); + } + return newArray; + } else if (typeof source == 'object') { + // Clone the object and all the subproperties. + let newObject = {}; + for (let name in source) { + newObject[name] = this.clone(source[name]); + } + return newObject; + } else { + // Primitive type or unknown, return it as it is. + return source; + } + } + /** * Copy properties from one object to another. *