MOBILE-3565 scripts: Add WS helper scripts

scripts/get_all_ws_structures.php

@@ -0,0 +1,48 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Script for getting the PHP structure of a WS returns or params.
+ *
+ * The first parameter (required) is the path to the Moodle installation to use.
+ * The second parameter (required) is the name to the WS to convert.
+ * The third parameter (optional) is a number: 1 to convert the params structure,
+ * 0 to convert the returns structure. Defaults to 0.
+ */
+
+if (!isset($argv[1])) {
+    echo "ERROR: Please pass the Moodle path as the first parameter.\n";
+    die();
+}
+
+$moodlepath = $argv[1];
+
+define('CLI_SCRIPT', true);
+
+require($moodlepath . '/config.php');
+require($CFG->dirroot . '/webservice/lib.php');
+require_once('ws_to_ts_functions.php');
+
+$structures = get_all_ws_structures();
+
+foreach ($structures as $wsname => $structure) {
+
+    remove_default_closures($structure->parameters_desc);
+    print_ws_structure($wsname, $structure->parameters_desc, true);
+
+    remove_default_closures($structure->returns_desc);
+    print_ws_structure($wsname, $structure->returns_desc, false);
+}

scripts/get_ws_changes.php

@@ -0,0 +1,105 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Script for detecting changes in a WS params or return data, version by version.
+ *
+ * The first parameter (required) is the path to the Moodle installation to use.
+ * The second parameter (required) is the name to the WS to convert.
+ * The third parameter (optional) is a number: 1 to convert the params structure,
+ * 0 to convert the returns structure. Defaults to 0.
+ */
+
+if (!isset($argv[1])) {
+    echo "ERROR: Please pass the path to the folder containing the Moodle installations as the first parameter.\n";
+    die();
+}
+
+if (!isset($argv[2])) {
+    echo "ERROR: Please pass the WS name as the second parameter.\n";
+    die();
+}
+
+define('CLI_SCRIPT', true);
+define('CACHE_DISABLE_ALL', true);
+define('SERIALIZED', true);
+require_once('ws_to_ts_functions.php');
+
+$versions = array('master', '310', '39', '38', '37', '36', '35', '34', '33', '32', '31');
+
+$moodlespath = $argv[1];
+$wsname = $argv[2];
+$useparams = !!(isset($argv[3]) && $argv[3]);
+$pathseparator = '/';
+
+// Get the path to the script.
+$index = strrpos(__FILE__, $pathseparator);
+if ($index === false) {
+    $pathseparator = '\\';
+    $index = strrpos(__FILE__, $pathseparator);
+}
+$scriptfolder = substr(__FILE__, 0, $index);
+$scriptpath = concatenate_paths($scriptfolder, 'get_ws_structure.php', $pathseparator);
+
+$previousstructure = null;
+$previousversion = null;
+$libsloaded = false;
+
+foreach ($versions as $version) {
+    $moodlepath = concatenate_paths($moodlespath, 'stable_' . $version . '/moodle', $pathseparator);
+
+    if (!file_exists($moodlepath)) {
+        echo "Folder does not exist for version $version, skipping...\n";
+        continue;
+    }
+
+    if (!$libsloaded) {
+        $libsloaded = true;
+
+        require($moodlepath . '/config.php');
+        require($CFG->dirroot . '/webservice/lib.php');
+    }
+
+    // Get the structure in this Moodle version.
+    $structure = shell_exec("php $scriptpath $moodlepath $wsname " . ($useparams ? 'true' : ''));
+
+    if (strpos($structure, 'ERROR:') === 0) {
+        echo "WS not found in version $version. Stop.\n";
+        break;
+    }
+
+    $structure = unserialize($structure);
+
+    if ($previousstructure != null) {
+        echo "*** Check changes from version $version to $previousversion ***\n";
+
+        $messages = detect_ws_changes($previousstructure, $structure);
+
+        if (count($messages) > 0) {
+            $haschanged = true;
+
+            foreach($messages as $message) {
+                echo "$message\n";
+            }
+        } else {
+            echo "No changes found.\n";
+        }
+        echo "\n";
+    }
+
+    $previousstructure = $structure;
+    $previousversion = $version;
+}

scripts/get_ws_structure.php

@@ -0,0 +1,63 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Script for getting the PHP structure of a WS returns or params.
+ *
+ * The first parameter (required) is the path to the Moodle installation to use.
+ * The second parameter (required) is the name to the WS to convert.
+ * The third parameter (optional) is a number: 1 to convert the params structure,
+ * 0 to convert the returns structure. Defaults to 0.
+ */
+
+if (!isset($argv[1])) {
+    echo "ERROR: Please pass the Moodle path as the first parameter.\n";
+    die();
+}
+
+if (!isset($argv[2])) {
+    echo "ERROR: Please pass the WS name as the second parameter.\n";
+    die();
+}
+
+if (!defined('SERIALIZED')) {
+    define('SERIALIZED', false);
+}
+
+$moodlepath = $argv[1];
+$wsname = $argv[2];
+$useparams = !!(isset($argv[3]) && $argv[3]);
+
+define('CLI_SCRIPT', true);
+
+require($moodlepath . '/config.php');
+require($CFG->dirroot . '/webservice/lib.php');
+require_once('ws_to_ts_functions.php');
+
+$structure = get_ws_structure($wsname, $useparams);
+
+if ($structure === false) {
+    echo "ERROR: The WS wasn't found in this Moodle installation.\n";
+    die();
+}
+
+remove_default_closures($structure);
+
+if (SERIALIZED) {
+    echo serialize($structure);
+} else {
+    print_ws_structure($wsname, $structure, $useparams);
+}

scripts/ws_to_ts_functions.php

@@ -0,0 +1,288 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Helper functions for converting a Moodle WS structure to a TS type.
+ */
+
+/**
+ * Get the structure of a WS params or returns.
+ */
+function get_ws_structure($wsname, $useparams) {
+    global $DB;
+
+    // get all the function descriptions
+    $function = $DB->get_record('external_functions', array('services' => 'moodle_mobile_app', 'name' => $wsname));
+    if (!$function) {
+        return false;
+    }
+
+    $functiondesc = external_api::external_function_info($function);
+
+    if ($useparams) {
+        return $functiondesc->parameters_desc;
+    } else {
+        return $functiondesc->returns_desc;
+    }
+}
+
+/**
+ * Return all WS structures.
+ */
+function get_all_ws_structures() {
+    global $DB;
+
+    // get all the function descriptions
+    $functions = $DB->get_records('external_functions', array('services' => 'moodle_mobile_app'), 'name');
+    $functiondescs = array();
+    foreach ($functions as $function) {
+        $functiondescs[$function->name] = external_api::external_function_info($function);
+    }
+
+    return $functiondescs;
+}
+
+/**
+ * Fix a comment: make sure first letter is uppercase and add a dot at the end if needed.
+ */
+function fix_comment($desc) {
+    $desc = trim($desc);
+    $desc = ucfirst($desc);
+
+    if (substr($desc, -1) !== '.') {
+        $desc .= '.';
+    }
+
+    $lines = explode("\n", $desc);
+    if (count($lines) > 1) {
+        $desc = array_shift($lines)."\n";
+
+        foreach ($lines as $i => $line) {
+            $spaces = strlen($line) - strlen(ltrim($line));
+            $desc .= str_repeat(' ', $spaces - 3) . '// '. ltrim($line)."\n";
+        }
+    }
+
+    return $desc;
+}
+
+/**
+ * Get an inline comment based on a certain text.
+ */
+function get_inline_comment($desc) {
+    if (empty($desc)) {
+        return '';
+    }
+
+    return ' // ' . fix_comment($desc);
+}
+
+/**
+ * Add the TS documentation of a certain element.
+ */
+function get_ts_doc($type, $desc, $indentation) {
+    if (empty($desc)) {
+        // If no key, it's probably in an array. We only document object properties.
+        return '';
+    }
+
+    return $indentation . "/**\n" .
+           $indentation . " * " . fix_comment($desc) . "\n" .
+           (!empty($type) ? ($indentation . " * @type {" . $type . "}\n") : '') .
+           $indentation . " */\n";
+}
+
+/**
+ * Specify a certain type, with or without a key.
+ */
+function convert_key_type($key, $type, $required, $indentation) {
+    if ($key) {
+        // It has a key, it's inside an object.
+        return $indentation . "$key" . ($required == VALUE_OPTIONAL || $required == VALUE_DEFAULT ? '?' : '') . ": $type";
+    } else {
+        // No key, it's probably in an array. Just include the type.
+        return $type;
+    }
+}
+
+/**
+ * Convert a certain element into a TS structure.
+ */
+function convert_to_ts($key, $value, $boolisnumber = false, $indentation = '', $arraydesc = '') {
+    if ($value instanceof external_value || $value instanceof external_warnings || $value instanceof external_files) {
+        // It's a basic field or a pre-defined type like warnings.
+        $type = 'string';
+
+        if ($value instanceof external_warnings) {
+            $type = 'CoreWSExternalWarning[]';
+        } else if ($value instanceof external_files) {
+            $type = 'CoreWSExternalFile[]';
+        } else if ($value->type == PARAM_BOOL && !$boolisnumber) {
+            $type = 'boolean';
+        } else if (($value->type == PARAM_BOOL && $boolisnumber) || $value->type == PARAM_INT || $value->type == PARAM_FLOAT ||
+                $value->type == PARAM_LOCALISEDFLOAT || $value->type == PARAM_PERMISSION || $value->type == PARAM_INTEGER ||
+                $value->type == PARAM_NUMBER) {
+            $type = 'number';
+        }
+
+        $result = convert_key_type($key, $type, $value->required, $indentation);
+
+        return $result;
+
+    } else if ($value instanceof external_single_structure) {
+        // It's an object.
+        $result = convert_key_type($key, '{', $value->required, $indentation);
+
+        if ($arraydesc) {
+            // It's an array of objects. Print the array description now.
+            $result .= get_inline_comment($arraydesc);
+        }
+
+        $result .= "\n";
+
+        foreach ($value->keys as $key => $value) {
+            $result .= convert_to_ts($key, $value, $boolisnumber, $indentation . '    ') . ';';
+
+            if (!$value instanceof external_multiple_structure || !$value->content instanceof external_single_structure) {
+                // Add inline comments after the field, except for arrays of objects where it's added at the start.
+                $result .= get_inline_comment($value->desc);
+            }
+
+            $result .= "\n";
+        }
+
+        $result .= "$indentation}";
+
+        return $result;
+
+    } else if ($value instanceof external_multiple_structure) {
+        // It's an array.
+        $result = convert_key_type($key, '', $value->required, $indentation);
+
+        $result .= convert_to_ts(null, $value->content, $boolisnumber, $indentation, $value->desc);
+
+        $result .= "[]";
+
+        return $result;
+    } else if ($value == null) {
+        return "{}; // WARNING: Null structure found";
+    } else {
+        return "{}; // WARNING: Unknown structure: $key " . get_class($value);
+    }
+}
+
+/**
+ * Print structure ready to use.
+ */
+function print_ws_structure($name, $structure, $useparams) {
+    if ($useparams) {
+        $type = implode('', array_map('ucfirst', explode('_', $name))) . 'WSParams';
+        $comment = "Params of $name WS.";
+    } else {
+        $type = implode('', array_map('ucfirst', explode('_', $name))) . 'WSResponse';
+        $comment = "Data returned by $name WS.";
+    }
+
+    echo "
+/**
+ * $comment
+ */
+export type $type = ".convert_to_ts(null, $structure).";\n";
+}
+
+/**
+ * Concatenate two paths.
+ */
+function concatenate_paths($left, $right, $separator = '/') {
+    if (!is_string($left) || $left == '') {
+        return $right;
+    } else if (!is_string($right) || $right == '') {
+        return $left;
+    }
+
+    $lastCharLeft = substr($left, -1);
+    $firstCharRight = $right[0];
+
+    if ($lastCharLeft === $separator && $firstCharRight === $separator) {
+        return $left . substr($right, 1);
+    } else if ($lastCharLeft !== $separator && $firstCharRight !== '/') {
+        return $left . '/' . $right;
+    } else {
+        return $left . $right;
+    }
+}
+
+/**
+ * Detect changes between 2 WS structures. We only detect fields that have been added or modified, not removed fields.
+ */
+function detect_ws_changes($new, $old, $key = '', $path = '') {
+    $messages = [];
+
+    if (gettype($new) != gettype($old)) {
+        // The type has changed.
+        $messages[] = "Property '$key' has changed type, from '" . gettype($old) . "' to '" . gettype($new) .
+                        ($path != '' ? "' inside $path." : "'.");
+
+    } else if ($new instanceof external_value && $new->type != $old->type) {
+        // The type has changed.
+        $messages[] = "Property '$key' has changed type, from '" . $old->type . "' to '" . $new->type .
+                        ($path != '' ? "' inside $path." : "'.");
+
+    } else if ($new instanceof external_warnings || $new instanceof external_files) {
+        // Ignore these types.
+
+    } else if ($new instanceof external_single_structure) {
+        // Check each subproperty.
+        $newpath = ($path != '' ? "$path." : '') . $key;
+
+        foreach ($new->keys as $subkey => $value) {
+            if (!isset($old->keys[$subkey])) {
+                // New property.
+                $messages[] = "New property '$subkey' found" . ($newpath != '' ? " inside '$newpath'." : '.');
+            } else {
+                $messages = array_merge($messages, detect_ws_changes($value, $old->keys[$subkey], $subkey, $newpath));
+            }
+        }
+    } else if ($new instanceof external_multiple_structure) {
+        // Recursive call with the content.
+        $messages = array_merge($messages, detect_ws_changes($new->content, $old->content, $key, $path));
+    }
+
+    return $messages;
+}
+
+/**
+ * Remove all closures (anonymous functions) in the default values so the object can be serialized.
+ */
+function remove_default_closures($value) {
+    if ($value instanceof external_warnings || $value instanceof external_files) {
+        // Ignore these types.
+
+    } else if ($value instanceof external_value) {
+        if ($value->default instanceof Closure) {
+            $value->default = null;
+        }
+
+    } else if ($value instanceof external_single_structure) {
+
+        foreach ($value->keys as $key => $subvalue) {
+            remove_default_closures($subvalue);
+        }
+
+    } else if ($value instanceof external_multiple_structure) {
+        remove_default_closures($value->content);
+    }
+}

src/app/services/sync.db.ts

@@ -18,7 +18,7 @@ import { CoreSiteSchema, registerSiteSchema } from '@services/sites';
  * Database variables for CoreSync service.
  */
 export const SYNC_TABLE_NAME = 'sync';
-export const SITE_SCHEMA: CoreSiteSchema = {
+const SITE_SCHEMA: CoreSiteSchema = {
     name: 'CoreSyncProvider',
     version: 1,
     tables: [

src/app/services/sync.ts

@@ -14,7 +14,7 @@
 
 import { Injectable } from '@angular/core';
 import { CoreEvents } from '@singletons/events';
-import { CoreSites, CoreSiteSchema } from '@services/sites';
+import { CoreSites } from '@services/sites';
 import { makeSingleton } from '@singletons/core.singletons';
 import { SYNC_TABLE_NAME, CoreSyncRecord } from '@services/sync.db';
 
@@ -24,38 +24,6 @@ import { SYNC_TABLE_NAME, CoreSyncRecord } from '@services/sync.db';
 @Injectable()
 export class CoreSyncProvider {
 
-    // Variables for the database.
-    protected siteSchema: CoreSiteSchema = {
-        name: 'CoreSyncProvider',
-        version: 1,
-        tables: [
-            {
-                name: SYNC_TABLE_NAME,
-                columns: [
-                    {
-                        name: 'component',
-                        type: 'TEXT',
-                        notNull: true,
-                    },
-                    {
-                        name: 'id',
-                        type: 'TEXT',
-                        notNull: true,
-                    },
-                    {
-                        name: 'time',
-                        type: 'INTEGER',
-                    },
-                    {
-                        name: 'warnings',
-                        type: 'TEXT',
-                    },
-                ],
-                primaryKeys: ['component', 'id'],
-            },
-        ],
-    };
-
     // Store blocked sync objects.
     protected blockedItems: { [siteId: string]: { [blockId: string]: { [operation: string]: boolean } } } = {};
 
@@ -129,8 +97,10 @@ export class CoreSyncProvider {
      * @param siteId Site ID. If not defined, current site.
      * @return Record if found or reject.
      */
-    getSyncRecord(component: string, id: string | number, siteId?: string): Promise<CoreSyncRecord> {
+    async getSyncRecord(component: string, id: string | number, siteId?: string): Promise<CoreSyncRecord> {
-        return CoreSites.instance.getSiteDb(siteId).then((db) => db.getRecord(SYNC_TABLE_NAME, { component: component, id: id }));
+        const db = await CoreSites.instance.getSiteDb(siteId);
+
+        return await db.getRecord(SYNC_TABLE_NAME, { component: component, id: id });
     }
 
     /**

src/app/services/ws.ts

@@ -981,6 +981,15 @@ export type CoreWSExternalWarning = {
 
 };
 
+/**
+ * Special response structure of many webservices that contains success status and warnings.
+ */
+export type CoreStatusWithWarningsWSResponse = {
+    status: boolean; // Status: true if success.
+    offline?: boolean; // True if information has been stored in offline for future use.
+    warnings?: CoreWSExternalWarning[];
+};
+
 /**
  * Structure of files returned by WS.
  */