MOBILE-3875 ci: Configure performance tests

.github/workflows/performance.yml

@@ -0,0 +1,55 @@
+name: Performance
+
+on: [push, pull_request]
+
+jobs:
+  performance:
+    runs-on: ubuntu-latest
+    env:
+      MOODLE_DOCKER_DB: pgsql
+      MOODLE_DOCKER_BROWSER: chrome
+      MOODLE_DOCKER_PHP_VERSION: 7.3
+    steps:
+    - uses: actions/checkout@v2
+    - id: nvmrc
+      uses: browniebroke/read-nvmrc-action@v1
+    - uses: actions/setup-node@v1
+      with:
+        node-version: '${{ steps.nvmrc.outputs.node_version }}'
+    - name: Additional checkouts
+      run: |
+        git clone --branch master --depth 1 git://github.com/moodle/moodle $GITHUB_WORKSPACE/moodle
+        git clone --branch master --depth 1 git://github.com/moodlehq/moodle-local_moodlemobileapp $GITHUB_WORKSPACE/moodle/local/moodlemobileapp
+        git clone --branch master --depth 1 git://github.com/moodlehq/moodle-docker $GITHUB_WORKSPACE/moodle-docker
+    - name: Install npm packages
+      run: npm ci
+    - name: Generate Behat tests plugin
+      run: |
+        export MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle
+        npx gulp behat
+    - name: Configure & launch Moodle with Docker
+      run: |
+        export MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle
+        cp $GITHUB_WORKSPACE/moodle-docker/config.docker-template.php $GITHUB_WORKSPACE/moodle/config.php
+        sed -i "70i\$CFG->behat_ionic_wwwroot = \"http://moodleapp\";" $GITHUB_WORKSPACE/moodle/config.php
+        $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-compose pull
+        $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-compose up -d
+        $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-wait-for-db
+    - name: Compile & launch production app with Docker
+      run: |
+        docker build -t moodlehq/moodleapp:performance .
+        docker run -d --rm --name moodleapp moodlehq/moodleapp:performance
+        docker network connect moodle-docker_default moodleapp --alias moodleapp
+    - name: Init Behat
+      run: |
+        export MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle
+        $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-compose exec -T webserver sh -c "php admin/tool/behat/cli/init.php"
+    - name: Run performance tests
+      run: |
+        export MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle
+        for i in {0..3}
+        do
+          $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-compose exec -T webserver sh -c "php admin/tool/behat/cli/run.php --tags="@performance" --auto-rerun"
+        done
+    - name: Show performance results
+      run: node ./scripts/print-measure-timings.js $GITHUB_WORKSPACE/moodle/behatmeasuretimings/

scripts/print-measure-timings.js

@@ -0,0 +1,40 @@
+#!/usr/bin/env node
+
+// (C) Copyright 2015 Moodle Pty Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+const { readdirSync, readFileSync } = require('fs');
+
+if (process.argv.length < 3) {
+    console.error('Missing measure timings storage path argument');
+    process.exit(1);
+}
+
+const measureTimingsStoragePath = process.argv[2].trimRight('/') + '/';
+const files = readdirSync(measureTimingsStoragePath);
+const measureTimingsDurations = {};
+
+for (const file of files) {
+    const measureTiming = JSON.parse(readFileSync(measureTimingsStoragePath + file));
+
+    measureTimingsDurations[measureTiming.measure] = measureTimingsDurations[measureTiming.measure] ?? [];
+    measureTimingsDurations[measureTiming.measure].push(measureTiming.duration);
+}
+
+for (const [measure, durations] of Object.entries(measureTimingsDurations)) {
+    const totalRuns = durations.length;
+    const averageDuration = Math.round(durations.reduce((total, duration) => total + duration) / totalRuns);
+
+    console.log(`${measure} took an average of ${averageDuration}ms per run (in ${totalRuns} runs)`);
+}

tests/behat/behat_performance.php

@@ -0,0 +1,128 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+use Behat\Mink\Exception\DriverException;
+use Behat\Mink\Exception\ExpectationException;
+
+require_once(__DIR__ . '/../../../../lib/behat/behat_base.php');
+require_once(__DIR__ . '/classes/measure_timing.php');
+
+/**
+ * Behat step definitions to measure performance.
+ */
+class behat_performance extends behat_base {
+
+    /**
+     * @var array
+     */
+    private $timings = [];
+
+    /**
+     * Start timing a performance measure.
+     *
+     * @When /^I start timing "([^"]+)"$/
+     */
+    public function i_start_timing(string $measure) {
+        $this->timings[$measure] = new measure_timing($measure);
+        $this->timings[$measure]->start();
+    }
+
+    /**
+     * Stop timing a performance measure.
+     *
+     * @When /^I stop timing "([^"]+)"$/
+     */
+    public function i_stop_timing(string $measure) {
+        $this->get_measure_timing($measure)->end();
+    }
+
+    /**
+     * Assert how long a performance measure took.
+     *
+     * @Then /^"([^"]+)" should have taken (less than|more than|exactly) (\d+(?:\.\d+)? (?:seconds|milliseconds))$/
+     */
+    public function timing_should_have_taken(string $measure, Closure $comparison, float $expectedtime) {
+        $measuretiming = $this->get_measure_timing($measure);
+
+        if (!call_user_func($comparison, $measuretiming->duration, $expectedtime)) {
+            throw new ExpectationException(
+                "Expected timing for '$measure' measure failed! (took {$measuretiming->duration}ms)",
+                $this->getSession()->getDriver()
+            );
+        }
+
+        $measuretiming->store();
+    }
+
+    /**
+     * Parse time.
+     *
+     * @Transform /^\d+(?:\.\d+)? (?:seconds|milliseconds)$/
+     * @param string $text Time string.
+     * @return float
+     */
+    public function parse_time(string $text): float {
+        $spaceindex = strpos($text, ' ');
+        $value = floatval(substr($text, 0, $spaceindex));
+
+        switch (substr($text, $spaceindex + 1)) {
+            case 'seconds':
+                $value *= 1000;
+                break;
+        }
+
+        return $value;
+    }
+
+    /**
+     * Parse a comparison function.
+     *
+     * @Transform /^less than|more than|exactly$/
+     * @param string $text Comparison string.
+     * @return Closure
+     */
+    public function parse_comparison(string $text): Closure {
+        switch ($text) {
+            case 'less than':
+                return function ($a, $b) {
+                    return $a < $b;
+                };
+            case 'more than':
+                return function ($a, $b) {
+                    return $a > $b;
+                };
+            case 'exactly':
+                return function ($a, $b) {
+                    return $a === $b;
+                };
+        }
+    }
+
+    /**
+     * Get measure timing by name.
+     *
+     * @param string $measure Measure timing name.
+     * @return measure_timing Measure timing.
+     */
+    private function get_measure_timing(string $measure): measure_timing {
+        if (!isset($this->timings[$measure])) {
+            throw new DriverException("Timing for '$measure' measure does not exist.");
+        }
+
+        return $this->timings[$measure];
+    }
+
+}

tests/behat/classes/measure_timing.php

@@ -0,0 +1,94 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Performance timing for one particular measure.
+ */
+class measure_timing {
+
+    const STORAGE_FOLDER = '/behatmeasuretimings/';
+
+    /**
+     * @var string
+     */
+    public $measure;
+
+    /**
+     * @var int
+     */
+    public $start;
+
+    /**
+     * @var int
+     */
+    public $end;
+
+    /**
+     * @var int
+     */
+    public $duration;
+
+    public function __construct(string $measure) {
+        $this->measure = $measure;
+    }
+
+    /**
+     * Start timing.
+     */
+    public function start(): void {
+        $this->start = $this->now();
+    }
+
+    /**
+     * Stop timing.
+     */
+    public function end(): void {
+        $this->end = $this->now();
+        $this->duration = $this->end - $this->start;
+    }
+
+    /**
+     * Persist measure timing in storage.
+     */
+    public function store(): void {
+        global $CFG;
+
+        $storagefolderpath = $CFG->dirroot . static::STORAGE_FOLDER;
+
+        if (!file_exists($storagefolderpath)) {
+            mkdir($storagefolderpath);
+        }
+
+        $data = [
+            'measure' => $this->measure,
+            'start' => $this->start,
+            'end' => $this->end,
+            'duration' => $this->duration,
+        ];
+
+        file_put_contents($storagefolderpath . time() . '.json', json_encode($data));
+    }
+
+    /**
+     * Get current time.
+     *
+     * @return int Current time in milliseconds.
+     */
+    private function now(): int {
+        return round(microtime(true) * 1000);
+    }
+
+}

tests/behat/performance.feature

@@ -0,0 +1,32 @@
+@app @javascript @performance
+Feature: Measure performance.
+
+Scenario: [FCP] First Contentful Paint
+    Given I start timing "FCP"
+    When I launch the app runtime
+    Then I should find "Welcome to the Moodle App!" in the app
+
+    When I stop timing "FCP"
+    Then "FCP" should have taken less than 5 seconds
+
+Scenario: [TTI] Time to Interactive
+    Given I start timing "TTI"
+    When I launch the app runtime
+    Then I should find "Welcome to the Moodle App!" in the app
+
+    When I press "Skip" in the app
+    Then I should find "Connect to Moodle" in the app
+
+    When I stop timing "TTI"
+    Then "TTI" should have taken less than 6 seconds
+
+Scenario: [TBT] Total Blocking Time
+    Given I launch the app runtime
+    Then I should find "Welcome to the Moodle App!" in the app
+
+    When I start timing "TBT"
+    And I press "Skip" in the app
+    Then I should find "Connect to Moodle" in the app
+
+    When I stop timing "TBT"
+    Then "TBT" should have taken less than 2 seconds