From 1688fc0d0b4b653330506d023515857f1d32d6c9 Mon Sep 17 00:00:00 2001 From: Noel De Martin Date: Thu, 23 Sep 2021 12:39:35 +0200 Subject: [PATCH] MOBILE-3875 ci: Configure performance tests --- .github/workflows/performance.yml | 55 +++++++++++ scripts/print-measure-timings.js | 40 ++++++++ tests/behat/behat_performance.php | 128 +++++++++++++++++++++++++ tests/behat/classes/measure_timing.php | 94 ++++++++++++++++++ tests/behat/performance.feature | 32 +++++++ 5 files changed, 349 insertions(+) create mode 100644 .github/workflows/performance.yml create mode 100755 scripts/print-measure-timings.js create mode 100644 tests/behat/behat_performance.php create mode 100644 tests/behat/classes/measure_timing.php create mode 100644 tests/behat/performance.feature diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml new file mode 100644 index 000000000..6b894c8d8 --- /dev/null +++ b/.github/workflows/performance.yml @@ -0,0 +1,55 @@ +name: Performance + +on: [push, pull_request] + +jobs: + performance: + runs-on: ubuntu-latest + env: + MOODLE_DOCKER_DB: pgsql + MOODLE_DOCKER_BROWSER: chrome + MOODLE_DOCKER_PHP_VERSION: 7.3 + steps: + - uses: actions/checkout@v2 + - id: nvmrc + uses: browniebroke/read-nvmrc-action@v1 + - uses: actions/setup-node@v1 + with: + node-version: '${{ steps.nvmrc.outputs.node_version }}' + - name: Additional checkouts + run: | + git clone --branch master --depth 1 git://github.com/moodle/moodle $GITHUB_WORKSPACE/moodle + git clone --branch master --depth 1 git://github.com/moodlehq/moodle-local_moodlemobileapp $GITHUB_WORKSPACE/moodle/local/moodlemobileapp + git clone --branch master --depth 1 git://github.com/moodlehq/moodle-docker $GITHUB_WORKSPACE/moodle-docker + - name: Install npm packages + run: npm ci + - name: Generate Behat tests plugin + run: | + export MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle + npx gulp behat + - name: Configure & launch Moodle with Docker + run: | + export MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle + cp $GITHUB_WORKSPACE/moodle-docker/config.docker-template.php $GITHUB_WORKSPACE/moodle/config.php + sed -i "70i\$CFG->behat_ionic_wwwroot = \"http://moodleapp\";" $GITHUB_WORKSPACE/moodle/config.php + $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-compose pull + $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-compose up -d + $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-wait-for-db + - name: Compile & launch production app with Docker + run: | + docker build -t moodlehq/moodleapp:performance . + docker run -d --rm --name moodleapp moodlehq/moodleapp:performance + docker network connect moodle-docker_default moodleapp --alias moodleapp + - name: Init Behat + run: | + export MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle + $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-compose exec -T webserver sh -c "php admin/tool/behat/cli/init.php" + - name: Run performance tests + run: | + export MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle + for i in {0..3} + do + $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-compose exec -T webserver sh -c "php admin/tool/behat/cli/run.php --tags="@performance" --auto-rerun" + done + - name: Show performance results + run: node ./scripts/print-measure-timings.js $GITHUB_WORKSPACE/moodle/behatmeasuretimings/ diff --git a/scripts/print-measure-timings.js b/scripts/print-measure-timings.js new file mode 100755 index 000000000..ccd9356e1 --- /dev/null +++ b/scripts/print-measure-timings.js @@ -0,0 +1,40 @@ +#!/usr/bin/env node + +// (C) Copyright 2015 Moodle Pty Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const { readdirSync, readFileSync } = require('fs'); + +if (process.argv.length < 3) { + console.error('Missing measure timings storage path argument'); + process.exit(1); +} + +const measureTimingsStoragePath = process.argv[2].trimRight('/') + '/'; +const files = readdirSync(measureTimingsStoragePath); +const measureTimingsDurations = {}; + +for (const file of files) { + const measureTiming = JSON.parse(readFileSync(measureTimingsStoragePath + file)); + + measureTimingsDurations[measureTiming.measure] = measureTimingsDurations[measureTiming.measure] ?? []; + measureTimingsDurations[measureTiming.measure].push(measureTiming.duration); +} + +for (const [measure, durations] of Object.entries(measureTimingsDurations)) { + const totalRuns = durations.length; + const averageDuration = Math.round(durations.reduce((total, duration) => total + duration) / totalRuns); + + console.log(`${measure} took an average of ${averageDuration}ms per run (in ${totalRuns} runs)`); +} diff --git a/tests/behat/behat_performance.php b/tests/behat/behat_performance.php new file mode 100644 index 000000000..1097c982e --- /dev/null +++ b/tests/behat/behat_performance.php @@ -0,0 +1,128 @@ +. + +use Behat\Mink\Exception\DriverException; +use Behat\Mink\Exception\ExpectationException; + +require_once(__DIR__ . '/../../../../lib/behat/behat_base.php'); +require_once(__DIR__ . '/classes/measure_timing.php'); + +/** + * Behat step definitions to measure performance. + */ +class behat_performance extends behat_base { + + /** + * @var array + */ + private $timings = []; + + /** + * Start timing a performance measure. + * + * @When /^I start timing "([^"]+)"$/ + */ + public function i_start_timing(string $measure) { + $this->timings[$measure] = new measure_timing($measure); + $this->timings[$measure]->start(); + } + + /** + * Stop timing a performance measure. + * + * @When /^I stop timing "([^"]+)"$/ + */ + public function i_stop_timing(string $measure) { + $this->get_measure_timing($measure)->end(); + } + + /** + * Assert how long a performance measure took. + * + * @Then /^"([^"]+)" should have taken (less than|more than|exactly) (\d+(?:\.\d+)? (?:seconds|milliseconds))$/ + */ + public function timing_should_have_taken(string $measure, Closure $comparison, float $expectedtime) { + $measuretiming = $this->get_measure_timing($measure); + + if (!call_user_func($comparison, $measuretiming->duration, $expectedtime)) { + throw new ExpectationException( + "Expected timing for '$measure' measure failed! (took {$measuretiming->duration}ms)", + $this->getSession()->getDriver() + ); + } + + $measuretiming->store(); + } + + /** + * Parse time. + * + * @Transform /^\d+(?:\.\d+)? (?:seconds|milliseconds)$/ + * @param string $text Time string. + * @return float + */ + public function parse_time(string $text): float { + $spaceindex = strpos($text, ' '); + $value = floatval(substr($text, 0, $spaceindex)); + + switch (substr($text, $spaceindex + 1)) { + case 'seconds': + $value *= 1000; + break; + } + + return $value; + } + + /** + * Parse a comparison function. + * + * @Transform /^less than|more than|exactly$/ + * @param string $text Comparison string. + * @return Closure + */ + public function parse_comparison(string $text): Closure { + switch ($text) { + case 'less than': + return function ($a, $b) { + return $a < $b; + }; + case 'more than': + return function ($a, $b) { + return $a > $b; + }; + case 'exactly': + return function ($a, $b) { + return $a === $b; + }; + } + } + + /** + * Get measure timing by name. + * + * @param string $measure Measure timing name. + * @return measure_timing Measure timing. + */ + private function get_measure_timing(string $measure): measure_timing { + if (!isset($this->timings[$measure])) { + throw new DriverException("Timing for '$measure' measure does not exist."); + } + + return $this->timings[$measure]; + } + +} diff --git a/tests/behat/classes/measure_timing.php b/tests/behat/classes/measure_timing.php new file mode 100644 index 000000000..4baa699f9 --- /dev/null +++ b/tests/behat/classes/measure_timing.php @@ -0,0 +1,94 @@ +. + +/** + * Performance timing for one particular measure. + */ +class measure_timing { + + const STORAGE_FOLDER = '/behatmeasuretimings/'; + + /** + * @var string + */ + public $measure; + + /** + * @var int + */ + public $start; + + /** + * @var int + */ + public $end; + + /** + * @var int + */ + public $duration; + + public function __construct(string $measure) { + $this->measure = $measure; + } + + /** + * Start timing. + */ + public function start(): void { + $this->start = $this->now(); + } + + /** + * Stop timing. + */ + public function end(): void { + $this->end = $this->now(); + $this->duration = $this->end - $this->start; + } + + /** + * Persist measure timing in storage. + */ + public function store(): void { + global $CFG; + + $storagefolderpath = $CFG->dirroot . static::STORAGE_FOLDER; + + if (!file_exists($storagefolderpath)) { + mkdir($storagefolderpath); + } + + $data = [ + 'measure' => $this->measure, + 'start' => $this->start, + 'end' => $this->end, + 'duration' => $this->duration, + ]; + + file_put_contents($storagefolderpath . time() . '.json', json_encode($data)); + } + + /** + * Get current time. + * + * @return int Current time in milliseconds. + */ + private function now(): int { + return round(microtime(true) * 1000); + } + +} diff --git a/tests/behat/performance.feature b/tests/behat/performance.feature new file mode 100644 index 000000000..203737163 --- /dev/null +++ b/tests/behat/performance.feature @@ -0,0 +1,32 @@ +@app @javascript @performance +Feature: Measure performance. + +Scenario: [FCP] First Contentful Paint + Given I start timing "FCP" + When I launch the app runtime + Then I should find "Welcome to the Moodle App!" in the app + + When I stop timing "FCP" + Then "FCP" should have taken less than 5 seconds + +Scenario: [TTI] Time to Interactive + Given I start timing "TTI" + When I launch the app runtime + Then I should find "Welcome to the Moodle App!" in the app + + When I press "Skip" in the app + Then I should find "Connect to Moodle" in the app + + When I stop timing "TTI" + Then "TTI" should have taken less than 6 seconds + +Scenario: [TBT] Total Blocking Time + Given I launch the app runtime + Then I should find "Welcome to the Moodle App!" in the app + + When I start timing "TBT" + And I press "Skip" in the app + Then I should find "Connect to Moodle" in the app + + When I stop timing "TBT" + Then "TBT" should have taken less than 2 seconds