diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 917bdc39a..8807b802d 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -15,14 +15,21 @@ jobs: uses: actions/setup-node@v1 with: node-version: '12.x' - - run: npm ci - - run: git clone --branch master --depth 1 git://github.com/moodle/moodle $GITHUB_WORKSPACE/moodle - - run: git clone --branch ionic5 --depth 1 git://github.com/moodlehq/moodle-local_moodlemobileapp $GITHUB_WORKSPACE/moodle/local/moodlemobileapp - - run: git clone --branch MOBILE-3738 --depth 1 git://github.com/NoelDeMartin/moodle-docker $GITHUB_WORKSPACE/moodle-docker - - run: cp $GITHUB_WORKSPACE/moodle-docker/config.docker-template.php $GITHUB_WORKSPACE/moodle/config.php - - run: MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle MOODLE_DOCKER_APP_PATH=$GITHUB_WORKSPACE $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-compose pull - - run: MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle MOODLE_DOCKER_APP_PATH=$GITHUB_WORKSPACE $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-compose up -d - - run: MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle MOODLE_DOCKER_APP_PATH=$GITHUB_WORKSPACE $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-wait-for-db - - run: MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle MOODLE_DOCKER_APP_PATH=$GITHUB_WORKSPACE $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-wait-for-app - - run: MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle MOODLE_DOCKER_APP_PATH=$GITHUB_WORKSPACE $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-compose exec -T webserver sh -c "php admin/tool/behat/cli/init.php" - - run: MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle MOODLE_DOCKER_APP_PATH=$GITHUB_WORKSPACE $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-compose exec -T webserver sh -c "php admin/tool/behat/cli/run.php --tags="@app" --auto-rerun" + - name: Install npm packages + run: npm ci + - name: Additional checkouts + run: | + git clone --branch master --depth 1 git://github.com/moodle/moodle $GITHUB_WORKSPACE/moodle + git clone --branch ionic5 --depth 1 git://github.com/moodlehq/moodle-local_moodlemobileapp $GITHUB_WORKSPACE/moodle/local/moodlemobileapp + git clone --branch MOBILE-3738 --depth 1 git://github.com/NoelDeMartin/moodle-docker $GITHUB_WORKSPACE/moodle-docker + - name: Setup docker machine + run: | + cp $GITHUB_WORKSPACE/moodle-docker/config.docker-template.php $GITHUB_WORKSPACE/moodle/config.php + MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle MOODLE_DOCKER_APP_PATH=$GITHUB_WORKSPACE $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-compose pull + MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle MOODLE_DOCKER_APP_PATH=$GITHUB_WORKSPACE $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-compose up -d + MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle MOODLE_DOCKER_APP_PATH=$GITHUB_WORKSPACE $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-wait-for-db + MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle MOODLE_DOCKER_APP_PATH=$GITHUB_WORKSPACE $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-wait-for-app + - name: Init behat + run: MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle MOODLE_DOCKER_APP_PATH=$GITHUB_WORKSPACE $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-compose exec -T webserver sh -c "php admin/tool/behat/cli/init.php" + - name: Run tests + run: MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle MOODLE_DOCKER_APP_PATH=$GITHUB_WORKSPACE $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-compose exec -T webserver sh -c "php admin/tool/behat/cli/run.php --tags="@app" --auto-rerun" diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index bb24d1c75..7c8d1e4fe 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -13,8 +13,45 @@ jobs: uses: actions/setup-node@v1 with: node-version: '12.x' - - run: npm ci - - run: npm run lint - - run: npm run test:ci - - run: npm run build:prod - - run: result=$(npx check-es-compat www/*.js 2> /dev/null | grep -v -E "Array\.prototype\.includes|Promise\.prototype\.finally|String\.prototype\.(matchAll|trimRight)|globalThis" | grep -Po "(?<=error).*?(?=\s+ecmascript)" | wc -l); test $result -eq 0 + - name: Install npm packages + run: npm ci + - name: Check langindex + run: | + result=$(cat scripts/langindex.json | grep \"TBD\" | wc -l); test $result -eq 0 + if [ $result -ne 0 ]; then + echo "There are lang strings to be decided on langindex.json" + exit 1 + fi + gulp + langlength=`jq -r '. | length' src/assets/lang/en.json` + langindexlength=`jq -r '. | length' scripts/langindex.json` + if [ $langlength -ne $langindexlength ]; then + echo "Lang file has $langlength while langindex $langindexlength" + exit 1 + fi + + langkeys=`jq -r 'keys[]' src/assets/lang/en.json` + langindex=`jq -r 'keys[]' scripts/langindex.json` + found=0 + for i in $langkeys; do + skip= + for j in $langindex; do + if [ "$i" == "$j" ]; then + skip=1 + break; + fi + done + [[ -n $skip ]] || { echo "$i key not found"; found=$(($found + 1)); } + done + if [ $found -ne 0 ]; then + echo "Found $found missing langkeys" + exit 1 + fi + - name: Run Linter + run: npm run lint + - name: Run tests + run: npm run test:ci + - name: Production builds + run: npm run build:prod + - name: JavaScript code compatibility + run: result=$(npx check-es-compat www/*.js 2> /dev/null | grep -v -E "Array\.prototype\.includes|Promise\.prototype\.finally|String\.prototype\.(matchAll|trimRight)|globalThis" | grep -Po "(?<=error).*?(?=\s+ecmascript)" | wc -l); test $result -eq 0 diff --git a/scripts/create_langindex.sh b/scripts/create_langindex.sh index 432419b3c..c17d77c20 100755 --- a/scripts/create_langindex.sh +++ b/scripts/create_langindex.sh @@ -1,5 +1,12 @@ #!/bin/bash +# +# Script to create langindex from available language packs. +# ./create_langindex.sh [findbetter] +# If findbetter is set it will try to find a better solution for every key. +# + source "functions.sh" +source "lang_functions.sh" #Saves or updates a key on langindex_old.json function save_key { @@ -48,7 +55,8 @@ function exists_in_mobile { } function do_match { - match=$1 + match=${1/\{\{/\{} + match=${match/\}\}/\}} filematch="" coincidence=`grep "$match" $LANGPACKSFOLDER/en/*.php | wc -l` @@ -56,7 +64,7 @@ function do_match { filematch=`grep "$match" $LANGPACKSFOLDER/en/*.php | cut -d'/' -f5 | cut -d'.' -f1` exists_in_file $filematch $plainid elif [ $coincidence -gt 0 ] && [ "$#" -gt 1 ]; then - print_message $2 + print_message "$2" tput setaf 6 grep "$match" $LANGPACKSFOLDER/en/*.php fi @@ -67,18 +75,21 @@ function find_matches { do_match "string\[\'$plainid\'\] = \'$value\'" "Found EXACT match for $key in the following paths" if [ $coincidence -gt 0 ]; then case=1 + save_key $key "TBD" return fi do_match " = \'$value\'" "Found some string VALUES for $key in the following paths" if [ $coincidence -gt 0 ]; then case=2 + save_key $key "TBD" return fi do_match "string\[\'$plainid\'\]" "Found some string KEYS for $key in the following paths, value $value" if [ $coincidence -gt 0 ]; then case=3 + save_key $key "TBD" return fi @@ -297,17 +308,13 @@ function array_contains { done } + print_title 'Generating language from code...' gulp lang print_title 'Getting languages' -git clone https://git.in.moodle.com/moodle/moodle-langpacks.git $LANGPACKSFOLDER -pushd $LANGPACKSFOLDER -BRANCHES=($(git branch -r --format="%(refname:lstrip=3)" --sort="refname" | grep MOODLE_)) -BRANCH=${BRANCHES[${#BRANCHES[@]}-1]} -git checkout $BRANCH -git pull -popd + +get_language en print_title 'Processing file' #Create langindex.json if not exists. diff --git a/scripts/functions.sh b/scripts/functions.sh index f90399189..45d2ab331 100644 --- a/scripts/functions.sh +++ b/scripts/functions.sh @@ -1,7 +1,5 @@ #!/bin/bash -LANGPACKSFOLDER='../../moodle-langpacks' - function check_success_exit { if [ $? -ne 0 ]; then print_error "$1" diff --git a/scripts/lang_functions.sh b/scripts/lang_functions.sh new file mode 100755 index 000000000..d8bcf38ae --- /dev/null +++ b/scripts/lang_functions.sh @@ -0,0 +1,132 @@ +#!/bin/bash +# +# Functions to fetch languages. +# + +LANGPACKSFOLDER='../../moodle-langpacks' +BUCKET='moodle-lang-prod' +MOODLEORG_URL='https://download.moodle.org/download.php/direct/langpack' +DEFAULT_LASTVERSION='4.0' + +# Checks if AWS is available and configured. +function check_aws { + aws --version &> /dev/null + AWS_SERVICE=1 + if [ $? -ne 0 ]; then + AWS_SERVICE=0 + echo 'AWS not installed. Check https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html for more info.' + return + fi + + # In order to login to AWS, use credentials file or AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY vars. + if [ ! -f ~/.aws/credentials ] && [ [ -z $AWS_ACCESS_KEY_ID ] || [ -z $AWS_SECRET_ACCESS_KEY ] ]; then + AWS_SERVICE=0 + lastversion=$DEFAULT_LASTVERSION + echo 'AWS Cannot authenticate. Use aws configure or set the proper env vars.' + return + fi +} + +# Get last version of Moodle to fetch latest languages. +function get_last_version { + if [ ! -z "${lastversion}" ]; then + return + fi + + check_aws + if [ $AWS_SERVICE -eq 0 ]; then + lastversion=$DEFAULT_LASTVERSION + echo "Using default version $lastversion" + return + fi + + list=`aws s3 ls s3://$BUCKET/` + if [ $? -ne 0 ]; then + AWS_SERVICE=0 + lastversion=$DEFAULT_LASTVERSION + echo "AWS Cannot authenticate. Using default version $lastversion" + return + fi + + lastversion='' + for folder in $list; do + if [ $folder != 'PRE' ]; then + lastversion=${folder/\//} + fi + done + + if [ ! -z "${lastversion}" ]; then + echo "Last version $lastversion detected" + return + fi + + lastversion=$DEFAULT_LASTVERSION +} + +# Get all language list from AWS. +function get_all_languages_aws { + langsfiles=`aws s3 ls s3://$BUCKET/$lastversion/` + langs="" + for file in $langsfiles; do + if [[ "$file" == *.zip ]]; then + file=${file/\.zip/} + langs+="$file " + fi + done +} + +# Get language list from the installed ones (will not discover new translations). +function get_installed_languages { + langs=`jq -r '.languages | keys[]' ../moodle.config.json` +} + +# Entry function to get a language file. +function get_language { + lang=$1 + lang=${lang/-/_} + + get_last_version + + echo "Getting $lang language" + + pushd $LANGPACKSFOLDER > /dev/null + + curl -s $MOODLEORG_URL/$lastversion/$lang.zip --output $lang.zip > /dev/null + rm -R $lang + unzip -o -u $lang.zip > /dev/null + + # This is the AWS version to get the language but right now it's slower. + # aws s3 cp s3://$BUCKET/$lastversion/$lang.zip . > /dev/null + + rm $lang.zip + popd > /dev/null +} + +# Entry function to get all language files. +function get_languages { + get_last_version + + if [ -d $LANGPACKSFOLDER ]; then + lastupdate=`date -r $LANGPACKSFOLDER +%s` + currenttime=`date +%s` + ellapsedtime=$((currenttime - lastupdate)) + if [ $ellapsedtime -lt 3600 ]; then + echo 'Recently updated, skip update languages' + return + fi + else + mkdir $LANGPACKSFOLDER + fi + + + if [ $AWS_SERVICE -eq 1 ]; then + get_all_languages_aws + else + echo "Fallback language list will only get current installation languages" + get_installed_languages + fi + + for lang in $langs; do + get_language "$lang" + done +} diff --git a/scripts/update_lang.sh b/scripts/update_lang.sh index dac901789..b47df3a7d 100755 --- a/scripts/update_lang.sh +++ b/scripts/update_lang.sh @@ -1,23 +1,23 @@ #!/bin/bash +# +# Script to update language packs on assets and detect new translated languages. +# ./update_lang.sh [language] +# If language is set it will only update the selected language. +# source "functions.sh" -forceLang=$1 +source "lang_functions.sh" -print_title 'Getting languages' -git clone --depth 1 --no-single-branch https://git.in.moodle.com/moodle/moodle-langpacks.git $LANGPACKSFOLDER -pushd $LANGPACKSFOLDER -BRANCHES=($(git branch -r --format="%(refname:lstrip=3)" --sort="refname" | grep MOODLE_)) -BRANCH=${BRANCHES[${#BRANCHES[@]}-1]} -git checkout $BRANCH -git pull -popd +forceLang=$1 print_title 'Getting local mobile langs' git clone --depth 1 https://github.com/moodlehq/moodle-local_moodlemobileapp.git ../../moodle-local_moodlemobileapp if [ -z $forceLang ]; then + get_languages php -f moodle_to_json.php else + get_language "$forceLang" php -f moodle_to_json.php "$forceLang" fi -print_ok 'All done!' \ No newline at end of file +print_ok 'All done!'