EVOgeek
/
Vmeda.Online
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MOBILE-3320 lang: Use AWS to update lang files

main
Pau Ferrer Ocaña 2021-05-13 12:13:26 +02:00
parent 0181661327
commit ea2750073a
6 changed files with 218 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
.github/workflows/e2e.yml vendored
View File

@ -15,14 +15,21 @@ jobs:
uses: actions/setup-node@v1
with:
node-version: '12.x'
- run: npm ci
- run: git clone --branch master --depth 1 git://github.com/moodle/moodle $GITHUB_WORKSPACE/moodle
- run: git clone --branch ionic5 --depth 1 git://github.com/moodlehq/moodle-local_moodlemobileapp $GITHUB_WORKSPACE/moodle/local/moodlemobileapp
- run: git clone --branch MOBILE-3738 --depth 1 git://github.com/NoelDeMartin/moodle-docker $GITHUB_WORKSPACE/moodle-docker
- run: cp $GITHUB_WORKSPACE/moodle-docker/config.docker-template.php $GITHUB_WORKSPACE/moodle/config.php
- run: MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle MOODLE_DOCKER_APP_PATH=$GITHUB_WORKSPACE $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-compose pull
- run: MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle MOODLE_DOCKER_APP_PATH=$GITHUB_WORKSPACE $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-compose up -d
- run: MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle MOODLE_DOCKER_APP_PATH=$GITHUB_WORKSPACE $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-wait-for-db
- run: MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle MOODLE_DOCKER_APP_PATH=$GITHUB_WORKSPACE $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-wait-for-app
- run: MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle MOODLE_DOCKER_APP_PATH=$GITHUB_WORKSPACE $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-compose exec -T webserver sh -c "php admin/tool/behat/cli/init.php"
- run: MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle MOODLE_DOCKER_APP_PATH=$GITHUB_WORKSPACE $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-compose exec -T webserver sh -c "php admin/tool/behat/cli/run.php --tags="@app" --auto-rerun"
- name: Install npm packages
run: npm ci
- name: Additional checkouts
run: |
git clone --branch master --depth 1 git://github.com/moodle/moodle $GITHUB_WORKSPACE/moodle
git clone --branch ionic5 --depth 1 git://github.com/moodlehq/moodle-local_moodlemobileapp $GITHUB_WORKSPACE/moodle/local/moodlemobileapp
git clone --branch MOBILE-3738 --depth 1 git://github.com/NoelDeMartin/moodle-docker $GITHUB_WORKSPACE/moodle-docker
- name: Setup docker machine
run: |
cp $GITHUB_WORKSPACE/moodle-docker/config.docker-template.php $GITHUB_WORKSPACE/moodle/config.php
MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle MOODLE_DOCKER_APP_PATH=$GITHUB_WORKSPACE $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-compose pull
MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle MOODLE_DOCKER_APP_PATH=$GITHUB_WORKSPACE $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-compose up -d
MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle MOODLE_DOCKER_APP_PATH=$GITHUB_WORKSPACE $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-wait-for-db
MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle MOODLE_DOCKER_APP_PATH=$GITHUB_WORKSPACE $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-wait-for-app
- name: Init behat
run: MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle MOODLE_DOCKER_APP_PATH=$GITHUB_WORKSPACE $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-compose exec -T webserver sh -c "php admin/tool/behat/cli/init.php"
- name: Run tests
run: MOODLE_DOCKER_WWWROOT=$GITHUB_WORKSPACE/moodle MOODLE_DOCKER_APP_PATH=$GITHUB_WORKSPACE $GITHUB_WORKSPACE/moodle-docker/bin/moodle-docker-compose exec -T webserver sh -c "php admin/tool/behat/cli/run.php --tags="@app" --auto-rerun"

47
.github/workflows/testing.yml vendored
View File

@ -13,8 +13,45 @@ jobs:
uses: actions/setup-node@v1
with:
node-version: '12.x'
- run: npm ci
- run: npm run lint
- run: npm run test:ci
- run: npm run build:prod
- run: result=$(npx check-es-compat www/*.js 2> /dev/null | grep -v -E "Array\.prototype\.includes|Promise\.prototype\.finally|String\.prototype\.(matchAll|trimRight)|globalThis" | grep -Po "(?<=error).*?(?=\s+ecmascript)" | wc -l); test $result -eq 0
- name: Install npm packages
run: npm ci
- name: Check langindex
run: |
result=$(cat scripts/langindex.json | grep \"TBD\" | wc -l); test $result -eq 0
if [ $result -ne 0 ]; then
echo "There are lang strings to be decided on langindex.json"
exit 1
fi
gulp
langlength=`jq -r '. | length' src/assets/lang/en.json`
langindexlength=`jq -r '. | length' scripts/langindex.json`
if [ $langlength -ne $langindexlength ]; then
echo "Lang file has $langlength while langindex $langindexlength"
exit 1
fi
langkeys=`jq -r 'keys[]' src/assets/lang/en.json`
langindex=`jq -r 'keys[]' scripts/langindex.json`
found=0
for i in $langkeys; do
skip=
for j in $langindex; do
if [ "$i" == "$j" ]; then
skip=1
break;
fi
done
[[ -n $skip ]] || { echo "$i key not found"; found=$(($found + 1)); }
done
if [ $found -ne 0 ]; then
echo "Found $found missing langkeys"
exit 1
fi
- name: Run Linter
run: npm run lint
- name: Run tests
run: npm run test:ci
- name: Production builds
run: npm run build:prod
- name: JavaScript code compatibility
run: result=$(npx check-es-compat www/*.js 2> /dev/null | grep -v -E "Array\.prototype\.includes|Promise\.prototype\.finally|String\.prototype\.(matchAll|trimRight)|globalThis" | grep -Po "(?<=error).*?(?=\s+ecmascript)" | wc -l); test $result -eq 0

25
scripts/create_langindex.sh
View File

@ -1,5 +1,12 @@
#!/bin/bash
#
# Script to create langindex from available language packs.
# ./create_langindex.sh [findbetter]
# If findbetter is set it will try to find a better solution for every key.
#
source "functions.sh"
source "lang_functions.sh"
#Saves or updates a key on langindex_old.json
function save_key {
@ -48,7 +55,8 @@ function exists_in_mobile {
}
function do_match {
match=$1
match=${1/\{\{/\{}
match=${match/\}\}/\}}
filematch=""
coincidence=`grep "$match" $LANGPACKSFOLDER/en/*.php | wc -l`
@ -56,7 +64,7 @@ function do_match {
filematch=`grep "$match" $LANGPACKSFOLDER/en/*.php | cut -d'/' -f5 | cut -d'.' -f1`
exists_in_file $filematch $plainid
elif [ $coincidence -gt 0 ] && [ "$#" -gt 1 ]; then
print_message $2
print_message "$2"
tput setaf 6
grep "$match" $LANGPACKSFOLDER/en/*.php
fi
@ -67,18 +75,21 @@ function find_matches {
do_match "string\[\'$plainid\'\] = \'$value\'" "Found EXACT match for $key in the following paths"
if [ $coincidence -gt 0 ]; then
case=1
save_key $key "TBD"
return
fi
do_match " = \'$value\'" "Found some string VALUES for $key in the following paths"
if [ $coincidence -gt 0 ]; then
case=2
save_key $key "TBD"
return
fi
do_match "string\[\'$plainid\'\]" "Found some string KEYS for $key in the following paths, value $value"
if [ $coincidence -gt 0 ]; then
case=3
save_key $key "TBD"
return
fi
@ -297,17 +308,13 @@ function array_contains {
done
}
print_title 'Generating language from code...'
gulp lang
print_title 'Getting languages'
git clone https://git.in.moodle.com/moodle/moodle-langpacks.git $LANGPACKSFOLDER
pushd $LANGPACKSFOLDER
BRANCHES=($(git branch -r --format="%(refname:lstrip=3)" --sort="refname" | grep MOODLE_))
BRANCH=${BRANCHES[${#BRANCHES[@]}-1]}
git checkout $BRANCH
git pull
popd
get_language en
print_title 'Processing file'
#Create langindex.json if not exists.

2
scripts/functions.sh
View File

@ -1,7 +1,5 @@
#!/bin/bash
LANGPACKSFOLDER='../../moodle-langpacks'
function check_success_exit {
if [ $? -ne 0 ]; then
print_error "$1"

132
scripts/lang_functions.sh 100755
View File

@ -0,0 +1,132 @@
#!/bin/bash
#
# Functions to fetch languages.
#
LANGPACKSFOLDER='../../moodle-langpacks'
BUCKET='moodle-lang-prod'
MOODLEORG_URL='https://download.moodle.org/download.php/direct/langpack'
DEFAULT_LASTVERSION='4.0'
# Checks if AWS is available and configured.
function check_aws {
aws --version &> /dev/null
AWS_SERVICE=1
if [ $? -ne 0 ]; then
AWS_SERVICE=0
echo 'AWS not installed. Check https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html for more info.'
return
fi
# In order to login to AWS, use credentials file or AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY vars.
if [ ! -f ~/.aws/credentials ] && [ [ -z $AWS_ACCESS_KEY_ID ] || [ -z $AWS_SECRET_ACCESS_KEY ] ]; then
AWS_SERVICE=0
lastversion=$DEFAULT_LASTVERSION
echo 'AWS Cannot authenticate. Use aws configure or set the proper env vars.'
return
fi
}
# Get last version of Moodle to fetch latest languages.
function get_last_version {
if [ ! -z "${lastversion}" ]; then
return
fi
check_aws
if [ $AWS_SERVICE -eq 0 ]; then
lastversion=$DEFAULT_LASTVERSION
echo "Using default version $lastversion"
return
fi
list=`aws s3 ls s3://$BUCKET/`
if [ $? -ne 0 ]; then
AWS_SERVICE=0
lastversion=$DEFAULT_LASTVERSION
echo "AWS Cannot authenticate. Using default version $lastversion"
return
fi
lastversion=''
for folder in $list; do
if [ $folder != 'PRE' ]; then
lastversion=${folder/\//}
fi
done
if [ ! -z "${lastversion}" ]; then
echo "Last version $lastversion detected"
return
fi
lastversion=$DEFAULT_LASTVERSION
}
# Get all language list from AWS.
function get_all_languages_aws {
langsfiles=`aws s3 ls s3://$BUCKET/$lastversion/`
langs=""
for file in $langsfiles; do
if [[ "$file" == *.zip ]]; then
file=${file/\.zip/}
langs+="$file "
fi
done
}
# Get language list from the installed ones (will not discover new translations).
function get_installed_languages {
langs=`jq -r '.languages | keys[]' ../moodle.config.json`
}
# Entry function to get a language file.
function get_language {
lang=$1
lang=${lang/-/_}
get_last_version
echo "Getting $lang language"
pushd $LANGPACKSFOLDER > /dev/null
curl -s $MOODLEORG_URL/$lastversion/$lang.zip --output $lang.zip > /dev/null
rm -R $lang
unzip -o -u $lang.zip > /dev/null
# This is the AWS version to get the language but right now it's slower.
# aws s3 cp s3://$BUCKET/$lastversion/$lang.zip . > /dev/null
rm $lang.zip
popd > /dev/null
}
# Entry function to get all language files.
function get_languages {
get_last_version
if [ -d $LANGPACKSFOLDER ]; then
lastupdate=`date -r $LANGPACKSFOLDER +%s`
currenttime=`date +%s`
ellapsedtime=$((currenttime - lastupdate))
if [ $ellapsedtime -lt 3600 ]; then
echo 'Recently updated, skip update languages'
return
fi
else
mkdir $LANGPACKSFOLDER
fi
if [ $AWS_SERVICE -eq 1 ]; then
get_all_languages_aws
else
echo "Fallback language list will only get current installation languages"
get_installed_languages
fi
for lang in $langs; do
get_language "$lang"
done
}

20
scripts/update_lang.sh
View File

@ -1,23 +1,23 @@
#!/bin/bash
#
# Script to update language packs on assets and detect new translated languages.
# ./update_lang.sh [language]
# If language is set it will only update the selected language.
#
source "functions.sh"
forceLang=$1
source "lang_functions.sh"
print_title 'Getting languages'
git clone --depth 1 --no-single-branch https://git.in.moodle.com/moodle/moodle-langpacks.git $LANGPACKSFOLDER
pushd $LANGPACKSFOLDER
BRANCHES=($(git branch -r --format="%(refname:lstrip=3)" --sort="refname" | grep MOODLE_))
BRANCH=${BRANCHES[${#BRANCHES[@]}-1]}
git checkout $BRANCH
git pull
popd
forceLang=$1
print_title 'Getting local mobile langs'
git clone --depth 1 https://github.com/moodlehq/moodle-local_moodlemobileapp.git ../../moodle-local_moodlemobileapp
if [ -z $forceLang ]; then
get_languages
php -f moodle_to_json.php
else
get_language "$forceLang"
php -f moodle_to_json.php "$forceLang"
fi
print_ok 'All done!'
print_ok 'All done!'