diff --git a/doc.zih.tu-dresden.de/.markdownlintrc b/doc.zih.tu-dresden.de/.markdownlintrc index 4a9cce8fa8c1ae5e3a08b42433b22a325eab0252..390d6eca08d75dbcdfe2ad1c1b3b82872950c323 100644 --- a/doc.zih.tu-dresden.de/.markdownlintrc +++ b/doc.zih.tu-dresden.de/.markdownlintrc @@ -3,8 +3,6 @@ "first-heading-h1": true, "header-increment": true, "heading-style": { "style": "atx" }, - "no-missing-space-atx": true, - "no-multiple-space-atx": true, "blanks-around-headings": true, "no-emphasis-as-heading": true, "first-line-heading": true, diff --git a/doc.zih.tu-dresden.de/docs/application/request_for_resources.md b/doc.zih.tu-dresden.de/docs/application/request_for_resources.md index 5181f8393ce8109208f2e3720afe05b09de2dcad..09c5161147505b4ac79814cbf8d5aec654beb15c 100644 --- a/doc.zih.tu-dresden.de/docs/application/request_for_resources.md +++ b/doc.zih.tu-dresden.de/docs/application/request_for_resources.md @@ -1,26 +1,22 @@ # Request for Resources -Important note: ZIH systems are based on the Linux system. Thus for the effective work, you should -have to know how to work with Linux based systems and Linux Shell. Beginners can find a lot of -different tutorials on the internet, for example. +Important note: ZIH systems run Linux. Thus, for effective work, you should know your way around +Linux based systems and the Bash Shell. Beginners can find a lot of tutorials on the internet. ## Determine the Required CPU and GPU Hours -ZIH systems are focused on data-intensive computing. The cluster is oriented on the work with the -high parallel code. Please keep it in mind for the transfer sequential code from a local machine. -So far you will have execution time for the sequential program it is reasonable to use -[Amdahl's law][1] to roughly predict execution time in parallel. Think in advance about the -parallelization strategy for your project. +ZIH systems are focused on data-intensive computing. They are meant to be used for highly +parallelized code. Please take that into account when migrating sequential code from a local +machine to our HPC systems. To estimate your execution time when executing your previously +sequential program in parallel, you can use [Amdahl's law](https://en.wikipedia.org/wiki/Amdahl%27s_law). +Think in advance about the parallelization strategy for your project and how to effectively use HPC resources. ## Available Software -The good practice for the HPC clusters is to use software and packages where parallelization is -possible. The open-source software is more preferable than proprietary. However, the majority of -popular programming languages, scientific applications, software, packages available or could be -installed on the HPC cluster in different ways. First of all, check the [Software module list][2]. -There are [two different software environments](../software/modules.md): `scs5` (the regular one) -and `ml` (environment for the Machine Learning partition). Keep in mind that ZIH systems have a -Linux based operating system. - -[1]: https://en.wikipedia.org/wiki/Amdahl%27s_law -[2]: https://gauss-allianz.de/de/application?organizations%5B0%5D=1200 +Pre-installed software on our HPC systems is managed via [modules](../software/modules.md). +You can see the +[list of software that's already installed and accessible via modules](https://gauss-allianz.de/de/application?organizations%5B0%5D=1200). +However, there are many +different variants of these modules available. We have divided these into two different software +environments: `scs5` (for regular partitions) and `ml` (for the Machine Learning partition). Within +each environment there are further dependencies and variants. diff --git a/doc.zih.tu-dresden.de/docs/jobs_and_resources/partitions_and_limits.md b/doc.zih.tu-dresden.de/docs/jobs_and_resources/partitions_and_limits.md index ee4343c8b2eff0cf6e9eac29cb8b0cd1d7abc135..ae450f8b46a75b3457428deaee8a24caff355e85 100644 --- a/doc.zih.tu-dresden.de/docs/jobs_and_resources/partitions_and_limits.md +++ b/doc.zih.tu-dresden.de/docs/jobs_and_resources/partitions_and_limits.md @@ -32,7 +32,7 @@ Shorter jobs come with multiple advantages: To bring down the percentage of long running jobs we restrict the number of cores with jobs longer than 2 days to approximately 50% and with jobs longer than 24 to 75% of the total number of cores. -(These numbers are subject to changes.) As best practice we advise a run time of about 8h. +(These numbers are subject to change.) As best practice we advise a run time of about 8h. !!! hint "Please always try to make a good estimation of your needed time limit." @@ -58,10 +58,11 @@ not capable of checkpoint/restart can be adapted. Please refer to the section Memory requirements for your job can be specified via the `sbatch/srun` parameters: -`--mem-per-cpu=<MB>` or `--mem=<MB>` (which is "memory per node"). The **default limit** is quite -low at **300 MB** per CPU. +`--mem-per-cpu=<MB>` or `--mem=<MB>` (which is "memory per node"). The **default limit** regardless +of the partition it runs on is quite low at **300 MB** per CPU. If you need more memory, you need +to request it. -ZIH systems comprises different sets of nodes with different amount of installed memory which affect +ZIH systems comprise different sets of nodes with different amount of installed memory which affect where your job may be run. To achieve the shortest possible waiting time for your jobs, you should be aware of the limits shown in the following table. diff --git a/doc.zih.tu-dresden.de/util/check-code-style.sh b/doc.zih.tu-dresden.de/util/check-code-style.sh index 64dce7c637e9909fd1ebd8cd07a289bfc151417b..8a06519389846bcff38c4d9c22d23c4a1273ff50 100755 --- a/doc.zih.tu-dresden.de/util/check-code-style.sh +++ b/doc.zih.tu-dresden.de/util/check-code-style.sh @@ -30,9 +30,6 @@ EOF } function pattern_matches() { - local any_fails - any_fails=false - local myfile myfile=$1 @@ -49,21 +46,20 @@ function pattern_matches() { test_res_count=0 if [[ "${ext}" == "sh" ]]; then # If shell script is provided - test_res_count=$(grep -cP "${pattern}" $myfile || true) - if [[ "${test_res_count}" -gt "0" ]]; then + if grep -qP "${pattern}" "$myfile"; then echo -e "[WARNING] ${warning}" - echo "[WARNING] This coding style was not used for following lines in file $(realpath ${myfile}):" - grep -nP "${pattern}" $myfile + echo "[WARNING] This coding style was not used for following lines in file $(realpath "${myfile}"):" + grep -nP "${pattern}" "$myfile" echo "" - any_fails=true + return 0 fi elif [[ "${ext}" == "md" ]]; then # If markdown file is provided # Check if the code snippet exists in the markdown file - if sed -n '/^```bash$/,/^```$/p' $myfile | grep -qv '```'; then + if sed -n '/^```bash$/,/^```$/p' "$myfile" | grep -qv '```'; then # Extracting code snippet within ```bash ... ``` local test_string - test_string=$(sed -n '/^```bash$/,/^```$/p' $myfile | grep -v '```') + test_string=$(sed -n '/^```bash$/,/^```$/p' "$myfile" | grep -v '```') # Check the exit code of pattern match if echo "${test_string}" | grep -qP "${pattern}"; then @@ -71,24 +67,22 @@ function pattern_matches() { fi if [[ "${test_res_count}" -gt "0" ]]; then echo -e "[WARNING] ${warning}" - echo "[WARNING] This coding style was not used for following lines in file $(realpath ${myfile}):" + echo "[WARNING] This coding style was not used for following lines in file $(realpath "${myfile}"):" grep -no -F "$(echo "${test_string}" | grep -P "${pattern}")" "${myfile}" echo "" - any_fails=true + return 0 fi fi fi - if [[ "${any_fails}" == true ]]; then - return 0 - fi + # Reached here: Return false/non-zero, i.e. no error/match found return 1 } # ----------------------------------------------------------------------------- # Functions End scriptpath=${BASH_SOURCE[0]} -basedir=`dirname "${scriptpath}"` -basedir=`dirname "${basedir}"` +basedir=$(dirname "${scriptpath}") +basedir=$(dirname "${basedir}") branch="origin/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-preview}" @@ -101,7 +95,7 @@ if [[ $# -eq 1 ]]; then ;; -a | --all) echo "Checking in all files." - files=$(find $basedir -name '*.md' -o -name '*.sh') + files=$(find "$basedir" -name '*.md' -o -name '*.sh') ;; *) files="$1" @@ -109,7 +103,7 @@ if [[ $# -eq 1 ]]; then esac elif [[ $# -eq 0 ]]; then echo "Search in git-changed files." - files=`git diff --name-only "$(git merge-base HEAD "$branch")" | grep -e '.md$' -e '.sh$' || true` + files=$(git diff --name-only "$(git merge-base HEAD "$branch")" | grep -e '.md$' -e '.sh$' || true) else usage fi @@ -118,7 +112,7 @@ any_fails=false for file in $files; do # Skip the check of this current ($0) script. - if [[ "$(echo "${file}" | grep -cP "check-code-style.sh$")" -gt "0" ]]; then + if echo "${file}" | grep -qP "check-code-style.sh$"; then continue fi @@ -193,5 +187,3 @@ done if [[ "${any_fails}" == true ]]; then exit 1 fi -#------------------------------------------------------------------------------ -# Script End diff --git a/doc.zih.tu-dresden.de/util/pre-commit b/doc.zih.tu-dresden.de/util/pre-commit index f2550813b3c060d5658a14ca4093b20dd11f27e0..223a011052c42b483b4bbc8005c05ce5fcdf227a 100755 --- a/doc.zih.tu-dresden.de/util/pre-commit +++ b/doc.zih.tu-dresden.de/util/pre-commit @@ -1,11 +1,17 @@ #!/bin/bash function testPath(){ -path_to_test=doc.zih.tu-dresden.de/docs/$1 -test -f "$path_to_test" || echo $path_to_test does not exist + path_to_test="doc.zih.tu-dresden.de/docs/$1" + test -f "$path_to_test" || echo "$path_to_test does not exist" } -if ! `docker image inspect hpc-compendium:latest > /dev/null 2>&1` -then +function runInDocker(){ + docker run --name=hpc-compendium --rm -w /docs --mount src="$(pwd)",target=/docs,type=bind hpc-compendium "$@" +} +function runInDockerDocOnly(){ + docker run --name=hpc-compendium --rm -w /docs --mount src="$(pwd)"/doc.zih.tu-dresden.de,target=/docs,type=bind hpc-compendium "$@" +} + +if ! docker image inspect hpc-compendium:latest > /dev/null 2>&1; then echo Container not built, building... docker build -t hpc-compendium . fi @@ -14,90 +20,77 @@ export -f testPath exit_ok=yes branch="origin/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-preview}" -if [ -f "$GIT_DIR/MERGE_HEAD" ] -then - source_hash=`git merge-base HEAD "$branch"` +if [ -f "$GIT_DIR/MERGE_HEAD" ]; then + source_hash=$(git merge-base HEAD "$branch") else - source_hash=`git rev-parse HEAD` + source_hash=$(git rev-parse HEAD) fi #Remove everything except lines beginning with --- or +++ -files=`git diff $source_hash | sed -E -n 's#^(---|\+\+\+) ((/|./)[^[:space:]]+)$#\2#p'` +files=$(git diff "$source_hash" | sed -E -n 's#^(---|\+\+\+) ((/|./)[^[:space:]]+)$#\2#p') #Assume that we have pairs of lines (starting with --- and +++). -while read oldfile; do - read newfile +while read -r; do + read -r newfile - if [ "$newfile" == doc.zih.tu-dresden.de/mkdocs.yml ] - then + if [ "$newfile" == doc.zih.tu-dresden.de/mkdocs.yml ]; then echo Testing "$newfile" - sed -n '/^ *- /s#.*: \([A-Za-z_/]*.md\).*#\1#p' doc.zih.tu-dresden.de/mkdocs.yml | xargs -L1 -I {} bash -c "testPath '{}'" - if [ $? -ne 0 ] - then + if ! sed -n '/^ *- /s#.*: \([A-Za-z_/]*.md\).*#\1#p' doc.zih.tu-dresden.de/mkdocs.yml | xargs -L1 -I {} bash -c "testPath '{}'"; then + echo -e "\tFailed" exit_ok=no fi - elif [[ "$newfile" =~ ^b/doc.zih.tu-dresden.de/(.*.md)$ ]] - then - filepattern=${BASH_REMATCH[1]} + elif [[ "$newfile" =~ ^b/doc.zih.tu-dresden.de/(.*.md)$ ]]; then + doc_filepath="${BASH_REMATCH[1]}" echo "Linting..." - docker run --name=hpc-compendium --rm -w /docs --mount src="$(pwd)"/doc.zih.tu-dresden.de,target=/docs,type=bind hpc-compendium markdownlint $filepattern - if [ $? -ne 0 ] - then + if ! runInDockerDocOnly markdownlint "$doc_filepath"; then + echo -e "\tFailed" exit_ok=no fi echo "Checking links..." - docker run --name=hpc-compendium --rm -w /docs --mount src="$(pwd)"/doc.zih.tu-dresden.de,target=/docs,type=bind hpc-compendium markdown-link-check $filepattern - if [ $? -ne 0 ] - then + if ! runInDockerDocOnly markdown-link-check "$doc_filepath"; then + echo -e "\tFailed" exit_ok=no fi fi done <<< "$files" echo "Testing syntax of bash files..." -docker run --name=hpc-compendium --rm -w /docs --mount src="$(pwd)",target=/docs,type=bind hpc-compendium ./doc.zih.tu-dresden.de/util/check-bash-syntax.sh -if [ $? -ne 0 ] -then +if ! runInDocker ./doc.zih.tu-dresden.de/util/check-bash-syntax.sh; then + echo -e "\tFailed" exit_ok=no fi echo "Spell-checking..." -docker run --name=hpc-compendium --rm -w /docs --mount src="$(pwd)",target=/docs,type=bind hpc-compendium ./doc.zih.tu-dresden.de/util/check-spelling.sh -if [ $? -ne 0 ] -then +if ! runInDocker ./doc.zih.tu-dresden.de/util/check-spelling.sh; then + echo -e "\tFailed" exit_ok=no fi echo "Forbidden words checking..." -docker run --name=hpc-compendium --rm -w /docs --mount src="$(pwd)",target=/docs,type=bind hpc-compendium ./doc.zih.tu-dresden.de/util/grep-forbidden-patterns.sh -if [ $? -ne 0 ] -then +if ! runInDocker ./doc.zih.tu-dresden.de/util/grep-forbidden-patterns.sh; then + echo -e "\tFailed" exit_ok=no fi echo "Looking for empty files..." -docker run --name=hpc-compendium --rm -w /docs --mount src="$(pwd)",target=/docs,type=bind hpc-compendium ./doc.zih.tu-dresden.de/util/check-empty-page.sh -if [ $? -ne 0 ] -then +if ! runInDocker ./doc.zih.tu-dresden.de/util/check-empty-page.sh; then + echo -e "\tFailed" exit_ok=no fi echo "Looking for files with templates but without examples..." -docker run --name=hpc-compendium --rm -w /docs --mount src="$(pwd)",target=/docs,type=bind hpc-compendium ./doc.zih.tu-dresden.de/util/check-templated-code-snippets.sh -if [ $? -ne 0 ] -then +if ! runInDocker ./doc.zih.tu-dresden.de/util/check-templated-code-snippets.sh; then + echo -e "\tFailed" exit_ok=no fi echo "Checking code styling for files..." -docker run --name=hpc-compendium --rm -w /docs --mount src="$(pwd)",target=/docs,type=bind hpc-compendium ./doc.zih.tu-dresden.de/util/check-code-style.sh -if [ $? -ne 0 ] -then +if ! runInDocker ./doc.zih.tu-dresden.de/util/check-code-style.sh; then + echo -e "\tFailed" exit_ok=no fi -if [ $exit_ok == yes ] -then +if [ $exit_ok == yes ]; then exit 0 else exit 1