diff --git a/Dockerfile b/Dockerfile index 0563d423fba55d28a2dfbf2ea5f3e3a3ec7e23fc..fb5d2ad4baa90fa0b4c52d9846e41059b706cd57 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ SHELL ["/bin/bash", "-c"] # Base # ######## -RUN pip install mkdocs>=1.6.0 mkdocs-material==9.5.22 mkdocs-htmlproofer-plugin==1.2.1 mkdocs-video==1.5.0 +RUN pip install setuptools mkdocs>=1.6.0 mkdocs-material==9.5.22 mkdocs-htmlproofer-plugin==1.3.0 mkdocs-video==1.5.0 ########## # Linter # diff --git a/doc.zih.tu-dresden.de/docs/contrib/contribute_container.md b/doc.zih.tu-dresden.de/docs/contrib/contribute_container.md index 639e03f090f72f518058d705e7b3ebd4cebe2063..36d7e4ee986cca175c589bd958f8721e9f60983f 100644 --- a/doc.zih.tu-dresden.de/docs/contrib/contribute_container.md +++ b/doc.zih.tu-dresden.de/docs/contrib/contribute_container.md @@ -205,22 +205,33 @@ wikiscript doc.zih.tu-dresden.de/util/check-no-floating.sh doc.zih.tu-dresden.de No one likes dead links. -Therefore, we check the internal and external links within the markdown source files. To check a -single file, e.g. `doc.zih.tu-dresden.de/docs/software/big_data_frameworks.md`, use: +Therefore, we check the internal and external links within the markdown source files. +With the script `doc.zih.tu-dresden.de/util/check-links.sh`, you can either check +[a single file](#single-file), [all modified files](#all-modified-files) or +[all files](#all-files) of the compendium. + +##### Single File + +To check the links within a single file, e.g. +`doc.zih.tu-dresden.de/docs/software/big_data_frameworks.md`, use: ```bash wikiscript doc.zih.tu-dresden.de/util/check-links.sh docs/software/big_data_frameworks.md ``` -The script can also check all modified files, i.e., markdown files which are part of the repository -and different to the `main` branch. Use this script before committing your changes to make sure -your commit passes the CI/CD pipeline: +##### All Modified Files + +The script can also check the links in all modified files, i.e., markdown files which are part +of the repository and different to the `preview` branch. Use this script before committing your +changes to make sure your commit passes the CI/CD pipeline. ```bash -wikiscript doc.zih.tu-dresden.de/util/check-links.sh +wikiscript doc.zih.tu-dresden.de/util/check-links.sh -c ``` -To check all markdown file, which may take a while and give a lot of output, use: +##### All Files + +Checking the links of all markdown files takes a moment of time: ```bash wikiscript doc.zih.tu-dresden.de/util/check-links.sh -a diff --git a/doc.zih.tu-dresden.de/mkdocs.yml b/doc.zih.tu-dresden.de/mkdocs.yml index dc2141aac470a835798e9a188479c2c3818ec3c1..bc76a7c6e8b9f00d306b2fd9bf86c0a4846689d4 100644 --- a/doc.zih.tu-dresden.de/mkdocs.yml +++ b/doc.zih.tu-dresden.de/mkdocs.yml @@ -233,6 +233,8 @@ plugins: ignore_urls: - mailto:hpc-support@tu-dresden.de - https://jupyterhub.hpc.tu-dresden.de* + ignore_pages: + - archive/* skip_downloads: True # Toggle validating external URLs validate_external_urls: !ENV [ENABLED_HTMLPROOFER_EXTERNAL_URLS, False] diff --git a/doc.zih.tu-dresden.de/util/check-bash-syntax.sh b/doc.zih.tu-dresden.de/util/check-bash-syntax.sh index c56749b7955fe183a185e64113847940b36ffc5b..d7b83d6eee83104f04f9ab8003f1951329c91564 100755 --- a/doc.zih.tu-dresden.de/util/check-bash-syntax.sh +++ b/doc.zih.tu-dresden.de/util/check-bash-syntax.sh @@ -52,7 +52,7 @@ elif [[ ! -z "${file}" ]]; then files=${file} else echo "Search in git-changed files." - files=`git diff --name-only "$(git merge-base HEAD "${branch}")" | grep '\.sh$' || true` + files=`git diff --name-only --diff-filter=d "$(git merge-base HEAD "${branch}")" | grep '\.sh$' || true` fi diff --git a/doc.zih.tu-dresden.de/util/check-links-non-mkdocs.sh b/doc.zih.tu-dresden.de/util/check-links-non-mkdocs.sh deleted file mode 100755 index 5de2fbce738dc1827f0990e75c36d6cede4a786f..0000000000000000000000000000000000000000 --- a/doc.zih.tu-dresden.de/util/check-links-non-mkdocs.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/bash -## Purpose: -## Checks internal links for all (git-)changed markdown files (.md) of the repository. -## We use the markdown-link-check script from https://github.com/tcort/markdown-link-check, -## which can either be installed as local or global module -## nmp npm install [--save-dev|-g] markdown-link-check -## module. -## -## Author: Martin.Schroschk@tu-dresden.de - -set -eo pipefail - -scriptpath=${BASH_SOURCE[0]} -basedir=`dirname "${scriptpath}"` -basedir=`dirname "${basedir}"` - -usage() { - cat <<-EOF -usage: $0 [file | -a] -If file is given, checks whether all links in it are reachable. -If parameter -a (or --all) is given instead of the file, checks all markdown files. -Otherwise, checks whether any changed file contains broken links. -EOF -} - -mlc=markdown-link-check -if ! command -v ${mlc} &> /dev/null; then - echo "INFO: ${mlc} not found in PATH (global module)" - mlc=./node_modules/markdown-link-check/${mlc} - if ! command -v ${mlc} &> /dev/null; then - echo "INFO: ${mlc} not found (local module)" - exit 1 - fi -fi - -echo "mlc: ${mlc}" - -LINK_CHECK_CONFIG="${basedir}/util/link-check-config.json" -if [[ ! -f "${LINK_CHECK_CONFIG}" ]]; then - echo ${LINK_CHECK_CONFIG} does not exist - exit 1 -fi - -branch="preview" -if [[ -n "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}" ]]; then - branch="origin/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}" -fi - -function check_single_file(){ - theFile="$1" - if [[ -e "${theFile}" ]]; then - echo "Checking links in ${theFile}" - if ! ${mlc} -q -c "${LINK_CHECK_CONFIG}" -p "${theFile}"; then - return 1 - fi - fi - return 0 -} - -function check_files(){ -any_fails=false -echo "Check files:" -echo "${files}" -echo "" -for f in ${files}; do - if ! check_single_file "${f}"; then - any_fails=true - fi -done - -if [[ "${any_fails}" == true ]]; then - exit 1 -fi -} - -function check_all_files(){ -files=$(git ls-tree --full-tree -r --name-only HEAD ${basedir}/ | grep '.md$' || true) -check_files -} - -function check_changed_files(){ -files=$(git diff --name-only "$(git merge-base HEAD "${branch}")" | grep '.md$' || true) -check_files -} - -if [[ $# -eq 1 ]]; then - case $1 in - help | -help | --help) - usage - exit - ;; - -a | --all) - check_all_files - ;; - *) - checkSingleFile "$1" - ;; - esac -elif [[ $# -eq 0 ]]; then - check_changed_files -else - usage -fi diff --git a/doc.zih.tu-dresden.de/util/check-links.sh b/doc.zih.tu-dresden.de/util/check-links.sh index c876dcd78807a9e70758cdad396c370bb3425e13..e935fdff8044cd8abd02b0a6986397f134873848 100755 --- a/doc.zih.tu-dresden.de/util/check-links.sh +++ b/doc.zih.tu-dresden.de/util/check-links.sh @@ -1,46 +1,142 @@ #!/bin/bash # Purpose: -# Checks internal and external links for all markdown files (.md) of the repository. -# For link checking, we use html-proofer-plugin, which is configured in mkdocs.yml. +# Checks internal and external links for markdown files (.md) of the repository +# (i.e. HPC compendium). This script can check the URLs +# 1. in all markdown files +# 2. git-changed markdown files +# 3. single markdown file +# +# For option 1, we make use of the html-proofer-plugin, which is configured in mkdocs.yml. # By default the link checker only inspects internal links. It will inspect internal and external -# links, when the env. variable ENABLED_HTMLPROOFER_EXTERNAL_URLS is set to true. +# links, since we set the env. variable ENABLED_HTMLPROOFER_EXTERNAL_URLS to true in this script. +# +# For option 2 and 3, we use the markdown-link-check script from +# https://github.com/tcort/markdown-link-check, which can either be installed as local or global +# module (nmp npm install [--save-dev|-g] markdown-link-check). +# +# General remarks for future me: +# - html-proofer-plugin is heavily customizable (ignore certain urls or pages, etc.) +# - html-proofer-plugin is quite slow, since it requires to build the whole compendium +# - html-proofer-plugin works on rendered html files +# - html-proofer-plugin has no option to check a single page set -eo pipefail -export ENABLED_HTMLPROOFER=true +scriptpath=${BASH_SOURCE[0]} +basedir=`dirname "${scriptpath}"` +basedir=`dirname "${basedir}"` -function usage(){ -cat << EOF -$0 [options] -Check links in documentation. +usage() { + cat <<-EOF +usage: $0 [file | -a | -c] Options: - -a Check internal and external links (default: internal links only) - -h Show help message + <file> Check links in the provided markdown file + -a; --all Check links in all markdown files + -i; --internal Check links in all markdown files (internal links only) + -c; --changed Check links in all git-changed files EOF } -while getopts ":a" option; do - case ${option} in - a) - export ENABLED_HTMLPROOFER_EXTERNAL_URLS=true - echo "Info: Checking internal and external links. Might take some time." - ;; - h) - usage - exit;; - \?) - echo "Error: Invalid option." - usage - exit;; - esac -done +mlc=markdown-link-check +if ! command -v ${mlc} &> /dev/null; then + echo "INFO: ${mlc} not found in PATH (global module)" + mlc=./node_modules/markdown-link-check/${mlc} + if ! command -v ${mlc} &> /dev/null; then + echo "INFO: ${mlc} not found (local module)" + exit 1 + fi +fi + +echo "mlc: ${mlc}" + +LINK_CHECK_CONFIG="${basedir}/util/link-check-config.json" +if [[ ! -f "${LINK_CHECK_CONFIG}" ]]; then + echo ${LINK_CHECK_CONFIG} does not exist + exit 1 +fi -# Switch for wikiscript -if [[ -d "doc.zih.tu-dresden.de" ]]; then - cd doc.zih.tu-dresden.de +branch="preview" +if [[ -n "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}" ]]; then + branch="origin/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}" fi -mkdocs build -rm -rf public +function check_single_file(){ + theFile="$1" + if [[ -e "${theFile}" ]]; then + echo "Checking links in ${theFile}" + if ! ${mlc} -q -c "${LINK_CHECK_CONFIG}" -p "${theFile}"; then + return 1 + fi + else + echo "$1 not found" + fi + + return 0 +} + +function check_files(){ + any_fails=false + echo "Check files:" + echo "${files}" + echo "" + for f in ${files}; do + if ! check_single_file "${f}"; then + any_fails=true + fi + done + + if [[ "${any_fails}" == true ]]; then + exit 1 + fi +} + +function check_changed_files(){ + files=$(git diff --name-only "$(git merge-base HEAD "${branch}")" | grep '.md$' || true) + check_files +} + +# Make use of the html-proofer-plugin and build documentation +function check_all_files_hpp() { + export ENABLED_HTMLPROOFER=true + # Switch for wikiscript + if [[ -d "doc.zih.tu-dresden.de" ]]; then + cd doc.zih.tu-dresden.de + fi + mkdocs build + rm -rf public +} + +if [[ $# -ge 1 ]]; then + case $1 in + -h | help | -help | --help) + usage + exit 0 + ;; + -a | --all) + export ENABLED_HTMLPROOFER_EXTERNAL_URLS=true + echo "Info: Building documentation and checking internal and external links. Might take some time." + check_all_files_hpp + ;; + -i | --internal) + export ENABLED_HTMLPROOFER_EXTERNAL_URLS=false + echo "Info: Building documentation and checking internal links. Might take some time." + check_all_files_hpp + ;; + -c | --changed) + echo "Info: Checking links in modified files." + check_changed_files + ;; + *) + echo "Info: Checking links in file $1" + check_single_file "$1" + ;; + esac +else + echo -e "Remark: The script changed in Oct. 2024. You did not provide any option. + If you intend to check the links of all modified files, please + provide the option \"-c\"\n" + usage + exit 1 +fi diff --git a/doc.zih.tu-dresden.de/util/pre-commit b/doc.zih.tu-dresden.de/util/pre-commit index 2a3305b8f9af415817caaac71803283103f269e4..e0f4c030fc625a46d83ab60d3fcb28fa1956f92c 100755 --- a/doc.zih.tu-dresden.de/util/pre-commit +++ b/doc.zih.tu-dresden.de/util/pre-commit @@ -47,7 +47,7 @@ while read -r; do fi echo "Checking links..." - if ! runInDockerDocOnly markdown-link-check "$doc_filepath"; then + if ! runInDocker ./doc.zih.tu-dresden.de/util/check-links.sh -c; then echo -e "\tFailed" exit_ok=no fi