Merge branch 'review-check-links' into 'preview'

Refactor script check-links

See merge request !1129

Dockerfile

@@ -6,7 +6,7 @@ SHELL ["/bin/bash", "-c"]
 # Base #
 ########
 
-RUN pip install mkdocs>=1.6.0 mkdocs-material==9.5.22 mkdocs-htmlproofer-plugin==1.2.1 mkdocs-video==1.5.0
+RUN pip install setuptools mkdocs>=1.6.0 mkdocs-material==9.5.22 mkdocs-htmlproofer-plugin==1.3.0 mkdocs-video==1.5.0
 
 ##########
 # Linter #

doc.zih.tu-dresden.de/docs/contrib/contribute_container.md

@@ -205,22 +205,33 @@ wikiscript doc.zih.tu-dresden.de/util/check-no-floating.sh doc.zih.tu-dresden.de
 
     No one likes dead links.
 
-Therefore, we check the internal and external links within the markdown source files. To check a
-single file, e.g.  `doc.zih.tu-dresden.de/docs/software/big_data_frameworks.md`, use:
+Therefore, we check the internal and external links within the markdown source files.
+With the script `doc.zih.tu-dresden.de/util/check-links.sh`, you can either check
+[a single file](#single-file), [all modified files](#all-modified-files) or
+[all files](#all-files) of the compendium.
+
+##### Single File
+
+To check the links within a single file, e.g.
+`doc.zih.tu-dresden.de/docs/software/big_data_frameworks.md`, use:
 
 ```bash
 wikiscript doc.zih.tu-dresden.de/util/check-links.sh docs/software/big_data_frameworks.md
 ```
 
-The script can also check all modified files, i.e., markdown files which are part of the repository
-and different to the `main` branch. Use this script before committing your changes to make sure
-your commit passes the CI/CD pipeline:
+##### All Modified Files
+
+The script can also check the links in all modified files, i.e., markdown files which are part
+of the repository and different to the `preview` branch. Use this script before committing your
+changes to make sure your commit passes the CI/CD pipeline.
 
 ```bash
-wikiscript doc.zih.tu-dresden.de/util/check-links.sh
+wikiscript doc.zih.tu-dresden.de/util/check-links.sh -c
 ```
 
-To check all markdown file, which may take a while and give a lot of output, use:
+##### All Files
+
+Checking the links of all markdown files takes a moment of time:
 
 ```bash
 wikiscript doc.zih.tu-dresden.de/util/check-links.sh -a

doc.zih.tu-dresden.de/mkdocs.yml

@@ -235,6 +235,8 @@ plugins:
       ignore_urls:
           - mailto:hpc-support@tu-dresden.de
           - https://jupyterhub.hpc.tu-dresden.de*
+      ignore_pages:
+          - archive/*
       skip_downloads: True
       # Toggle validating external URLs
       validate_external_urls: !ENV [ENABLED_HTMLPROOFER_EXTERNAL_URLS, False]

doc.zih.tu-dresden.de/util/check-bash-syntax.sh

@@ -52,7 +52,7 @@ elif [[ ! -z "${file}" ]]; then
   files=${file}
 else
   echo "Search in git-changed files."
-  files=`git diff --name-only "$(git merge-base HEAD "${branch}")" | grep '\.sh$' || true`
+  files=`git diff --name-only --diff-filter=d "$(git merge-base HEAD "${branch}")" | grep '\.sh$' || true`
 fi
 
 

doc.zih.tu-dresden.de/util/check-links-non-mkdocs.sh

-#!/bin/bash
-## Purpose:
-##   Checks internal links for all (git-)changed markdown files (.md) of the repository.
-##   We use the markdown-link-check script from https://github.com/tcort/markdown-link-check,
-##   which can either be installed as local or global module
-##   nmp npm install [--save-dev|-g] markdown-link-check
-##   module.
-##
-## Author: Martin.Schroschk@tu-dresden.de
-
-set -eo pipefail
-
-scriptpath=${BASH_SOURCE[0]}
-basedir=`dirname "${scriptpath}"`
-basedir=`dirname "${basedir}"`
-
-usage() {
-  cat <<-EOF
-usage: $0 [file | -a]
-If file is given, checks whether all links in it are reachable.
-If parameter -a (or --all) is given instead of the file, checks all markdown files.
-Otherwise, checks whether any changed file contains broken links.
-EOF
-}
-
-mlc=markdown-link-check
-if ! command -v ${mlc} &> /dev/null; then
-  echo "INFO: ${mlc} not found in PATH (global module)"
-  mlc=./node_modules/markdown-link-check/${mlc}
-  if ! command -v ${mlc} &> /dev/null; then
-    echo "INFO: ${mlc} not found (local module)"
-    exit 1
-  fi
-fi
-
-echo "mlc: ${mlc}"
-
-LINK_CHECK_CONFIG="${basedir}/util/link-check-config.json"
-if [[ ! -f "${LINK_CHECK_CONFIG}" ]]; then
-  echo ${LINK_CHECK_CONFIG} does not exist
-  exit 1
-fi
-
-branch="preview"
-if [[ -n "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}" ]]; then
-    branch="origin/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}"
-fi
-
-function check_single_file(){
-  theFile="$1"
-  if [[ -e "${theFile}" ]]; then
-    echo "Checking links in ${theFile}"
-    if ! ${mlc} -q -c "${LINK_CHECK_CONFIG}" -p "${theFile}"; then
-      return 1
-    fi
-  fi
-  return 0
-}
-
-function check_files(){
-any_fails=false
-echo "Check files:"
-echo "${files}"
-echo ""
-for f in ${files}; do
-  if ! check_single_file "${f}"; then
-    any_fails=true
-  fi
-done
-
-if [[ "${any_fails}" == true ]]; then
-    exit 1
-fi
-}
-
-function check_all_files(){
-files=$(git ls-tree --full-tree -r --name-only HEAD ${basedir}/ | grep '.md$' || true)
-check_files
-}
-
-function check_changed_files(){
-files=$(git diff --name-only "$(git merge-base HEAD "${branch}")" | grep '.md$' || true)
-check_files
-}
-
-if [[ $# -eq 1 ]]; then
-  case $1 in
-  help | -help | --help)
-    usage
-    exit
-  ;;
-  -a | --all)
-    check_all_files
-  ;;
-  *)
-    checkSingleFile "$1"
-  ;;
-  esac
-elif [[ $# -eq 0 ]]; then
-  check_changed_files
-else
-  usage
-fi

doc.zih.tu-dresden.de/util/check-links.sh

 #!/bin/bash
 
 # Purpose:
-#   Checks internal and external links for all markdown files (.md) of the repository.
-#   For link checking, we use html-proofer-plugin, which is configured in mkdocs.yml.
+#   Checks internal and external links for markdown files (.md) of the repository
+#   (i.e. HPC compendium). This script can check the URLs
+#     1. in all markdown files
+#     2. git-changed markdown files
+#     3. single markdown file
+#
+#   For option 1, we make use of the html-proofer-plugin, which is configured in mkdocs.yml.
 #   By default the link checker only inspects internal links. It will inspect internal and external
-#   links, when the env. variable ENABLED_HTMLPROOFER_EXTERNAL_URLS is set to true.
+#   links, since we set the env. variable ENABLED_HTMLPROOFER_EXTERNAL_URLS to true in this script.
+#
+#   For option 2 and 3, we use the markdown-link-check script from
+#   https://github.com/tcort/markdown-link-check, which can either be installed as local or global
+#   module (nmp npm install [--save-dev|-g] markdown-link-check).
+#
+#   General remarks for future me:
+#     - html-proofer-plugin is heavily customizable (ignore certain urls or pages, etc.)
+#     - html-proofer-plugin is quite slow, since it requires to build the whole compendium
+#       - html-proofer-plugin works on rendered html files
+#     - html-proofer-plugin has no option to check a single page
 
 set -eo pipefail
 
-export ENABLED_HTMLPROOFER=true
+scriptpath=${BASH_SOURCE[0]}
+basedir=`dirname "${scriptpath}"`
+basedir=`dirname "${basedir}"`
 
-function usage(){
-cat << EOF
-$0 [options]
-Check links in documentation.
+usage() {
+  cat <<-EOF
+usage: $0 [file | -a | -c]
 
 Options:
-  -a    Check internal and external links (default: internal links only)
-  -h    Show help message
+  <file>          Check links in the provided markdown file
+  -a; --all       Check links in all markdown files
+  -i; --internal   Check links in all markdown files (internal links only)
+  -c; --changed   Check links in all git-changed files
 EOF
 }
 
-while getopts ":a" option; do
-  case ${option} in
-    a)
-      export ENABLED_HTMLPROOFER_EXTERNAL_URLS=true
-      echo "Info: Checking internal and external links. Might take some time."
-      ;;
-    h)
-      usage
-      exit;;
-    \?)
-      echo "Error: Invalid option."
-      usage
-      exit;;
-  esac
-done
+mlc=markdown-link-check
+if ! command -v ${mlc} &> /dev/null; then
+  echo "INFO: ${mlc} not found in PATH (global module)"
+  mlc=./node_modules/markdown-link-check/${mlc}
+  if ! command -v ${mlc} &> /dev/null; then
+    echo "INFO: ${mlc} not found (local module)"
+    exit 1
+  fi
+fi
+
+echo "mlc: ${mlc}"
+
+LINK_CHECK_CONFIG="${basedir}/util/link-check-config.json"
+if [[ ! -f "${LINK_CHECK_CONFIG}" ]]; then
+  echo ${LINK_CHECK_CONFIG} does not exist
+  exit 1
+fi
 
-# Switch for wikiscript
-if [[ -d "doc.zih.tu-dresden.de" ]]; then
-  cd doc.zih.tu-dresden.de
+branch="preview"
+if [[ -n "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}" ]]; then
+    branch="origin/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}"
 fi
 
-mkdocs build
-rm -rf public
+function check_single_file(){
+  theFile="$1"
+  if [[ -e "${theFile}" ]]; then
+    echo "Checking links in ${theFile}"
+    if ! ${mlc} -q -c "${LINK_CHECK_CONFIG}" -p "${theFile}"; then
+      return 1
+    fi
+  else
+    echo "$1 not found"
+  fi
+
+  return 0
+}
+
+function check_files(){
+  any_fails=false
+  echo "Check files:"
+  echo "${files}"
+  echo ""
+  for f in ${files}; do
+    if ! check_single_file "${f}"; then
+      any_fails=true
+    fi
+  done
+  
+  if [[ "${any_fails}" == true ]]; then
+    exit 1
+  fi
+}
+
+function check_changed_files(){
+  files=$(git diff --name-only "$(git merge-base HEAD "${branch}")" | grep '.md$' || true)
+  check_files
+}
+
+# Make use of the html-proofer-plugin and build documentation
+function check_all_files_hpp() {
+  export ENABLED_HTMLPROOFER=true
+  # Switch for wikiscript
+  if [[ -d "doc.zih.tu-dresden.de" ]]; then
+    cd doc.zih.tu-dresden.de
+  fi
+  mkdocs build
+  rm -rf public
+}
+
+if [[ $# -ge 1 ]]; then
+  case $1 in
+  -h | help | -help | --help)
+    usage
+    exit 0
+  ;;
+  -a | --all)
+    export ENABLED_HTMLPROOFER_EXTERNAL_URLS=true
+    echo "Info: Building documentation and checking internal and external links. Might take some time."
+    check_all_files_hpp
+  ;;
+  -i | --internal)
+    export ENABLED_HTMLPROOFER_EXTERNAL_URLS=false
+    echo "Info: Building documentation and checking internal links. Might take some time."
+    check_all_files_hpp
+  ;;
+  -c | --changed)
+    echo "Info: Checking links in modified files."
+    check_changed_files
+  ;;
+  *)
+    echo "Info: Checking links in file $1"
+    check_single_file "$1"
+  ;;
+  esac
+else
+  echo -e "Remark: The script changed in Oct. 2024. You did not provide any option.
+        If you intend to check the links of all modified files, please
+        provide the option \"-c\"\n"
+  usage
+  exit 1
+fi

doc.zih.tu-dresden.de/util/pre-commit

@@ -47,7 +47,7 @@ while read -r; do
     fi
 
     echo "Checking links..."
-    if ! runInDockerDocOnly markdown-link-check "$doc_filepath"; then
+    if ! runInDocker ./doc.zih.tu-dresden.de/util/check-links.sh -c; then
       echo -e "\tFailed"
       exit_ok=no
     fi