Merge pull request 'Modify to better match https://git.fsfe.org/FSFE/fsfe-website/pulls/4371' (#7) from status-improvements into master

Reviewed-on: https://git.fsfe.org/FSFE/fsfe-local-build/pulls/7

.gitignore

@@ -1,3 +1,14 @@
 config.cfg
 lighttpd-fsfe.conf
 pywebserver-fsfe.log
+fsfe.org
+fsfe-website
+
+lighttpd-fsfe*.log
+php-errors.log
+
+# ignore everything in fsfe.org/ and status/ but keep the folders
+fsfe.org/*
+!fsfe.org/.gitkeep
+status/*
+!status/.gitkeep

Dockerfile

@@ -0,0 +1,22 @@
+FROM alpine:3.9
+
+RUN apk add --no-cache \
+    bash \
+    bash-completion \
+    coreutils \
+    diffutils \
+    findutils \
+    inotify-tools \
+    libxml2-utils \
+    libxslt \
+    make \
+    procps \
+    python3 \
+    rsync
+
+RUN mkdir -p /fsfe-local-build/fsfe.org
+
+EXPOSE 8000
+
+CMD sh /fsfe-local-build/docker-cmd.sh
+

README.md

@@ -1,5 +1,49 @@
-# FSFE Local Build
+# FSFE Website Local Build
 
-This is a collection of scripts and configuration files necessary to build fsfe.org websites locally. 
+This is a collection of scripts, tools and configuration files necessary to build fsfe.org websites locally. 
 
 Full instructions can be found in FSFE's wiki: https://wiki.fsfe.org/TechDocs/Mainpage/BuildLocally
+
+## Docker based development environment
+
+For convenience local development you may want to use the docker contaienr provided by this repo.
+
+The container contains everything you need to work on the website including serving a preview, building the static pages and the CSS files.
+
+### Requirements
+
+* Docker https://docs.docker.com/install/
+* Docker Compose https://docs.docker.com/compose/install/
+* Patience (for the website build)
+
+### Website Dev Instructions
+
+#### Initial set up
+
+Just run the following command for the initial setup:
+
+```
+./docker-setup.sh
+```
+
+⚠ The command may run some hours, since it does the initial full build of the website.
+
+#### Development
+
+Spin up the container (⚠ takes some minutes):
+
+    docker-compose up
+
+The website should now be available on your machine under http://127.0.0.1:8000/.
+
+Build a single page after changes (e.g. `index.de.xhtml`):
+
+```
+docker exec \
+    --workdir /fsfe-local-build/fsfe.org \
+    fsfe-local-build \
+    bash ../fsfe-preview.sh ../fsfe-website/index.de.xhtml
+```
+
+Style modifications in `fsfe-website/look` trigger a re build of the styles.
+After modifications a page reload shoud show the changes.

config.cfg.docker

@@ -0,0 +1,32 @@
+### BUILD DIRECTORIES ###
+
+# Cloned Git directory (repository with XHTML source files)
+LOC_trunk=/fsfe-local-build/fsfe-website
+
+# Directory in which the result of the build process (html files) will be saved
+LOC_out=/fsfe-local-build/fsfe.org
+
+
+### LOCAL WEBSERVER ###
+
+# Choose your local webserver. 
+# Default is "python" which is python3's built-in http.server module (SimpleHTTP) that is available on most distributions
+# Use "lighttpd" if you want to use the Lighttpd webserver. This has to be configured manually (see guide)
+HTTPD=lighttpd
+
+
+### ADVANCED ###
+
+# Directory of the secondary Git directory with which you made a first full build. 
+# Avoids having some generated files in your clean trunk. 
+# Default: Leave empty if you just want to use one Git source directory (recommended)
+LOC_trunk_dev=
+
+# Choose on which port the webserver runs. This has no effect for "lighttpd". 
+HTTPD_port=5080
+
+# If you use other webservers like "lighttpd", this is the path to its config file 
+# Default: points to a lighttpd config file in the same directory as the script
+# Has no effect if you use the "python" webserver
+HTTPD_conf=$ROOT/lighttpd-fsfe.conf
+

config.cfg.sample

@@ -1,15 +1,10 @@
 ### BUILD DIRECTORIES ###
 
-# SVN trunk directory (repository with XHTML source files)
+# Cloned Git directory (repository with XHTML source files)
-LOC_trunk=~/subversion/fsfe/fsfe-web/trunk
+LOC_trunk=~/Web/FSFE/fsfe-website
 
 # Directory in which the result of the build process (html files) will be saved
-LOC_out=~/subversion/fsfe/local-build/fsfe.org
+LOC_out=~/Web/FSFE/fsfe-local-build/fsfe.org
-
-# Directory of the secondary SVN directory with which you made a first full build. 
-# Avoids having some generated files in your clean trunk. 
-# Default: Leave empty if you just want to use one SVN source directory (recommended)
-LOC_trunk_dev=
 
 
 ### LOCAL WEBSERVER ###
@@ -19,6 +14,14 @@ LOC_trunk_dev=
 # Use "lighttpd" if you want to use the Lighttpd webserver. This has to be configured manually (see guide)
 HTTPD=python
 
+
+### ADVANCED ###
+
+# Directory of the secondary Git directory with which you made a first full build. 
+# Avoids having some generated files in your clean trunk. 
+# Default: Leave empty if you just want to use one Git source directory (recommended)
+LOC_trunk_dev=
+
 # Choose on which port the webserver runs. This has no effect for "lighttpd". 
 HTTPD_port=5080
 

docker-cmd.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+########################################################################
+#  Copyright (C) 2019 Michael Weimann <mweimann@fsfe.org>
+########################################################################
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+########################################################################
+#
+#  This script is executed in docker CMD.
+#
+#######################################################################
+
+# Run the preview server
+cd /fsfe-local-build/fsfe.org
+python3 -m http.server
+

docker-compose.yml

@@ -0,0 +1,12 @@
+version: '3'
+
+services:
+  fsfe-local-build:
+    build: .
+    image: fsfe/local-build:1.0
+    container_name: fsfe-local-build
+    ports:
+      - 8000:8000
+    volumes:
+      - .:/fsfe-local-build
+

docker-setup.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+########################################################################
+#  Copyright (C) 2019 Michael Weimann <mweimann@fsfe.org>
+########################################################################
+#  
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#  
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#  
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#  
+########################################################################
+#  
+#  This script runs the docker dev setup.
+#  
+#######################################################################
+
+echo -e "\e[96m→ Cloning the website repo\e[0m"
+git clone git@git.fsfe.org:FSFE/fsfe-website.git
+echo ""
+
+echo -e "\e[96m→ Setting up the config file\e[0m"
+cp config.cfg.docker config.cfg
+echo ""
+
+echo -e "\e[96m→ Linking the look directory\e[0m"
+ln -s ../fsfe-website/look fsfe.org/look
+echo ""
+
+echo -e "\e[96m→ Starting the container\e[0m"
+docker-compose up --build -d
+echo ""
+
+echo -e "\e[96m→ Initial full build\e[0m"
+echo -e "\e[33m⚠ This may take some hours\e[0m"
+docker exec fsfe-local-build \
+ bash /fsfe-local-build/fsfe-website/build/build_main.sh \
+ build_into /fsfe-local-build/fsfe.org/ \
+ --statusdir /fsfe-local-build/status/ \
+ || true
+echo ""
+
+echo -e "\e[96m→ Stopping the container\e[0m"
+docker-compose stop
+echo ""
+

fsfe-build.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# =============================================================================
+# Start a build of the fsfe-website checkout in the current working directory
+# =============================================================================
+# SPDX-FileCopyrightText: © 2020, 2021 Reinhard Müller <reinhard@fsfe.org>
+# SPDX-License-Identifier: WTFPL
+# =============================================================================
+
+set -e
+
+# Find out fsfe-website directory
+gitdir="$(git rev-parse --show-toplevel)"
+
+# Find out fsfe-local-build directory
+builddir="$(dirname "$(readlink -f "$0")")"
+
+# Start the build
+echo "Building from ${gitdir} into ${builddir}/fsfe.org"
+case "$1" in
+  --full|-f)
+    "${gitdir}/build/build_main.sh" build_into "${builddir}/fsfe.org" --statusdir "${builddir}/fsfe.org/status.fsfe.org/fsfe.org"
+    ;;
+  *)
+    "${gitdir}/build/build_main.sh" build_run "${builddir}/fsfe.org" --statusdir "${builddir}/fsfe.org/status.fsfe.org/fsfe.org"
+    ;;
+esac

fsfe-preview.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 ########################################################################
 #  Copyright (C) 2016 Max Mehl <max.mehl@fsfe.org>
 ########################################################################
@@ -25,6 +25,38 @@
 #  
 #######################################################################
 
+# Coloured and tagged output
+function echo_ERR {
+  echo -e '\033[0;31m'"[ERROR] $1"'\033[0m'
+}
+function echo_SUC {
+  echo -e '\033[0;32m'"[SUCCESS] $1"'\033[0m'
+}
+function echo_WARN {
+  echo -e '\033[0;33m'"[WARNING] $1"'\033[0m'
+}
+function echo_INFO {
+  echo -e '\033[0;37m'"[INFO] $1"'\033[0m'
+}
+
+# Check dependencies (stolen from build_main.sh)
+deperrors=''
+for depend in realpath rsync xsltproc xmllint sed find egrep grep wc make tee date iconv; do
+  if ! which "$depend" >/dev/null 2>&1; then
+    deperrors="$depend $deperrors"
+  fi
+done
+if [ -n "$deperrors" ]; then
+  printf '\033[1;31m'
+  cat <<-EOF
+	The build script depends on some other programs to function.
+	Not all of those programs could be located on your system.
+	Please use your package manager to install the following programs:
+	EOF
+  printf '\n\033[0;31m%s\n' "$deperrors"
+  exit 1
+fi >>/dev/stderr
+
 ROOT=$(dirname "$(readlink -f "$0")")
 if [ ! -e "$ROOT"/config.cfg ]; then echo "Missing config.cfg file. Edit and rename config.cfg.sample"; exit 1; fi
 source "$ROOT"/config.cfg
@@ -32,17 +64,20 @@ source "$ROOT"/config.cfg
 # Help listing
 if [ "$1" = "" ]; then
   self=$(basename $0) 
-  echo "No parameters or variables given!"
+  echo_WARN "No parameters or variables given!"
   echo
   echo "Usage: "
   echo
   echo "$self file.en.xhtml"
-  echo "   -- build single XHTML file from your source to the local web directory"
+  echo "   -- build single XHTML file from your source to the local web "
-  echo "      the destination will be on the same relative level of the selected XHTML file"
+  echo "      directory"
+  echo "      The destination will be on the same relative level of the "
+  echo "      selected XHTML file"
   echo "      If not a XHTML file, it will be copied to the web directory"
   echo
   echo "$self --copy picture.png look/style.css"
-  echo "   -- Just copies several files to the local web directory without even trying to build them"
+  echo "   -- Just copies several files to the local web directory without "
+  echo "      even trying to build them"
   exit 0
 fi
 
@@ -54,84 +89,122 @@ LOC_trunk_dev=$(echo $LOC_trunk_dev | sed 's|/$||')
 # Pure copy
 if [ "$1" = "--copy" ]; then
   numargs=$(($# - 1))
-  echo "[INFO] Starting a plain mass copy with $numargs files..."
+  echo_INFO "Starting a plain mass copy with $numargs file(s)..."
   for ((i=1; i <= $numargs; i++)); do
     SRC_full=$(realpath $2)
     DST_full=$(echo $SRC_full | sed -E "s|$LOC_trunk|$LOC_out|")
-    echo "[INFO] Copying $SRC_full to $DST_full..."
+    echo_INFO "Copying $SRC_full to $DST_full..."
-    cp -R $SRC_full $DST_full
+    if [ -d "$SRC_full" ] && [ -e "$DST_full" ]; then
+      rm -rf "$DST_full"
+      cp -R $SRC_full $DST_full
+    else
+      cp -R $SRC_full $DST_full
+    fi
+    
     shift
   done
-  echo "[SUCCESS] Copying finished. Files should be visible via the local webserver now."
+  echo_SUC "Copying finished. Files should be visible via the local webserver now."
   exit 0
 fi
 
-SRC_rel=$1  # e.g. index.en.xhtml
+# Run through number of files given as arguments
-SRC_full=$(realpath $SRC_rel) # Full path to source file
+numargs=$#
-
+echo_INFO "Building $numargs file(s)"
-# CHECK whether the source file is located in LOC_trunk
+if [ $numargs -gt 1 ]; then
-if [ ! $(echo $SRC_full | grep "$LOC_trunk") ]; then
+  echo 
-  echo "[ERROR] Given source file is not part of \$LOC_trunk (currently set to \"$LOC_trunk\"). Please only build files which reside in or below this directory. Aborting now."
-  exit 1
 fi
+for ((i=1; i <= $numargs; i++)); do
+  SRC_rel=$1  # e.g. index.en.xhtml
+  SRC_full=$(realpath $SRC_rel) # Full path to source file
 
-# TEST if a DEV SVN directory is used. If yes, copy clean file to this Dir
+  # CHECK whether the source file is located in LOC_trunk
-if [ "$LOC_trunk_dev" != "" ]; then
+  if [ ! $(echo $SRC_full | grep "$LOC_trunk") ]; then
-  SRC_full_dev=$(echo $SRC_full | sed -E "s|$LOC_trunk|$LOC_trunk_dev|")
+    echo_ERR "Given source file is not part of \$LOC_trunk (currently set to \"$LOC_trunk\"). Please only build files which reside in or below this directory. Aborting now."
-  echo "[INFO] A \"dirty\" Development SVN directory ($LOC_trunk_dev) is being used. Copy file from clean SVN directory ($LOC_trunk) to Development directory..."
+    exit 1
-  cp $SRC_full $SRC_full_dev  # copy file from clean SVN to dev/dirty SVN
-  SRC_full=$SRC_full_dev
-  LOC_trunk=$LOC_trunk_dev
-fi
-
-echo "[INFO] Using file $SRC_full as source..."
-
-### TRANSFORM XHTML file path
-DST_full=$(echo $SRC_full | sed -E "s|$LOC_trunk|$LOC_out|")  # replace SVN directory by build/HTTP destination
-
-### BUILD/COPY FILE depending on file extension
-ftype=${SRC_full##*.} # get extension of file
-ftype=$(echo "$ftype" | tr '[:upper:]' '[:lower:]')
-ftype_build="xhtml" # file types which have to be built
-
-if [[ "$ftype" == @($ftype_build) ]]; then  # XHTML file
-  DST_full=$(echo $DST_full | sed -E "s/$ftype_build/html/")  # Replace xhtml by html
-  echo "[INFO] XHTML file detected. Going to build into $DST_full ..."
-  
-  xmllint --noout "$SRC_full" || (echo; echo "[ERROR] Syntax error in $SRC_full. Error message above. Please fix!"; exit 1)
-  if [[ "$?" != "0" ]]; then exit 1; fi
-  
-  $LOC_trunk/build/build_main.sh process_file $SRC_full > $DST_full
-
-else # just copy file
-  echo "[INFO] File type ($ftype) is detected as not to be built. Just copying it to $DST_full ..."
-  cp $SRC_full $DST_full
-fi
-
-
-### START WEBSERVER if necessary
-if [ "$HTTPD" == "python" ]; then   # python
-  if [ ! $(pgrep -f "pywebserver-fsfe.py $LOC_out") ]; then
-    echo "[INFO] Starting $HTTPD webserver"
-    nohup "$ROOT/pywebserver-fsfe.py" "$LOC_out" "$HTTPD_port" > "$ROOT"/pywebserver-fsfe.log 2>&1 &
-  else
-    echo "[INFO] $HTTPD webserver already seems to be running."
   fi
-elif [ "$HTTPD" == "lighttpd" ]; then
-  if [ ! $(pgrep lighttpd) ]; then   # lighttpd
-    echo "[INFO] Starting $HTTPD webserver"
-    /usr/sbin/lighttpd -f "$HTTPD_conf"
-  else
-    echo "[INFO] $HTTPD webserver already seems to be running."
-  fi
-elif [ "$HTTPD" == "" ]; then   # empty variable
-  echo "[ERROR] No webserver specified. Please define variable HTTPD in config.cfg. Aborting now."
-  exit 1
-else    # something custom
-  echo "[WARNING] Unknown webserver defined. Please check variable HTTPD in config.cfg." 
-fi
 
-### SHOW RESULTS
+  # CHECK whether the source file is a directory (only --copy allowed)
-echo
+  if [ -d "$SRC_full" ]; then
-DST_rel=$(echo $DST_full | sed "s|$LOC_out||") # relative path
+    echo_ERR "The given file \"$SRC_full\" is a directory. Those can only be copied by using --copy as a parameter."
-echo "[SUCCESS] Finished. File can be viewed at http://localhost:5080$DST_rel"
+    echo_ERR "NOTE: Using --copy won't build potential .xhtml files inside a directory. If you have this case, please first copy an empty directory and then build the .xhtml files inside them."
+    exit 1
+  fi
+
+  # TEST if a DEV Git directory is used. If yes, copy clean file to this Dir
+  if [ "$LOC_trunk_dev" != "" ]; then
+    SRC_full_dev=$(echo $SRC_full | sed -E "s|$LOC_trunk|$LOC_trunk_dev|")
+    echo_INFO "A \"dirty\" Development Git directory ($LOC_trunk_dev) is being used. Copy file from clean Git directory ($LOC_trunk) to Development directory..."
+    cp $SRC_full $SRC_full_dev  # copy file from clean Git to dev/dirty Git
+    SRC_full=$SRC_full_dev
+    LOC_trunk=$LOC_trunk_dev
+  fi
+
+  echo_INFO "Using file $SRC_full as source..."
+
+  ### TRANSFORM XHTML file path
+  DST_full=$(echo $SRC_full | sed -E "s|$LOC_trunk|$LOC_out|")  # replace Git directory by build/HTTP destination
+
+  ### Create destination directory if necessary
+  DST_full_dir="$(dirname $DST_full)"
+  if [ ! -e "$DST_full_dir" ]; then
+    echo_INFO "Creating destination directory $DST_full_dir"
+    mkdir -p "$DST_full_dir"
+  fi
+
+  ### BUILD/COPY FILE depending on file extension
+  ftype=${SRC_full##*.} # get extension of file
+  ftype=$(echo "$ftype" | tr '[:upper:]' '[:lower:]')
+  ftype_build="xhtml" # file types which have to be built
+
+  if [[ "$ftype" == @($ftype_build) ]]; then  # XHTML file
+    DST_full=$(echo $DST_full | sed -E "s/$ftype_build/html/")  # Replace xhtml by html
+    echo_INFO "XHTML file detected. Going to build into $DST_full ..."
+    
+    xmllint --noout "$SRC_full" || (echo; echo "Syntax error in $SRC_full. Error message above. Please fix!"; exit 1)
+    if [[ "$?" != "0" ]]; then exit 1; fi
+    
+    $LOC_trunk/build/build_main.sh process_file $SRC_full > $DST_full
+
+  else # just copy file
+    echo_INFO "File type ($ftype) is detected as not to be built. Just linking it to $DST_full ..."
+    ln -sf $SRC_full $DST_full
+  fi
+
+
+  ### START WEBSERVER if necessary
+  if [ "$HTTPD" == "python" ]; then   # python
+    if [ ! $(pgrep -f "pywebserver-fsfe.py $LOC_out") ]; then
+      echo_INFO "Starting $HTTPD webserver"
+      nohup "$ROOT/pywebserver-fsfe.py" "$LOC_out" "$HTTPD_port" > "$ROOT"/pywebserver-fsfe.log 2>&1 &
+    else
+      echo_INFO "$HTTPD webserver already seems to be running."
+    fi
+  elif [ "$HTTPD" == "lighttpd" ]; then
+    if [ ! $(pgrep -f "lighttpd.*${HTTPD_conf}") ]; then   # lighttpd
+      echo_INFO "Starting $HTTPD webserver"
+      /usr/sbin/lighttpd -f "$HTTPD_conf"
+    else
+      echo_INFO "$HTTPD webserver already seems to be running."
+    fi
+  elif [ "$HTTPD" == "" ]; then   # empty variable
+    echo_ERR "No webserver specified. Please define variable HTTPD in config.cfg. Aborting now."
+    exit 1
+  else    # something custom
+    echo_WARN "Unknown webserver defined. Probably you won't be able to see the file in your browser. Building it anyway. Please check variable \"HTTPD\" in config.cfg." 
+  fi
+
+  ### SHOW RESULTS
+  DST_rel=$(echo $DST_full | sed "s|$LOC_out||") # relative path
+  echo_SUC "Finished. File can be viewed at http://localhost:$HTTPD_port$DST_rel"
+  
+  if [ $numargs -gt 1 ]; then
+  echo "++++++++++++++++++++++++++++++++++"
+  fi
+  
+  shift
+done
+
+if [ $numargs -gt 1 ]; then
+  echo 
+  echo_SUC "$numargs files have been processed. Preview links to each file can be found individually above."
+fi

fsfe.org/.gitignore

@@ -1,2 +0,0 @@
-*
-!.gitignore

lighttpd-fsfe.conf.sample

@@ -1,14 +1,19 @@
-server.modules = ( "mod_access" )
+server.modules = ( "mod_access", "mod_accesslog" )
 $HTTP["remoteip"] !~ "127.0.0.1" {
   url.access-deny = ("")    # prevent hosting to the network
 }
 
 # change port and document-root accordingly
 server.port                     = 5080
-server.document-root            = "/home/USER/subversion/fsfe/local-build/fsfe.org"
+server.document-root            = "/home/USER/Web/FSFE/fsfe-local-build/fsfe.org"
-server.errorlog                 = "/dev/stdout"
+server.errorlog                 = "/home/USER/Web/FSFE/fsfe-local-build/lighttpd-fsfe.error.log"
+accesslog.filename              = "/home/USER/Web/FSFE/fsfe-local-build/lighttpd-fsfe.access.log"
 server.dir-listing              = "enable"
 dir-listing.encoding            = "utf-8"
 index-file.names                = ("index.html", "index.en.html")
 
 include_shell "./create-mime.assign.pl"
+
+# Uncomment if your would like to enable php support
+# server.modules += ( "mod_cgi" )
+# cgi.assign = ( ".php" => "/usr/bin/php-cgi" )

status/.gitignore

@@ -1,2 +0,0 @@
-*
-!.gitignore