From: Christian Heller Date: Thu, 18 Nov 2021 15:08:38 +0000 (+0100) Subject: Add bullseye dumpsite setup. X-Git-Url: https://plomlompom.com/repos/%7B%7B%20web_path%20%7D%7D/static/%7B%7Bdb.prefix%7D%7D/day_todos?a=commitdiff_plain;h=6c7e3825ec99b00c3f9d68df8d16b8d68aac6229;p=config Add bullseye dumpsite setup. --- diff --git a/bullseye/apt-mark/dumpsite b/bullseye/apt-mark/dumpsite new file mode 100644 index 0000000..a87852a --- /dev/null +++ b/bullseye/apt-mark/dumpsite @@ -0,0 +1,13 @@ +wget +# for blog and zettel +pandoc +# for blog +html2text +uuid-runtime +python3 +# for url_catcher daemon +python3-venv +build-essential +python3-dev +screen +postfix diff --git a/bullseye/apt-mark/web b/bullseye/apt-mark/web new file mode 100644 index 0000000..4912b8a --- /dev/null +++ b/bullseye/apt-mark/web @@ -0,0 +1,4 @@ +nginx-light +# for SSL +certbot +python3-certbot-nginx diff --git a/bullseye/etc_files/dumpsite/etc/nginx/sites-available/dumpsite.nginx b/bullseye/etc_files/dumpsite/etc/nginx/sites-available/dumpsite.nginx new file mode 100644 index 0000000..25c2d62 --- /dev/null +++ b/bullseye/etc_files/dumpsite/etc/nginx/sites-available/dumpsite.nginx @@ -0,0 +1,28 @@ +server { + listen 443 ssl; + server_name REPLACE_fqdn_ECALPER; + ssl_certificate /etc/letsencrypt/live/REPLACE_fqdn_ECALPER/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/REPLACE_fqdn_ECALPER/privkey.pem; + root /var/www-dump/; + + location /dump/ { + autoindex on; + } + + location /geheim/ { + auth_basic "geheim geheim"; + auth_basic_user_file /var/www-dump/password_geheim; + autoindex on; + } + + location /zettel/ { + # rewrite non-suffixed filenames to .html ones + rewrite ^(/zettel/(.*/)*[^./]+)$ $1.html; + autoindex on; + } + + location /uwsgi/ { + include uwsgi_params; + uwsgi_pass 127.0.0.1:3031; + } +} diff --git a/bullseye/etc_files/dumpsite/etc/systemd/system/url_catcher.service b/bullseye/etc_files/dumpsite/etc/systemd/system/url_catcher.service new file mode 100644 index 0000000..45d079c --- /dev/null +++ b/bullseye/etc_files/dumpsite/etc/systemd/system/url_catcher.service @@ -0,0 +1,12 @@ +[Unit] +Description=url_catcher screen + +[Service] +Type=forking +User=plom +# The LC_ALL fixes submission failing on some articles. +ExecStart=/bin/sh -c 'LC_ALL=en_US.UTF8 cd ~/url-catcher && screen -d -m ./run.sh' +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/bullseye/etc_files/web/etc/nftables.conf b/bullseye/etc_files/web/etc/nftables.conf new file mode 100755 index 0000000..ec6732a --- /dev/null +++ b/bullseye/etc_files/web/etc/nftables.conf @@ -0,0 +1,22 @@ +#!/usr/sbin/nft -f + +flush ruleset + +table inet filter { + chain input { + type filter hook input priority 0; policy drop; + iif lo accept comment "accept localhost traffic" + ct state invalid drop comment "drop invalid connections" + ct state established, related accept comment "accept traffic originated from us" + tcp dport 22 accept comment "accept SSH on default port" + tcp dport 80 accept comment "accept HTTP on default port" + tcp dport 443 accept comment "accept HTTPS on default port" + ip protocol icmp icmp type echo-request accept comment "accept ICMP for pinging" + } + chain forward { + type filter hook forward priority 0; policy drop; + } + chain output { + type filter hook output priority 0; policy accept; + } +} diff --git a/bullseye/etc_files/web/etc/nginx/nginx.conf b/bullseye/etc_files/web/etc/nginx/nginx.conf new file mode 100644 index 0000000..8320425 --- /dev/null +++ b/bullseye/etc_files/web/etc/nginx/nginx.conf @@ -0,0 +1,38 @@ +# system integration +user www-data; +worker_processes auto; +pid /run/nginx.pid; +include /etc/nginx/modules-enabled/*.conf; + +# is expected even if empty +events { +} + +http { + # define content-type headers + include /etc/nginx/mime.types; + charset utf-8; + + # Some standard optimizations, i.e. Debian default. Explained in + # + # Not that I understand it all … + sendfile on; + tcp_nopush on; + tcp_nodelay on; + + # logging deactivated due to GDPR + #access_log /var/log/nginx/access.log; + #error_log /var/log/nginx/error.log; + access_log off; + error_log off; + + # virtual hosts: sites-enabled is the Debian way, conf.d the NGINX default + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-enabled/*; + + # Redirect all HTTP requests to HTTPS. + server { + listen 80; + return 301 https://$host$request_uri; + } +} diff --git a/bullseye/setup_scripts/setup_dumpsite.sh b/bullseye/setup_scripts/setup_dumpsite.sh new file mode 100755 index 0000000..4d1a14e --- /dev/null +++ b/bullseye/setup_scripts/setup_dumpsite.sh @@ -0,0 +1,102 @@ +#!/bin/sh +set -e + +if [ "$#" -ne 4 ]; then + echo 'Need domain name and mail and old server and repos source ("local" or "remote"?).' + false +fi +if [ ! "$4" = "local" ] && [ ! "$4" = "remote" ]; then + echo "Need legal repo source name." + false +fi +domain="$1" +mail="$2" +old_server="$3" +repos_source="$4" + +read -p"Only continue if hostname is not domain of url_catcher's target mail address, else abort!" ignore + +# Install configs, set up firewall. +echo "postfix postfix/main_mailer_type string 'Internet Site'" | debconf-set-selections +echo "postfix postfix/mailname string $(hostname -f)" | debconf-set-selections +config_tree_prefix="${HOME}/config/bullseye" +./install_for_target.sh web dumpsite +./copy_dirtree.sh "${config_tree_prefix}/etc_files" "" web dumpsite +nft -f /etc/nftables.conf + +# Set up letsencrypt certificate. TODO: Is it auto-renewed? +ln -sf /etc/nginx/sites-available/default /etc/nginx/sites-enabled/default +certbot --nginx --agree-tos --redirect --no-eff-email -m "${mail}" -d "${domain}" +rm /etc/nginx/sites-enabled/default + +# Set up connection to old dump server. +cp "${config_tree_prefix}/setup_scripts/prepare_to_meet_server.sh" /home/plom/ +chown plom:plom /home/plom/prepare_to_meet_server.sh +su -lc "./prepare_to_meet_server.sh ${old_server}" plom +read -p'Hit Enter when you are done.' ignore +rm /home/plom/prepare_to_meet_server.sh + +# Set up dump dirs. +mkdir /var/www-dump +chown plom:plom /var/www-dump +dump_dir=dump +geheim_dir=geheim +su -lc "ln -s /home/plom/${dump_dir} /var/www-dump/${dump_dir}" plom +su -lc "ln -s /home/plom/${geheim_dir} /var/www-dump/${geheim_dir}" plom +cp "${config_tree_prefix}/setup_scripts/mirror_dir.sh" /home/plom/ +su -lc "./mirror_dir.sh ${old_server} /home/plom/${dump_dir}" plom +su -lc "./mirror_dir.sh ${old_server} /home/plom/${geheim_dir}" plom +su -lc "scp plom@${old_server}:/var/www-dump/password_geheim ~" plom +mv /home/plom/password_geheim /var/www-dump/password_geheim +rm /home/plom/mirror_dir.sh + +# Set up redo. +wget http://news.dieweltistgarnichtso.net/bin/archives/redo-sh.tar.gz +tar -moxzf redo-sh.tar.gz -C /usr/local + +# Set up zettel. +su -lc "git clone --mirror ${old_server}:zettel.git" plom +cp "${config_tree_prefix}/other_files/zettel_hook_post-receive" /home/plom/zettel.git/hooks/post-receive +su -lc "git clone ~/zettel.git && cd zettel && redo" plom +su -lc "ln -s /home/plom/zettel /var/www-dump/zettel" plom +# NOTE: Locally, to update content, clone zettel.git, not zettel. + +# Set up redo blog. +su -lc "git clone --mirror ${old_server}:blog.git" plom +cp "${config_tree_prefix}/other_files/blog_hook_post-receive" /home/plom/blog.git/hooks/post-receive +su -lc "git clone ~/blog.git" plom +# TODO: set up like plomlombot repo (with post-recieve hook)? +if [ "$repos_source" = "local"]; then + su -lc "git clone /var/repos/redo-blog" plom +else + su -lc "git clone https://plomlompom.com/repos/clone/redo-blog" plom +fi +su -lc "cd redo-blog && ./add_dir.sh ~/blog" plom +su -lc "cd blog && redo" plom +su -lc "ln -s /home/plom/blog/public /var/www-dump/blog" plom +# NOTE: Locally, to update content, clone blog.git, not blog. + +# Set up url catcher. +# TODO: set up like plomlombot repo (with post-recieve hook)? +if [ "$repos_source" = "local"]; then + su -lc "git clone /var/repos/url-catcher" plom +else + su -lc "git clone https://plomlompom.com/repos/clone/url-catcher" plom +fi +su -lc "cd url-catcher && ln -s ../blog/captchas/linkable/ captchas" plom +cp "${config_tree_prefix}/other_files/url-catcher_customizations.json" /home/plom/url-catcher/customizations.json +systemctl enable url_catcher.service +service url_catcher start +cp "${config_tree_prefix}/setup_scripts/mirror_dir.sh" /home/plom/ +su -lc "./mirror_dir.sh ${old_server} /home/plom/url-catcher/ips" plom +su -lc "./mirror_dir.sh ${old_server} /home/plom/url-catcher/lists" plom +rm /home/plom/mirror_dir.sh + +# Set up index.html +cp "${config_tree_prefix}/other_files/dumpsite_index.html" /var/www-dump/index.html + +# Prepare NGINX. +sed -i "s/REPLACE_fqdn_ECALPER/${domain}/g" /etc/nginx/sites-available/dumpsite.nginx +ln -s /etc/nginx/sites-available/dumpsite.nginx /etc/nginx/sites-enabled/dumpsite.nginx + +service nginx restart