From: Christian Heller Date: Mon, 2 Jan 2017 21:10:53 +0000 (+0100) Subject: Re-write date handling, remove dependency on filesystem lastmod dates. X-Git-Url: https://plomlompom.com/repos/%7B%7Bprefix%7D%7D/static/%7B%7Bdb.prefix%7D%7D/do_tasks?a=commitdiff_plain;h=ff5b37d8bc4cfba8f1db69a279da887accaa0296;p=redo-blog Re-write date handling, remove dependency on filesystem lastmod dates. --- diff --git a/README.md b/README.md index bfb1174..822c08d 100644 --- a/README.md +++ b/README.md @@ -29,10 +29,13 @@ You can then enter the directory and run redo there. This will generate article These files will be linked to symbolically in a directory ./public/. Some metadata files will also be generated below ./metadata/: For each article, -there will be generated a .uuid and a .intermediate file; furthermore, files for +there will be generated a .automatic_metadata (to contain an article's UUID, +checksum, and creation/modification dates) and a .intermediate file (to contain +pandoc-formatted article content like title and body); furthermore, files for data used in ./feed.xml and ./index.html will, if non-existant, be built there -and can be edited to customize the blog – namely the files url, author, uuid, -title, index.tmpl, index_snippet.tmpl, article.tmpl. +and can be edited to customize the blog – namely the files url, author, title, +index.tmpl, index_snippet.tmpl, article.tmpl. A blog-specific UUID and creation +date is stored in ./metadata/automatic_metadata recipe to remotely manage a redo blog with git ---------------------------------------------- diff --git a/processor/all.do b/processor/all.do index e448a44..3d09097 100644 --- a/processor/all.do +++ b/processor/all.do @@ -15,11 +15,11 @@ for file in "$metadata_dir"/*.intermediate; do rm "$file" fi done -for file in "$metadata_dir"/*.uuid; do +for file in "$metadata_dir"/*.automatic_metadata; do basename=$(basename "$file") if test -f "$file" && - ! test -f "${basename%.uuid}.md" && - ! test -f "${basename%.uuid}.rst"; then + ! test -f "${basename%.automatic_metadata}.md" && + ! test -f "${basename%.automatic_metadata}.rst"; then rm "$file" fi done diff --git a/processor/default.html.do b/processor/default.html.do index c9bfbdb..c44d00b 100644 --- a/processor/default.html.do +++ b/processor/default.html.do @@ -3,8 +3,8 @@ # Pull in global dependencies. . ./helpers.sh metadata_dir=metadata -uuid_file="${metadata_dir}/${1%.html}.uuid" -redo-ifchange "$uuid_file" +meta_file="${metadata_dir}/${1%.html}.automatic_metadata" +redo-ifchange "$meta_file" intermediate_file="${metadata_dir}/${1%.html}.intermediate" redo-ifchange "$intermediate_file" title_file="${metadata_dir}"/title @@ -19,9 +19,9 @@ title_plaintext=`echo "$title_html" | html2text` title_html=$(printf "%s" "$title_html" | prep_sed) title_plaintext=$(escape_html "$title_plaintext" | prep_sed) body=$(cat "$intermediate_file" | sed 1d | prep_sed) -datetime_created_unix=$(stat -c%y "${uuid_file}") -datetime_created_rfc3339=$(date -u "+%Y-%m-%dT%TZ" -d "${datetime_created_unix}") -datetime_created_friendly=$(date -u "+%Y-%m-%d %T (UTC)" -d "${datetime_created_unix}") +datetime_created_unix=$(get_creation_date_from_meta_file_seconds "$meta_file") +datetime_created_rfc3339=$(date -u "+%Y-%m-%dT%TZ" -d "@${datetime_created_unix}") +datetime_created_friendly=$(date -u "+%Y-%m-%d %T (UTC)" -d "@${datetime_created_unix}") # Put data into template. template=$(cat "$template_file") diff --git a/processor/feed.xml.do b/processor/feed.xml.do index 513d538..eed86ed 100644 --- a/processor/feed.xml.do +++ b/processor/feed.xml.do @@ -4,23 +4,23 @@ . ./helpers.sh metadata_dir=metadata author_file="$metadata_dir"/author -uuid_file="$metadata_dir"/uuid +meta_file="$metadata_dir"/automatic_metadata title_file="$metadata_dir"/title url_file="$metadata_dir"/url redo-ifchange "$url_file" redo-ifchange "$author_file" -redo-ifchange "$uuid_file" +redo-ifchange "$meta_file" redo-ifchange "$title_file" # Build some variables. XML-escape even file contents that should not contain # dangerous characters, just to avoid any XML trouble. -srcdir=`pwd` -basepath=$(get_basepath "${metadata_dir}/") -title=`read_and_escape_file "$title_file" | head -1` -author=`read_and_escape_file "$author_file" | head -1` -uuid=`read_and_escape_file "$uuid_file" | head -1` +srcdir=$(pwd) tmp_snippets_dir=.tmp_feed_snippets -feed_gen_date=$(stat -c%Y "${uuid_file}") +basepath=$(get_basepath "${metadata_dir}/") +title=$(read_and_escape_file "$title_file" | head -1) +author=$(read_and_escape_file "$author_file" | head -1) +uuid=$(get_uuid_from_meta_file "$meta_file") +feed_gen_date=$(get_creation_date_from_meta_file_seconds "$meta_file") # Write majority of feed head. cat << EOF @@ -37,16 +37,16 @@ printf "urn:uuid:%s\n" "$uuid" mkdir -p "$tmp_snippets_dir" for file in ./*.rst ./*.md; do if [ -e "$file" ]; then - uuid_file="${metadata_dir}/${file%.*}.uuid" - redo-ifchange "$uuid_file" - published=$(stat -c%Y "${uuid_file}") + meta_file="${metadata_dir}/${file%.*}.automatic_metadata" + redo-ifchange "$meta_file" + published=$(get_creation_date_from_meta_file_nanoseconds "$meta_file") snippet_file=./${metadata_dir}/"${file%.*}.feed_snippet" redo-ifchange "$snippet_file" ln -s "$srcdir/$snippet_file" "./${tmp_snippets_dir}/${published}" fi done -# Derive feed modification date from snippets. Fallback to uuid file mod date. +# Derive feed modification date from snippets. Fallback to blog creation date. n_snippet_files=`ls -1 ./${metadata_dir}/*.feed_snippet 2>/dev/null | wc -l` if [ $n_snippet_files != 0 ] then diff --git a/processor/helpers.sh b/processor/helpers.sh index 2869a29..64d1123 100644 --- a/processor/helpers.sh +++ b/processor/helpers.sh @@ -6,10 +6,32 @@ escape_html() { } read_and_escape_file() { - in=`cat "$1"` + in=$(cat "$1") escape_html "$in" } +get_uuid_from_meta_file() { + probable_uuid=$(cat "$1" | head -1) + if printf "$probable_uuid" | grep -Eq "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"; then + printf "$probable_uuid" + else + echo "Malformed UUID in meta file." >&2 + exit 1 + fi +} + +get_creation_date_from_meta_file_seconds() { + cat "$1" | sed -n '2p' | cut -d'_' -f1 +} + +get_creation_date_from_meta_file_nanoseconds() { + cat "$1" | sed -n '2p' +} + +get_lastmod_date_from_meta_file() { + cat "$1" | sed -n '4p' +} + escape_url() { out=`python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1]))' "$1"` printf "%s" "$out" diff --git a/processor/index.html.do b/processor/index.html.do index 5149cac..0310294 100644 --- a/processor/index.html.do +++ b/processor/index.html.do @@ -17,13 +17,12 @@ tmp_snippets_dir=.tmp_index_snippets mkdir -p "$tmp_snippets_dir" for file in ./*.rst ./*.md; do if [ -e "$file" ]; then - uuid_file="${metadata_dir}/${file%.*}.uuid" - redo-ifchange "$uuid_file" - published=`stat -c%y "${uuid_file}"` - published_unix=$(date -u "+%s%N" -d "${published}") + meta_file="${metadata_dir}/${file%.*}.automatic_metadata" + redo-ifchange "$meta_file" + published=$(get_creation_date_from_meta_file_nanoseconds "$meta_file") snippet_file="${metadata_dir}/${file%.*}.index_snippet" redo-ifchange "$snippet_file" - ln -s "$srcdir/$snippet_file" "./${tmp_snippets_dir}/${published_unix}" + ln -s "$srcdir/$snippet_file" "./${tmp_snippets_dir}/${published}" fi done diff --git a/processor/metadata/automatic_metadata.do b/processor/metadata/automatic_metadata.do new file mode 100644 index 0000000..44285d2 --- /dev/null +++ b/processor/metadata/automatic_metadata.do @@ -0,0 +1,6 @@ +#!/bin/sh + +if [ ! -f "$1" ]; then + uuidgen + date -u "+%s" +fi diff --git a/processor/metadata/default.automatic_metadata.do b/processor/metadata/default.automatic_metadata.do new file mode 100644 index 0000000..85f0604 --- /dev/null +++ b/processor/metadata/default.automatic_metadata.do @@ -0,0 +1,8 @@ +#!/bin/sh + +if [ ! -f "$1" ]; then + uuidgen + date -u "+%s_%N" + echo 00000000000000000000000000000000 + echo 0 +fi diff --git a/processor/metadata/default.feed_snippet.do b/processor/metadata/default.feed_snippet.do index 3321ac4..8dca59b 100644 --- a/processor/metadata/default.feed_snippet.do +++ b/processor/metadata/default.feed_snippet.do @@ -3,20 +3,20 @@ # Pull in dependencies. . ../helpers.sh src_file=$(get_source_file "$1") -uuid_file="${1%.feed_snippet}.uuid" -redo-ifchange "$uuid_file" +meta_file="${1%.feed_snippet}.automatic_metadata" +redo-ifchange "$meta_file" intermediate_file="${1%.feed_snippet}.intermediate" redo-ifchange "$intermediate_file" # Get variables, write entry. html_file=$(escape_url "${1%.feed_snippet}.html") -lastmod=`stat -c%y "$src_file"` -lastmod_rfc3339=`date -u "+%Y-%m-%dT%TZ" -d "$lastmod"` -published=`stat -c%y "$uuid_file"` -published_rfc3339=`date -u "+%Y-%m-%dT%TZ" -d "${published}"` -title=`read_and_escape_file "$intermediate_file" | head -1` -uuid=`read_and_escape_file "$uuid_file" | head -1` -body=`read_and_escape_file "$intermediate_file" | sed 1d` +lastmod=$(get_lastmod_date_from_meta_file "$meta_file") +lastmod_rfc3339=$(date -u "+%Y-%m-%dT%TZ" -d "@$lastmod") +title=$(read_and_escape_file "$intermediate_file" | head -1) +uuid=$(get_uuid_from_meta_file "$meta_file") +published_unix=$(get_creation_date_from_meta_file_seconds "$meta_file") +published_rfc3339=$(date -u "+%Y-%m-%dT%TZ" -d "@${published_unix}") +body=$(read_and_escape_file "$intermediate_file" | sed 1d) printf "\n" printf "%s\n" "$title" printf "urn:uuid:%s\n" "$uuid" diff --git a/processor/metadata/default.index_snippet.do b/processor/metadata/default.index_snippet.do index a1ff6f9..9df84fb 100644 --- a/processor/metadata/default.index_snippet.do +++ b/processor/metadata/default.index_snippet.do @@ -3,8 +3,8 @@ # Pull in dependencies. . ../helpers.sh src_file=$(get_source_file "$1") -uuid_file="${1%.index_snippet}.uuid" -redo-ifchange "$uuid_file" +meta_file="${1%.index_snippet}.automatic_metadata" +redo-ifchange "$meta_file" intermediate_file="${1%.index_snippet}.intermediate" redo-ifchange "$intermediate_file" html_file="${src_file%.*}.html" @@ -15,13 +15,13 @@ redo-ifchange "$template_file" # Build entry data. title=$(cat "$intermediate_file" | head -1 | prep_sed) link=$(escape_url "${1%.index_snippet}.html" | prep_sed) -datetime_created_unix=$(stat -c%y "${uuid_file}") -date_created=$(date -u "+%Y-%m-%d" -d "${datetime_created_unix}") +datetime_created_unix=$(get_creation_date_from_meta_file_seconds "$meta_file") +date_created_human=$(date -u "+%Y-%m-%d" -d "@${datetime_created_unix}") # Put data into template. template=$(cat "$template_file") printf "%s\n" "$template" | \ sed 's/%TITLE%/'"$title"'/g' | \ sed 's/%LINK%/'"$link"'/g' | \ -sed 's/%DATE_CREATED%/'"$date_created"'/g' | \ +sed 's/%DATE_CREATED%/'"$date_created_human"'/g' | \ tr '\a' '%' diff --git a/processor/metadata/default.intermediate.do b/processor/metadata/default.intermediate.do index a09d4b2..bf87309 100644 --- a/processor/metadata/default.intermediate.do +++ b/processor/metadata/default.intermediate.do @@ -1,11 +1,14 @@ #!/bin/sh +# Pull in dependencies. template=intermediate.pandoc_tmpl -uuidfile="${1%.intermediate}.uuid" -redo-ifchange "$uuidfile" +meta_file="${1%.intermediate}.automatic_metadata" +redo-ifchange "$meta_file" redo-ifchange "$template" mdfile="../${1%.intermediate}.md" rstfile="../${1%.intermediate}.rst" + +# Build intermediate file. if [ -f "$rstfile" ]; then redo-ifchange "$rstfile" pandoc -f rst --template="$template" --mathml -t html5 "$rstfile" --base-header-level=2 > "$3" @@ -13,3 +16,13 @@ elif [ -f "$mdfile" ]; then redo-ifchange "$mdfile" pandoc -f markdown --template="$template" --mathml -t html5 "$mdfile" --base-header-level=2 > "$3" fi + +# Update meta file if appropriate. +md5_new=$(md5sum "$3" | cut -d ' ' -f 1) +md5_old=$(cat "$meta_file" | sed -n '3p') +if [ ! "$md5_new" = "$md5_old" ]; then + new_date=$(date -u "+%s") + sed -i '1,2!d' "$meta_file" + echo "$md5_new" >> "$meta_file" + echo "$new_date" >> "$meta_file" +fi diff --git a/processor/metadata/default.uuid.do b/processor/metadata/default.uuid.do deleted file mode 100644 index 5efffc8..0000000 --- a/processor/metadata/default.uuid.do +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -if [ ! -f "$1" ]; then - uuidgen > "$1" -fi diff --git a/processor/metadata/uuid.do b/processor/metadata/uuid.do deleted file mode 100644 index ba9e919..0000000 --- a/processor/metadata/uuid.do +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -if [ ! -f "$1" ]; then - uuidgen -fi diff --git a/test.sh b/test.sh index 025ebc4..60ce2bd 100755 --- a/test.sh +++ b/test.sh @@ -1,10 +1,27 @@ #!/bin/sh -uuid_test() +uuid_pattern='[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' + +blog_meta_file_test() +{ + meta_file="$1" + printf "== %s meta file pattern match test ==\n" "$meta_file" + if cat "$meta_file" | sed -n '1p' | grep -Eq '^'"$uuid_pattern"'$' \ + && cat "$meta_file" | sed -n '2p' | grep -Eq "^[0-9]+$"; then + echo "== test SUCCESS ==" + else + echo "== test FAILURE ==" + fi +} + +article_meta_file_test() { - uuid_file="$1" - printf "== %s UUID pattern match test ==\n" "$uuid_file" - if cat "$uuid_file" | grep -Eq "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"; then + meta_file="$1" + printf "== %s meta file pattern match test ==\n" "$meta_file" + if cat "$meta_file" | sed -n '1p' | grep -Eq '^'"$uuid_pattern"'$' \ + && cat "$meta_file" | sed -n '2p' | grep -Eq "^[0-9]+_[0-9]+$" \ + && cat "$meta_file" | sed -n '3p' | grep -Eq "^[0-9a-f]{32}$" \ + && cat "$meta_file" | sed -n '2p' | grep -Eq "^[0-9]+_[0-9]+$"; then echo "== test SUCCESS ==" else echo "== test FAILURE ==" @@ -24,7 +41,7 @@ diff_test() fi } -# Set up test directory. +# Set up test directory, run file creations. expected_files_dir="test/test_files" generated_files_dir="test/test_dir" rm -rf "$generated_files_dir" @@ -38,10 +55,35 @@ cp "$working_dir/$expected_files_dir"/bar\ baz.md . redo cp "$working_dir/$expected_files_dir"/foo.rst . redo -cd "$working_dir" + +# Test file modification tracking. +update_datetime_start=$(cat "metadata/bar baz.feed_snippet" | grep '') +sleep 1 +sed -i '2d' bar\ baz.md +redo +update_datetime_after_invisible_change=$(cat "metadata/bar baz.feed_snippet" | grep '') +printf "== testing \"bar baz\"' update tag remaining unchanged with invisible source file change ==\n" +if [ "$update_datetime_start" = "$update_datetime_after_invisible_change" ]; then + echo "== test SUCCESS ==" +else + echo "== test FAILURE ==" +fi +sleep 1 +sed -i '2d' bar\ baz.md +redo +update_datetime_after_visible_change=$(cat "metadata/bar baz.feed_snippet" | grep '') +printf "== testing \"bar baz\"' update tag changing with visible source file change ==\n" +if [ "$update_datetime_start" = "$update_datetime_after_visible_change" ]; then + echo "== test FAILURE ==" +else + echo "== test SUCCESS ==" +fi +cp "$working_dir/$expected_files_dir"/bar\ baz.md . +redo # Compare metadata files. -uuid_test "$generated_files_dir""/metadata/uuid" +cd "$working_dir" +blog_meta_file_test "$generated_files_dir""/metadata/automatic_metadata" for file in "$expected_files_dir"/metadata/*; do basename=$(basename "$file") cmp_file="$generated_files_dir/metadata/$basename" @@ -53,7 +95,7 @@ for file in "$expected_files_dir"/*.html.ignoring; do basename=$(basename "$file") cmp_file="$generated_files_dir/${basename%.ignoring}" if [ ! "$file" = "$expected_files_dir""/index.html.ignoring" ]; then - uuid_test "${generated_files_dir}/metadata/${basename%.html.ignoring}.uuid" + article_meta_file_test "${generated_files_dir}/metadata/${basename%.html.ignoring}.automatic_metadata" fi generated_file="$cmp_file".ignoring cat "$cmp_file" | \