From: Christian Heller <c.heller@plomlompom.de>
Date: Mon, 2 Jan 2017 21:10:53 +0000 (+0100)
Subject: Re-write date handling, remove dependency on filesystem lastmod dates.
X-Git-Url: https://plomlompom.com/repos/%7B%7B%20web_path%20%7D%7D/%7B%7Bdb.prefix%7D%7D/static/process?a=commitdiff_plain;h=ff5b37d8bc4cfba8f1db69a279da887accaa0296;p=redo-blog

Re-write date handling, remove dependency on filesystem lastmod dates.
---

diff --git a/README.md b/README.md
index bfb1174..822c08d 100644
--- a/README.md
+++ b/README.md
@@ -29,10 +29,13 @@ You can then enter the directory and run redo there. This will generate article
 These files will be linked to symbolically in a directory ./public/.
 
 Some metadata files will also be generated below ./metadata/: For each article,
-there will be generated a .uuid and a .intermediate file; furthermore, files for
+there will be generated a .automatic_metadata (to contain an article's UUID,
+checksum, and creation/modification dates) and a .intermediate file (to contain
+pandoc-formatted article content like title and body); furthermore, files for
 data used in ./feed.xml and ./index.html will, if non-existant, be built there
-and can be edited to customize the blog – namely the files url, author, uuid,
-title, index.tmpl, index_snippet.tmpl, article.tmpl.
+and can be edited to customize the blog – namely the files url, author, title,
+index.tmpl, index_snippet.tmpl, article.tmpl. A blog-specific UUID and creation
+date is stored in ./metadata/automatic_metadata
 
 recipe to remotely manage a redo blog with git
 ----------------------------------------------
diff --git a/processor/all.do b/processor/all.do
index e448a44..3d09097 100644
--- a/processor/all.do
+++ b/processor/all.do
@@ -15,11 +15,11 @@ for file in "$metadata_dir"/*.intermediate; do
     rm "$file"
   fi
 done
-for file in "$metadata_dir"/*.uuid; do
+for file in "$metadata_dir"/*.automatic_metadata; do
   basename=$(basename "$file")
   if   test -f "$file" &&
-     ! test -f "${basename%.uuid}.md" &&
-     ! test -f "${basename%.uuid}.rst"; then
+     ! test -f "${basename%.automatic_metadata}.md" &&
+     ! test -f "${basename%.automatic_metadata}.rst"; then
     rm "$file"
   fi
 done
diff --git a/processor/default.html.do b/processor/default.html.do
index c9bfbdb..c44d00b 100644
--- a/processor/default.html.do
+++ b/processor/default.html.do
@@ -3,8 +3,8 @@
 # Pull in global dependencies.
 . ./helpers.sh
 metadata_dir=metadata
-uuid_file="${metadata_dir}/${1%.html}.uuid"
-redo-ifchange "$uuid_file"
+meta_file="${metadata_dir}/${1%.html}.automatic_metadata"
+redo-ifchange "$meta_file"
 intermediate_file="${metadata_dir}/${1%.html}.intermediate"
 redo-ifchange "$intermediate_file"
 title_file="${metadata_dir}"/title
@@ -19,9 +19,9 @@ title_plaintext=`echo "$title_html" | html2text`
 title_html=$(printf "%s" "$title_html" | prep_sed)
 title_plaintext=$(escape_html "$title_plaintext" | prep_sed)
 body=$(cat "$intermediate_file" | sed 1d | prep_sed)
-datetime_created_unix=$(stat -c%y "${uuid_file}")
-datetime_created_rfc3339=$(date -u "+%Y-%m-%dT%TZ" -d "${datetime_created_unix}")
-datetime_created_friendly=$(date -u "+%Y-%m-%d %T (UTC)" -d "${datetime_created_unix}")
+datetime_created_unix=$(get_creation_date_from_meta_file_seconds "$meta_file")
+datetime_created_rfc3339=$(date -u "+%Y-%m-%dT%TZ" -d "@${datetime_created_unix}")
+datetime_created_friendly=$(date -u "+%Y-%m-%d %T (UTC)" -d "@${datetime_created_unix}")
 
 # Put data into template.
 template=$(cat "$template_file")
diff --git a/processor/feed.xml.do b/processor/feed.xml.do
index 513d538..eed86ed 100644
--- a/processor/feed.xml.do
+++ b/processor/feed.xml.do
@@ -4,23 +4,23 @@
 . ./helpers.sh
 metadata_dir=metadata
 author_file="$metadata_dir"/author
-uuid_file="$metadata_dir"/uuid
+meta_file="$metadata_dir"/automatic_metadata
 title_file="$metadata_dir"/title
 url_file="$metadata_dir"/url
 redo-ifchange "$url_file"
 redo-ifchange "$author_file"
-redo-ifchange "$uuid_file"
+redo-ifchange "$meta_file"
 redo-ifchange "$title_file"
 
 # Build some variables. XML-escape even file contents that should not contain
 # dangerous characters, just to avoid any XML trouble.
-srcdir=`pwd`
-basepath=$(get_basepath "${metadata_dir}/")
-title=`read_and_escape_file "$title_file" | head -1`
-author=`read_and_escape_file "$author_file" | head -1`
-uuid=`read_and_escape_file "$uuid_file" | head -1`
+srcdir=$(pwd)
 tmp_snippets_dir=.tmp_feed_snippets
-feed_gen_date=$(stat -c%Y "${uuid_file}")
+basepath=$(get_basepath "${metadata_dir}/")
+title=$(read_and_escape_file "$title_file" | head -1)
+author=$(read_and_escape_file "$author_file" | head -1)
+uuid=$(get_uuid_from_meta_file "$meta_file")
+feed_gen_date=$(get_creation_date_from_meta_file_seconds "$meta_file")
 
 # Write majority of feed head.
 cat << EOF
@@ -37,16 +37,16 @@ printf "<id>urn:uuid:%s</id>\n" "$uuid"
 mkdir -p "$tmp_snippets_dir"
 for file in ./*.rst ./*.md; do
   if [ -e "$file" ]; then
-    uuid_file="${metadata_dir}/${file%.*}.uuid"
-    redo-ifchange "$uuid_file"
-    published=$(stat -c%Y "${uuid_file}")
+    meta_file="${metadata_dir}/${file%.*}.automatic_metadata"
+    redo-ifchange "$meta_file"
+    published=$(get_creation_date_from_meta_file_nanoseconds "$meta_file")
     snippet_file=./${metadata_dir}/"${file%.*}.feed_snippet"
     redo-ifchange "$snippet_file"
     ln -s "$srcdir/$snippet_file" "./${tmp_snippets_dir}/${published}"
   fi
 done
 
-# Derive feed modification date from snippets. Fallback to uuid file mod date.
+# Derive feed modification date from snippets. Fallback to blog creation date.
 n_snippet_files=`ls -1 ./${metadata_dir}/*.feed_snippet 2>/dev/null | wc -l`
 if [ $n_snippet_files != 0 ]
 then
diff --git a/processor/helpers.sh b/processor/helpers.sh
index 2869a29..64d1123 100644
--- a/processor/helpers.sh
+++ b/processor/helpers.sh
@@ -6,10 +6,32 @@ escape_html() {
 }
 
 read_and_escape_file() {
-  in=`cat "$1"`
+  in=$(cat "$1")
   escape_html "$in"
 }
 
+get_uuid_from_meta_file() {
+  probable_uuid=$(cat "$1" | head -1)
+  if printf "$probable_uuid" | grep -Eq "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"; then
+    printf "$probable_uuid"
+  else
+    echo "Malformed UUID in meta file." >&2
+    exit 1
+  fi
+}
+
+get_creation_date_from_meta_file_seconds() {
+  cat "$1" | sed -n '2p' | cut -d'_' -f1
+}
+
+get_creation_date_from_meta_file_nanoseconds() {
+  cat "$1" | sed -n '2p'
+}
+
+get_lastmod_date_from_meta_file() {
+  cat "$1" | sed -n '4p'
+}
+
 escape_url() {
   out=`python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1]))' "$1"`
   printf "%s" "$out"
diff --git a/processor/index.html.do b/processor/index.html.do
index 5149cac..0310294 100644
--- a/processor/index.html.do
+++ b/processor/index.html.do
@@ -17,13 +17,12 @@ tmp_snippets_dir=.tmp_index_snippets
 mkdir -p "$tmp_snippets_dir"
 for file in ./*.rst ./*.md; do
   if [ -e "$file" ]; then
-    uuid_file="${metadata_dir}/${file%.*}.uuid"
-    redo-ifchange "$uuid_file"
-    published=`stat -c%y "${uuid_file}"`
-    published_unix=$(date -u "+%s%N" -d "${published}")
+    meta_file="${metadata_dir}/${file%.*}.automatic_metadata"
+    redo-ifchange "$meta_file"
+    published=$(get_creation_date_from_meta_file_nanoseconds "$meta_file")
     snippet_file="${metadata_dir}/${file%.*}.index_snippet"
     redo-ifchange "$snippet_file"
-    ln -s "$srcdir/$snippet_file" "./${tmp_snippets_dir}/${published_unix}"
+    ln -s "$srcdir/$snippet_file" "./${tmp_snippets_dir}/${published}"
   fi
 done
 
diff --git a/processor/metadata/automatic_metadata.do b/processor/metadata/automatic_metadata.do
new file mode 100644
index 0000000..44285d2
--- /dev/null
+++ b/processor/metadata/automatic_metadata.do
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+if [ ! -f "$1" ]; then
+  uuidgen
+  date -u "+%s"
+fi
diff --git a/processor/metadata/default.automatic_metadata.do b/processor/metadata/default.automatic_metadata.do
new file mode 100644
index 0000000..85f0604
--- /dev/null
+++ b/processor/metadata/default.automatic_metadata.do
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+if [ ! -f "$1" ]; then
+  uuidgen
+  date -u "+%s_%N"
+  echo 00000000000000000000000000000000
+  echo 0
+fi
diff --git a/processor/metadata/default.feed_snippet.do b/processor/metadata/default.feed_snippet.do
index 3321ac4..8dca59b 100644
--- a/processor/metadata/default.feed_snippet.do
+++ b/processor/metadata/default.feed_snippet.do
@@ -3,20 +3,20 @@
 # Pull in dependencies. 
 . ../helpers.sh
 src_file=$(get_source_file "$1")
-uuid_file="${1%.feed_snippet}.uuid"
-redo-ifchange "$uuid_file"
+meta_file="${1%.feed_snippet}.automatic_metadata"
+redo-ifchange "$meta_file"
 intermediate_file="${1%.feed_snippet}.intermediate"
 redo-ifchange "$intermediate_file"
 
 # Get variables, write entry.
 html_file=$(escape_url "${1%.feed_snippet}.html")
-lastmod=`stat -c%y "$src_file"`
-lastmod_rfc3339=`date -u "+%Y-%m-%dT%TZ" -d "$lastmod"`
-published=`stat -c%y "$uuid_file"`
-published_rfc3339=`date -u "+%Y-%m-%dT%TZ" -d "${published}"`
-title=`read_and_escape_file "$intermediate_file" | head -1`
-uuid=`read_and_escape_file "$uuid_file" | head -1`
-body=`read_and_escape_file "$intermediate_file" | sed 1d`
+lastmod=$(get_lastmod_date_from_meta_file "$meta_file")
+lastmod_rfc3339=$(date -u "+%Y-%m-%dT%TZ" -d "@$lastmod")
+title=$(read_and_escape_file "$intermediate_file" | head -1)
+uuid=$(get_uuid_from_meta_file "$meta_file")
+published_unix=$(get_creation_date_from_meta_file_seconds "$meta_file")
+published_rfc3339=$(date -u "+%Y-%m-%dT%TZ" -d "@${published_unix}")
+body=$(read_and_escape_file "$intermediate_file" | sed 1d)
 printf "<entry>\n"
 printf "<title type=\"html\">%s</title>\n" "$title"
 printf "<id>urn:uuid:%s</id>\n" "$uuid"
diff --git a/processor/metadata/default.index_snippet.do b/processor/metadata/default.index_snippet.do
index a1ff6f9..9df84fb 100644
--- a/processor/metadata/default.index_snippet.do
+++ b/processor/metadata/default.index_snippet.do
@@ -3,8 +3,8 @@
 # Pull in dependencies.
 . ../helpers.sh
 src_file=$(get_source_file "$1")
-uuid_file="${1%.index_snippet}.uuid"
-redo-ifchange "$uuid_file"
+meta_file="${1%.index_snippet}.automatic_metadata"
+redo-ifchange "$meta_file"
 intermediate_file="${1%.index_snippet}.intermediate"
 redo-ifchange "$intermediate_file"
 html_file="${src_file%.*}.html"
@@ -15,13 +15,13 @@ redo-ifchange "$template_file"
 # Build entry data.
 title=$(cat "$intermediate_file" | head -1 | prep_sed)
 link=$(escape_url "${1%.index_snippet}.html" | prep_sed)
-datetime_created_unix=$(stat -c%y "${uuid_file}")
-date_created=$(date -u "+%Y-%m-%d" -d "${datetime_created_unix}")
+datetime_created_unix=$(get_creation_date_from_meta_file_seconds "$meta_file")
+date_created_human=$(date -u "+%Y-%m-%d" -d "@${datetime_created_unix}")
 
 # Put data into template.
 template=$(cat "$template_file")
 printf "%s\n" "$template" | \
 sed 's/%TITLE%/'"$title"'/g' | \
 sed 's/%LINK%/'"$link"'/g' | \
-sed 's/%DATE_CREATED%/'"$date_created"'/g' | \
+sed 's/%DATE_CREATED%/'"$date_created_human"'/g' | \
 tr '\a' '%'
diff --git a/processor/metadata/default.intermediate.do b/processor/metadata/default.intermediate.do
index a09d4b2..bf87309 100644
--- a/processor/metadata/default.intermediate.do
+++ b/processor/metadata/default.intermediate.do
@@ -1,11 +1,14 @@
 #!/bin/sh
 
+# Pull in dependencies.
 template=intermediate.pandoc_tmpl
-uuidfile="${1%.intermediate}.uuid"
-redo-ifchange "$uuidfile"
+meta_file="${1%.intermediate}.automatic_metadata"
+redo-ifchange "$meta_file"
 redo-ifchange "$template"
 mdfile="../${1%.intermediate}.md"
 rstfile="../${1%.intermediate}.rst"
+
+# Build intermediate file.
 if [ -f "$rstfile" ]; then
   redo-ifchange "$rstfile"
   pandoc -f rst --template="$template" --mathml -t html5 "$rstfile" --base-header-level=2 > "$3"
@@ -13,3 +16,13 @@ elif [ -f "$mdfile" ]; then
   redo-ifchange "$mdfile"
   pandoc -f markdown --template="$template" --mathml -t html5 "$mdfile" --base-header-level=2 > "$3"
 fi
+
+# Update meta file if appropriate.
+md5_new=$(md5sum "$3" | cut -d ' ' -f 1)
+md5_old=$(cat "$meta_file" | sed -n '3p')
+if [ ! "$md5_new" = "$md5_old" ]; then
+  new_date=$(date -u "+%s")
+  sed -i '1,2!d' "$meta_file"
+  echo "$md5_new" >> "$meta_file"
+  echo "$new_date" >> "$meta_file"
+fi
diff --git a/processor/metadata/default.uuid.do b/processor/metadata/default.uuid.do
deleted file mode 100644
index 5efffc8..0000000
--- a/processor/metadata/default.uuid.do
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/sh
-
-if [ ! -f "$1" ]; then
-  uuidgen > "$1"
-fi
diff --git a/processor/metadata/uuid.do b/processor/metadata/uuid.do
deleted file mode 100644
index ba9e919..0000000
--- a/processor/metadata/uuid.do
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/sh
-
-if [ ! -f "$1" ]; then
-  uuidgen
-fi
diff --git a/test.sh b/test.sh
index 025ebc4..60ce2bd 100755
--- a/test.sh
+++ b/test.sh
@@ -1,10 +1,27 @@
 #!/bin/sh
 
-uuid_test()
+uuid_pattern='[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
+
+blog_meta_file_test()
+{
+  meta_file="$1"
+  printf "== %s meta file pattern match test ==\n" "$meta_file"
+  if cat "$meta_file" | sed -n '1p' | grep -Eq '^'"$uuid_pattern"'$' \
+      && cat "$meta_file" | sed -n '2p' | grep -Eq "^[0-9]+$"; then
+    echo "== test SUCCESS =="
+  else
+    echo "== test FAILURE =="
+  fi
+}
+
+article_meta_file_test()
 {
-  uuid_file="$1"
-  printf "== %s UUID pattern match test ==\n" "$uuid_file"
-  if cat "$uuid_file" | grep -Eq "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"; then
+  meta_file="$1"
+  printf "== %s meta file pattern match test ==\n" "$meta_file"
+  if cat "$meta_file" | sed -n '1p' | grep -Eq '^'"$uuid_pattern"'$' \
+      && cat "$meta_file" | sed -n '2p' | grep -Eq "^[0-9]+_[0-9]+$" \
+      && cat "$meta_file" | sed -n '3p' | grep -Eq "^[0-9a-f]{32}$" \
+      && cat "$meta_file" | sed -n '2p' | grep -Eq "^[0-9]+_[0-9]+$"; then
     echo "== test SUCCESS =="
   else
     echo "== test FAILURE =="
@@ -24,7 +41,7 @@ diff_test()
   fi
 }
 
-# Set up test directory. 
+# Set up test directory, run file creations.
 expected_files_dir="test/test_files"
 generated_files_dir="test/test_dir"
 rm -rf "$generated_files_dir" 
@@ -38,10 +55,35 @@ cp "$working_dir/$expected_files_dir"/bar\ baz.md .
 redo
 cp "$working_dir/$expected_files_dir"/foo.rst .
 redo
-cd "$working_dir"
+
+# Test file modification tracking.
+update_datetime_start=$(cat "metadata/bar baz.feed_snippet" | grep '<updated>')
+sleep 1
+sed -i '2d' bar\ baz.md
+redo
+update_datetime_after_invisible_change=$(cat "metadata/bar baz.feed_snippet" | grep '<updated>')
+printf "== testing \"bar baz\"' update tag remaining unchanged with invisible source file change ==\n"
+if [ "$update_datetime_start" = "$update_datetime_after_invisible_change" ]; then
+    echo "== test SUCCESS =="
+else
+    echo "== test FAILURE =="
+fi
+sleep 1
+sed -i '2d' bar\ baz.md
+redo
+update_datetime_after_visible_change=$(cat "metadata/bar baz.feed_snippet" | grep '<updated>')
+printf "== testing \"bar baz\"' update tag changing with visible source file change ==\n"
+if [ "$update_datetime_start" = "$update_datetime_after_visible_change" ]; then
+    echo "== test FAILURE =="
+else
+    echo "== test SUCCESS =="
+fi
+cp "$working_dir/$expected_files_dir"/bar\ baz.md .
+redo
 
 # Compare metadata files.
-uuid_test "$generated_files_dir""/metadata/uuid"
+cd "$working_dir"
+blog_meta_file_test "$generated_files_dir""/metadata/automatic_metadata"
 for file in "$expected_files_dir"/metadata/*; do
   basename=$(basename "$file")
   cmp_file="$generated_files_dir/metadata/$basename"
@@ -53,7 +95,7 @@ for file in "$expected_files_dir"/*.html.ignoring; do
   basename=$(basename "$file")
   cmp_file="$generated_files_dir/${basename%.ignoring}"
   if [ ! "$file" = "$expected_files_dir""/index.html.ignoring" ]; then
-    uuid_test "${generated_files_dir}/metadata/${basename%.html.ignoring}.uuid"
+    article_meta_file_test "${generated_files_dir}/metadata/${basename%.html.ignoring}.automatic_metadata"
   fi
   generated_file="$cmp_file".ignoring
   cat "$cmp_file" | \