home · contact · privacy
Add TaskHandler code to actually make previous commit work.
[plomtask] / plomtask / http.py
index b7040f76fa9c3c1d58b0ceedc58fabf02752f616..e307f1429f93227f7b960d2cad3d55b95e3d737c 100644 (file)
@@ -1,6 +1,5 @@
 """Web server stuff."""
 from __future__ import annotations
-from dataclasses import dataclass
 from typing import Any, Callable
 from base64 import b64encode, b64decode
 from binascii import Error as binascii_Exception
@@ -14,10 +13,11 @@ from plomtask.dating import date_in_n_days
 from plomtask.days import Day
 from plomtask.exceptions import (HandledException, BadFormatException,
                                  NotFoundException)
-from plomtask.db import DatabaseConnection, DatabaseFile
+from plomtask.db import DatabaseConnection, DatabaseFile, BaseModel
 from plomtask.processes import Process, ProcessStep, ProcessStepsNode
 from plomtask.conditions import Condition
-from plomtask.todos import Todo
+from plomtask.todos import Todo, TodoOrProcStepNode
+from plomtask.misc import DictableNode
 
 TEMPLATES_DIR = 'templates'
 
@@ -29,113 +29,79 @@ class TaskServer(HTTPServer):
                  *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.db = db_file
-        self.headers: list[tuple[str, str]] = []
-        self._render_mode = 'html'
-        self._jinja = JinjaEnv(loader=JinjaFSLoader(TEMPLATES_DIR))
-
-    def set_json_mode(self) -> None:
-        """Make server send JSON instead of HTML responses."""
-        self._render_mode = 'json'
-        self.headers += [('Content-Type', 'application/json')]
-
-    @staticmethod
-    def ctx_to_json(ctx: dict[str, object]) -> str:
-        """Render ctx into JSON string."""
-        def walk_ctx(node: object) -> Any:
-            if hasattr(node, 'as_dict_into_reference'):
-                if hasattr(node, 'id_') and node.id_ is not None:
-                    return node.as_dict_into_reference(library)
-            if hasattr(node, 'as_dict'):
-                return node.as_dict
-            if isinstance(node, (list, tuple)):
-                return [walk_ctx(x) for x in node]
-            if isinstance(node, dict):
-                d = {}
-                for k, v in node.items():
-                    d[k] = walk_ctx(v)
-                return d
-            if isinstance(node, HandledException):
-                return str(node)
-            return node
-        library: dict[str, dict[str | int, object]] = {}
-        for k, v in ctx.items():
-            ctx[k] = walk_ctx(v)
-        ctx['_library'] = library
-        return json_dumps(ctx)
-
-    def render(self, ctx: dict[str, object], tmpl_name: str = '') -> str:
-        """Render ctx according to self._render_mode.."""
-        tmpl_name = f'{tmpl_name}.{self._render_mode}'
-        if 'html' == self._render_mode:
-            template = self._jinja.get_template(tmpl_name)
-            return template.render(ctx)
-        return self.__class__.ctx_to_json(ctx)
+        self.render_mode = 'html'
+        self.jinja = JinjaEnv(loader=JinjaFSLoader(TEMPLATES_DIR))
 
 
 class InputsParser:
     """Wrapper for validating and retrieving dict-like HTTP inputs."""
 
-    def __init__(self, dict_: dict[str, list[str]],
-                 strictness: bool = True) -> None:
+    def __init__(self, dict_: dict[str, list[str]]) -> None:
         self.inputs = dict_
-        self.strict = strictness
 
-    def get_str(self, key: str, default: str = '',
-                ignore_strict: bool = False) -> str:
-        """Retrieve single/first string value of key, or default."""
-        if key not in self.inputs.keys() or 0 == len(self.inputs[key]):
-            if self.strict and not ignore_strict:
-                raise BadFormatException(f'no value found for key {key}')
-            return default
-        return self.inputs[key][0]
-
-    def get_first_strings_starting(self, prefix: str) -> dict[str, str]:
-        """Retrieve dict of (first) strings at key starting with prefix."""
-        ret = {}
-        for key in [k for k in self.inputs.keys() if k.startswith(prefix)]:
-            ret[key] = self.inputs[key][0]
-        return ret
+    def get_all_str(self, key: str) -> list[str]:
+        """Retrieve list of string values at key (empty if no key)."""
+        if key not in self.inputs.keys():
+            return []
+        return self.inputs[key]
 
-    def get_int(self, key: str) -> int:
-        """Retrieve single/first value of key as int, error if empty."""
-        val = self.get_int_or_none(key)
-        if val is None:
-            raise BadFormatException(f'unexpected empty value for: {key}')
-        return val
+    def get_all_int(self, key: str) -> list[int]:
+        """Retrieve list of int values at key."""
+        all_str = self.get_all_str(key)
+        try:
+            return [int(s) for s in all_str if len(s) > 0]
+        except ValueError as e:
+            msg = f'cannot int a form field value for key {key} in: {all_str}'
+            raise BadFormatException(msg) from e
+
+    def get_str(self, key: str, default: str | None = None) -> str | None:
+        """Retrieve single/first string value of key, or default."""
+        vals = self.get_all_str(key)
+        if vals:
+            return vals[0]
+        return default
+
+    def get_str_or_fail(self, key: str, default: str | None = None) -> str:
+        """Retrieve first string value of key, if none: fail or default."""
+        vals = self.get_all_str(key)
+        if not vals:
+            if default is not None:
+                return default
+            raise BadFormatException(f'no value found for key: {key}')
+        return vals[0]
 
     def get_int_or_none(self, key: str) -> int | None:
         """Retrieve single/first value of key as int, return None if empty."""
-        val = self.get_str(key, ignore_strict=True)
+        val = self.get_str_or_fail(key, '')
         if val == '':
             return None
         try:
             return int(val)
-        except ValueError as e:
+        except (ValueError, TypeError) as e:
             msg = f'cannot int form field value for key {key}: {val}'
             raise BadFormatException(msg) from e
 
-    def get_float(self, key: str) -> float:
-        """Retrieve float value of key from self.postvars."""
+    def get_bool_or_none(self, key: str) -> bool | None:
+        """Return value to key if truish; if no value to key, None."""
         val = self.get_str(key)
-        try:
-            return float(val)
-        except ValueError as e:
-            msg = f'cannot float form field value for key {key}: {val}'
-            raise BadFormatException(msg) from e
+        if val is None:
+            return None
+        return val in {'True', 'true', '1', 'on'}
 
-    def get_all_str(self, key: str) -> list[str]:
-        """Retrieve list of string values at key."""
-        if key not in self.inputs.keys():
-            return []
-        return self.inputs[key]
+    def get_all_of_key_prefixed(self, key_prefix: str) -> dict[str, list[str]]:
+        """Retrieve dict of strings at keys starting with key_prefix."""
+        ret = {}
+        for key in [k for k in self.inputs.keys() if k.startswith(key_prefix)]:
+            ret[key[len(key_prefix):]] = self.inputs[key]
+        return ret
 
-    def get_all_int(self, key: str) -> list[int]:
-        """Retrieve list of int values at key."""
-        all_str = self.get_all_str(key)
+    def get_float_or_fail(self, key: str) -> float:
+        """Retrieve float value of key from self.postvars, fail if none."""
+        val = self.get_str_or_fail(key)
         try:
-            return [int(s) for s in all_str if len(s) > 0]
+            return float(val)
         except ValueError as e:
-            msg = f'cannot int a form field value for key {key} in: {all_str}'
+            msg = f'cannot float form field value for key {key}: {val}'
             raise BadFormatException(msg) from e
 
     def get_all_floats_or_nones(self, key: str) -> list[float | None]:
@@ -157,24 +123,81 @@ class TaskHandler(BaseHTTPRequestHandler):
     """Handles single HTTP request."""
     # pylint: disable=too-many-public-methods
     server: TaskServer
-    conn: DatabaseConnection
+    _conn: DatabaseConnection
     _site: str
-    _form_data: InputsParser
+    _form: InputsParser
     _params: InputsParser
 
-    def _send_page(self,
-                   ctx: dict[str, Any],
-                   tmpl_name: str,
-                   code: int = 200
-                   ) -> None:
-        """Send ctx as proper HTTP response."""
-        body = self.server.render(ctx, tmpl_name)
+    def _send_page(
+            self, ctx: dict[str, Any], tmpl_name: str, code: int = 200
+            ) -> None:
+        """HTTP-send ctx as HTML or JSON, as defined by .server.render_mode.
+
+        The differentiation by .server.render_mode serves to allow easily
+        comparable JSON responses for automatic testing.
+        """
+        body: str
+        headers: list[tuple[str, str]] = []
+        if 'html' == self.server.render_mode:
+            tmpl = self.server.jinja.get_template(f'{tmpl_name}.html')
+            body = tmpl.render(ctx)
+        else:
+            body = self._ctx_to_json(ctx)
+            headers += [('Content-Type', 'application/json')]
         self.send_response(code)
-        for header_tuple in self.server.headers:
+        for header_tuple in headers:
             self.send_header(*header_tuple)
         self.end_headers()
         self.wfile.write(bytes(body, 'utf-8'))
 
+    def _ctx_to_json(self, ctx: dict[str, object]) -> str:
+        """Render ctx into JSON string.
+
+        Flattens any objects that json.dumps might not want to serialize, and
+        turns occurrences of BaseModel objects into listings of their .id_, to
+        be resolved to a full dict inside a top-level '_library' dictionary,
+        to avoid endless and circular nesting.
+        """
+
+        def flatten(node: object) -> object:
+
+            def update_library_with(
+                    item: BaseModel[int] | BaseModel[str]) -> None:
+                cls_name = item.__class__.__name__
+                if cls_name not in library:
+                    library[cls_name] = {}
+                if item.id_ not in library[cls_name]:
+                    d, refs = item.as_dict_and_refs
+                    id_key = '?' if item.id_ is None else item.id_
+                    library[cls_name][id_key] = d
+                    for ref in refs:
+                        update_library_with(ref)
+
+            if isinstance(node, BaseModel):
+                update_library_with(node)
+                return node.id_
+            if isinstance(node, DictableNode):
+                d, refs = node.as_dict_and_refs
+                for ref in refs:
+                    update_library_with(ref)
+                return d
+            if isinstance(node, (list, tuple)):
+                return [flatten(item) for item in node]
+            if isinstance(node, dict):
+                d = {}
+                for k, v in node.items():
+                    d[k] = flatten(v)
+                return d
+            if isinstance(node, HandledException):
+                return str(node)
+            return node
+
+        library: dict[str, dict[str | int, object]] = {}
+        for k, v in ctx.items():
+            ctx[k] = flatten(v)
+        ctx['_library'] = library
+        return json_dumps(ctx)
+
     @staticmethod
     def _request_wrapper(http_method: str, not_found_msg: str
                          ) -> Callable[..., Callable[[TaskHandler], None]]:
@@ -217,11 +240,13 @@ class TaskHandler(BaseHTTPRequestHandler):
                 # (because pylint here fails to detect the use of wrapper as a
                 # method to self with respective access privileges)
                 try:
-                    self.conn = DatabaseConnection(self.server.db)
+                    self._conn = DatabaseConnection(self.server.db)
                     parsed_url = urlparse(self.path)
                     self._site = path_split(parsed_url.path)[1]
-                    params = parse_qs(parsed_url.query, strict_parsing=True)
-                    self._params = InputsParser(params, False)
+                    params = parse_qs(parsed_url.query,
+                                      keep_blank_values=True,
+                                      strict_parsing=True)
+                    self._params = InputsParser(params)
                     handler_name = f'do_{http_method}_{self._site}'
                     if hasattr(self, handler_name):
                         handler = getattr(self, handler_name)
@@ -241,7 +266,7 @@ class TaskHandler(BaseHTTPRequestHandler):
                     ctx = {'msg': error}
                     self._send_page(ctx, 'msg', error.http_code)
                 finally:
-                    self.conn.close()
+                    self._conn.close()
             return wrapper
         return decorator
 
@@ -261,10 +286,10 @@ class TaskHandler(BaseHTTPRequestHandler):
         """Handle POST with handler, prepare redirection to result."""
         length = int(self.headers['content-length'])
         postvars = parse_qs(self.rfile.read(length).decode(),
-                            keep_blank_values=True, strict_parsing=True)
-        self._form_data = InputsParser(postvars)
+                            keep_blank_values=True)
+        self._form = InputsParser(postvars)
         redir_target = handler()
-        self.conn.commit()
+        self._conn.commit()
         return redir_target
 
     # GET handlers
@@ -281,9 +306,9 @@ class TaskHandler(BaseHTTPRequestHandler):
                 # method to self with respective access privileges)
                 id_ = self._params.get_int_or_none('id')
                 if target_class.can_create_by_id:
-                    item = target_class.by_id_or_create(self.conn, id_)
+                    item = target_class.by_id_or_create(self._conn, id_)
                 else:
-                    item = target_class.by_id(self.conn, id_)
+                    item = target_class.by_id(self._conn, id_)
                 return f(self, item)
             return wrapper
         return decorator
@@ -299,12 +324,12 @@ class TaskHandler(BaseHTTPRequestHandler):
         same, the only difference being the HTML template they are rendered to,
         which .do_GET selects from their method name.
         """
-        start = self._params.get_str('start')
-        end = self._params.get_str('end')
-        if not end:
-            end = date_in_n_days(366)
-        ret = Day.by_date_range_with_limits(self.conn, (start, end), 'id')
-        days, start, end = ret
+        start = self._params.get_str_or_fail('start', '')
+        end = self._params.get_str_or_fail('end', '')
+        end = end if end != '' else date_in_n_days(366)
+        #
+        days, start, end = Day.by_date_range_with_limits(self._conn,
+                                                         (start, end), 'id')
         days = Day.with_filled_gaps(days, start, end)
         today = date_in_n_days(0)
         return {'start': start, 'end': end, 'days': days, 'today': today}
@@ -319,9 +344,10 @@ class TaskHandler(BaseHTTPRequestHandler):
 
     def do_GET_day(self) -> dict[str, object]:
         """Show single Day of ?date=."""
-        date = self._params.get_str('date', date_in_n_days(0))
-        day = Day.by_id_or_create(self.conn, date)
-        make_type = self._params.get_str('make_type')
+        date = self._params.get_str_or_fail('date', date_in_n_days(0))
+        make_type = self._params.get_str_or_fail('make_type', 'full')
+        #
+        day = Day.by_id_or_create(self._conn, date)
         conditions_present = []
         enablers_for = {}
         disablers_for = {}
@@ -330,10 +356,10 @@ class TaskHandler(BaseHTTPRequestHandler):
                 if condition not in conditions_present:
                     conditions_present += [condition]
                     enablers_for[condition.id_] = [p for p in
-                                                   Process.all(self.conn)
+                                                   Process.all(self._conn)
                                                    if condition in p.enables]
                     disablers_for[condition.id_] = [p for p in
-                                                    Process.all(self.conn)
+                                                    Process.all(self._conn)
                                                     if condition in p.disables]
         seen_todos: set[int] = set()
         top_nodes = [t.get_step_tree(seen_todos)
@@ -344,33 +370,27 @@ class TaskHandler(BaseHTTPRequestHandler):
                 'enablers_for': enablers_for,
                 'disablers_for': disablers_for,
                 'conditions_present': conditions_present,
-                'processes': Process.all(self.conn)}
+                'processes': Process.all(self._conn)}
 
     @_get_item(Todo)
     def do_GET_todo(self, todo: Todo) -> dict[str, object]:
         """Show single Todo of ?id=."""
 
-        @dataclass
-        class TodoStepsNode:
-            """Collect what's useful for Todo steps tree display."""
-            id_: int
-            todo: Todo | None
-            process: Process | None
-            children: list[TodoStepsNode]  # pylint: disable=undefined-variable
-            fillable: bool = False
-
-        def walk_process_steps(id_: int,
+        def walk_process_steps(node_id: int,
                                process_step_nodes: list[ProcessStepsNode],
-                               steps_nodes: list[TodoStepsNode]) -> None:
+                               steps_nodes: list[TodoOrProcStepNode]) -> int:
             for process_step_node in process_step_nodes:
-                id_ += 1
-                node = TodoStepsNode(id_, None, process_step_node.process, [])
+                node_id += 1
+                proc = Process.by_id(self._conn,
+                                     process_step_node.step.step_process_id)
+                node = TodoOrProcStepNode(node_id, None, proc, [])
                 steps_nodes += [node]
-                walk_process_steps(id_, list(process_step_node.steps.values()),
-                                   node.children)
+                node_id = walk_process_steps(
+                        node_id, process_step_node.steps, node.children)
+            return node_id
 
-        def walk_todo_steps(id_: int, todos: list[Todo],
-                            steps_nodes: list[TodoStepsNode]) -> None:
+        def walk_todo_steps(node_id: int, todos: list[Todo],
+                            steps_nodes: list[TodoOrProcStepNode]) -> int:
             for todo in todos:
                 matched = False
                 for match in [item for item in steps_nodes
@@ -380,15 +400,18 @@ class TaskHandler(BaseHTTPRequestHandler):
                     matched = True
                     for child in match.children:
                         child.fillable = True
-                    walk_todo_steps(id_, todo.children, match.children)
+                    node_id = walk_todo_steps(
+                            node_id, todo.children, match.children)
                 if not matched:
-                    id_ += 1
-                    node = TodoStepsNode(id_, todo, None, [])
+                    node_id += 1
+                    node = TodoOrProcStepNode(node_id, todo, None, [])
                     steps_nodes += [node]
-                    walk_todo_steps(id_, todo.children, node.children)
+                    node_id = walk_todo_steps(
+                            node_id, todo.children, node.children)
+            return node_id
 
-        def collect_adoptables_keys(steps_nodes: list[TodoStepsNode]
-                                    ) -> set[int]:
+        def collect_adoptables_keys(
+                steps_nodes: list[TodoOrProcStepNode]) -> set[int]:
             ids = set()
             for node in steps_nodes:
                 if not node.todo:
@@ -399,37 +422,37 @@ class TaskHandler(BaseHTTPRequestHandler):
             return ids
 
         todo_steps = [step.todo for step in todo.get_step_tree(set()).children]
-        process_tree = todo.process.get_steps(self.conn, None)
-        steps_todo_to_process: list[TodoStepsNode] = []
-        walk_process_steps(0, list(process_tree.values()),
-                           steps_todo_to_process)
+        process_tree = todo.process.get_steps(self._conn, None)
+        steps_todo_to_process: list[TodoOrProcStepNode] = []
+        last_node_id = walk_process_steps(0, process_tree,
+                                          steps_todo_to_process)
         for steps_node in steps_todo_to_process:
             steps_node.fillable = True
-        walk_todo_steps(len(steps_todo_to_process), todo_steps,
-                        steps_todo_to_process)
+        walk_todo_steps(last_node_id, todo_steps, steps_todo_to_process)
         adoptables: dict[int, list[Todo]] = {}
-        any_adoptables = [Todo.by_id(self.conn, t.id_)
-                          for t in Todo.by_date(self.conn, todo.date)
+        any_adoptables = [Todo.by_id(self._conn, t.id_)
+                          for t in Todo.by_date(self._conn, todo.date)
                           if t.id_ is not None
                           and t != todo]
         for id_ in collect_adoptables_keys(steps_todo_to_process):
             adoptables[id_] = [t for t in any_adoptables
                                if t.process.id_ == id_]
-        return {'todo': todo, 'steps_todo_to_process': steps_todo_to_process,
+        return {'todo': todo,
+                'steps_todo_to_process': steps_todo_to_process,
                 'adoption_candidates_for': adoptables,
-                'process_candidates': Process.all(self.conn),
+                'process_candidates': sorted(Process.all(self._conn)),
                 'todo_candidates': any_adoptables,
-                'condition_candidates': Condition.all(self.conn)}
+                'condition_candidates': Condition.all(self._conn)}
 
     def do_GET_todos(self) -> dict[str, object]:
         """Show Todos from ?start= to ?end=, of ?process=, ?comment= pattern"""
-        sort_by = self._params.get_str('sort_by')
-        start = self._params.get_str('start')
-        end = self._params.get_str('end')
+        sort_by = self._params.get_str_or_fail('sort_by', 'title')
+        start = self._params.get_str_or_fail('start', '')
+        end = self._params.get_str_or_fail('end', '')
         process_id = self._params.get_int_or_none('process_id')
-        comment_pattern = self._params.get_str('comment_pattern')
-        todos = []
-        ret = Todo.by_date_range_with_limits(self.conn, (start, end))
+        comment_pattern = self._params.get_str_or_fail('comment_pattern', '')
+        #
+        ret = Todo.by_date_range_with_limits(self._conn, (start, end))
         todos_by_date_range, start, end = ret
         todos = [t for t in todos_by_date_range
                  if comment_pattern in t.comment
@@ -437,13 +460,14 @@ class TaskHandler(BaseHTTPRequestHandler):
         sort_by = Todo.sort_by(todos, sort_by)
         return {'start': start, 'end': end, 'process_id': process_id,
                 'comment_pattern': comment_pattern, 'todos': todos,
-                'all_processes': Process.all(self.conn), 'sort_by': sort_by}
+                'all_processes': Process.all(self._conn), 'sort_by': sort_by}
 
     def do_GET_conditions(self) -> dict[str, object]:
         """Show all Conditions."""
-        pattern = self._params.get_str('pattern')
-        sort_by = self._params.get_str('sort_by')
-        conditions = Condition.matching(self.conn, pattern)
+        pattern = self._params.get_str_or_fail('pattern', '')
+        sort_by = self._params.get_str_or_fail('sort_by', 'title')
+        #
+        conditions = Condition.matching(self._conn, pattern)
         sort_by = Condition.sort_by(conditions, sort_by)
         return {'conditions': conditions,
                 'sort_by': sort_by,
@@ -452,7 +476,7 @@ class TaskHandler(BaseHTTPRequestHandler):
     @_get_item(Condition)
     def do_GET_condition(self, c: Condition) -> dict[str, object]:
         """Show Condition of ?id=."""
-        ps = Process.all(self.conn)
+        ps = Process.all(self._conn)
         return {'condition': c, 'is_new': c.id_ is None,
                 'enabled_processes': [p for p in ps if c in p.conditions],
                 'disabled_processes': [p for p in ps if c in p.blockers],
@@ -475,26 +499,30 @@ class TaskHandler(BaseHTTPRequestHandler):
         owner_ids = self._params.get_all_int('step_to')
         owned_ids = self._params.get_all_int('has_step')
         title_64 = self._params.get_str('title_b64')
+        title_new = None
         if title_64:
             try:
-                title = b64decode(title_64.encode()).decode()
+                title_new = b64decode(title_64.encode()).decode()
             except binascii_Exception as exc:
                 msg = 'invalid base64 for ?title_b64='
                 raise BadFormatException(msg) from exc
-            process.title.set(title)
+        #
+        if title_new:
+            process.title.set(title_new)
         preset_top_step = None
-        owners = process.used_as_step_by(self.conn)
+        owners = process.used_as_step_by(self._conn)
         for step_id in owner_ids:
-            owners += [Process.by_id(self.conn, step_id)]
+            owners += [Process.by_id(self._conn, step_id)]
         for process_id in owned_ids:
-            Process.by_id(self.conn, process_id)  # to ensure ID exists
+            Process.by_id(self._conn, process_id)  # to ensure ID exists
             preset_top_step = process_id
         return {'process': process, 'is_new': process.id_ is None,
                 'preset_top_step': preset_top_step,
-                'steps': process.get_steps(self.conn), 'owners': owners,
-                'n_todos': len(Todo.by_process_id(self.conn, process.id_)),
-                'process_candidates': Process.all(self.conn),
-                'condition_candidates': Condition.all(self.conn)}
+                'steps': process.get_steps(self._conn),
+                'owners': owners,
+                'n_todos': len(Todo.by_process_id(self._conn, process.id_)),
+                'process_candidates': Process.all(self._conn),
+                'condition_candidates': Condition.all(self._conn)}
 
     @_get_item(Process)
     def do_GET_process_titles(self, p: Process) -> dict[str, object]:
@@ -513,9 +541,10 @@ class TaskHandler(BaseHTTPRequestHandler):
 
     def do_GET_processes(self) -> dict[str, object]:
         """Show all Processes."""
-        pattern = self._params.get_str('pattern')
-        sort_by = self._params.get_str('sort_by')
-        processes = Process.matching(self.conn, pattern)
+        pattern = self._params.get_str_or_fail('pattern', '')
+        sort_by = self._params.get_str_or_fail('sort_by', 'title')
+        #
+        processes = Process.matching(self._conn, pattern)
         sort_by = Process.sort_by(processes, sort_by)
         return {'processes': processes, 'sort_by': sort_by, 'pattern': pattern}
 
@@ -531,18 +560,18 @@ class TaskHandler(BaseHTTPRequestHandler):
                 # (because pylint here fails to detect the use of wrapper as a
                 # method to self with respective access privileges)
                 id_ = self._params.get_int_or_none('id')
-                for _ in self._form_data.get_all_str('delete'):
+                for _ in self._form.get_all_str('delete'):
                     if id_ is None:
                         msg = 'trying to delete non-saved ' +\
                                 f'{target_class.__name__}'
                         raise NotFoundException(msg)
-                    item = target_class.by_id(self.conn, id_)
-                    item.remove(self.conn)
+                    item = target_class.by_id(self._conn, id_)
+                    item.remove(self._conn)
                     return redir_target
                 if target_class.can_create_by_id:
-                    item = target_class.by_id_or_create(self.conn, id_)
+                    item = target_class.by_id_or_create(self._conn, id_)
                 else:
-                    item = target_class.by_id(self.conn, id_)
+                    item = target_class.by_id(self._conn, id_)
                 return f(self, item)
             return wrapper
         return decorator
@@ -550,106 +579,133 @@ class TaskHandler(BaseHTTPRequestHandler):
     def _change_versioned_timestamps(self, cls: Any, attr_name: str) -> str:
         """Update history timestamps for VersionedAttribute."""
         id_ = self._params.get_int_or_none('id')
-        item = cls.by_id(self.conn, id_)
+        item = cls.by_id(self._conn, id_)
         attr = getattr(item, attr_name)
-        for k, v in self._form_data.get_first_strings_starting('at:').items():
-            old = k[3:]
-            if old[19:] != v:
-                attr.reset_timestamp(old, f'{v}.0')
-        attr.save(self.conn)
+        for k, vals in self._form.get_all_of_key_prefixed('at:').items():
+            if k[19:] != vals[0]:
+                attr.reset_timestamp(k, f'{vals[0]}.0')
+        attr.save(self._conn)
         return f'/{cls.name_lowercase()}_{attr_name}s?id={item.id_}'
 
     def do_POST_day(self) -> str:
         """Update or insert Day of date and Todos mapped to it."""
         # pylint: disable=too-many-locals
-        date = self._params.get_str('date')
-        day_comment = self._form_data.get_str('day_comment')
-        make_type = self._form_data.get_str('make_type')
-        old_todos = self._form_data.get_all_int('todo_id')
-        new_todos = self._form_data.get_all_int('new_todo')
-        comments = self._form_data.get_all_str('comment')
-        efforts = self._form_data.get_all_floats_or_nones('effort')
-        done_todos = self._form_data.get_all_int('done')
-        for _ in [id_ for id_ in done_todos if id_ not in old_todos]:
-            raise BadFormatException('"done" field refers to unknown Todo')
+        date = self._params.get_str_or_fail('date')
+        day_comment = self._form.get_str_or_fail('day_comment')
+        make_type = self._form.get_str_or_fail('make_type')
+        old_todos = self._form.get_all_int('todo_id')
+        new_todos_by_process = self._form.get_all_int('new_todo')
+        comments = self._form.get_all_str('comment')
+        efforts = self._form.get_all_floats_or_nones('effort')
+        done_todos = self._form.get_all_int('done')
         is_done = [t_id in done_todos for t_id in old_todos]
         if not (len(old_todos) == len(is_done) == len(comments)
                 == len(efforts)):
             msg = 'not equal number each of number of todo_id, comments, ' +\
                     'and efforts inputs'
             raise BadFormatException(msg)
-        day = Day.by_id_or_create(self.conn, date)
+        for _ in [id_ for id_ in done_todos if id_ not in old_todos]:
+            raise BadFormatException('"done" field refers to unknown Todo')
+        #
+        day = Day.by_id_or_create(self._conn, date)
         day.comment = day_comment
-        day.save(self.conn)
-        for process_id in sorted(new_todos):
-            if 'empty' == make_type:
-                process = Process.by_id(self.conn, process_id)
-                todo = Todo(None, process, False, date)
-                todo.save(self.conn)
-            else:
-                Todo.create_with_children(self.conn, process_id, date)
+        day.save(self._conn)
+        new_todos = []
+        for process_id in sorted(new_todos_by_process):
+            process = Process.by_id(self._conn, process_id)
+            todo = Todo(None, process, False, date)
+            todo.save(self._conn)
+            new_todos += [todo]
+        if 'full' == make_type:
+            for todo in new_todos:
+                todo.ensure_children(self._conn)
         for i, todo_id in enumerate(old_todos):
-            todo = Todo.by_id(self.conn, todo_id)
+            todo = Todo.by_id(self._conn, todo_id)
             todo.is_done = is_done[i]
             todo.comment = comments[i]
             todo.effort = efforts[i]
-            todo.save(self.conn)
+            todo.save(self._conn)
         return f'/day?date={date}&make_type={make_type}'
 
     @_delete_or_post(Todo, '/')
     def do_POST_todo(self, todo: Todo) -> str:
         """Update Todo and its children."""
         # pylint: disable=too-many-locals
-        adopted_child_ids = self._form_data.get_all_int('adopt')
-        processes_to_make_full = self._form_data.get_all_int('make_full')
-        processes_to_make_empty = self._form_data.get_all_int('make_empty')
-        fill_fors = self._form_data.get_first_strings_starting('fill_for_')
-        effort = self._form_data.get_str('effort', ignore_strict=True)
-        conditions = self._form_data.get_all_int('conditions')
-        disables = self._form_data.get_all_int('disables')
-        blockers = self._form_data.get_all_int('blockers')
-        enables = self._form_data.get_all_int('enables')
-        is_done = len(self._form_data.get_all_str('done')) > 0
-        calendarize = len(self._form_data.get_all_str('calendarize')) > 0
-        comment = self._form_data.get_str('comment', ignore_strict=True)
-        for v in fill_fors.values():
-            if v.startswith('make_empty_'):
-                processes_to_make_empty += [int(v[11:])]
-            elif v.startswith('make_full_'):
-                processes_to_make_full += [int(v[10:])]
-            elif v != 'ignore':
-                adopted_child_ids += [int(v)]
-        to_remove = []
-        for child in todo.children:
-            assert isinstance(child.id_, int)
-            if child.id_ not in adopted_child_ids:
-                to_remove += [child.id_]
-        for id_ in to_remove:
-            child = Todo.by_id(self.conn, id_)
-            todo.remove_child(child)
-        for child_id in adopted_child_ids:
-            if child_id in [c.id_ for c in todo.children]:
-                continue
-            child = Todo.by_id(self.conn, child_id)
-            todo.add_child(child)
-        for process_id in processes_to_make_empty:
-            process = Process.by_id(self.conn, process_id)
-            made = Todo(None, process, False, todo.date)
-            made.save(self.conn)
-            todo.add_child(made)
-        for process_id in processes_to_make_full:
-            made = Todo.create_with_children(self.conn, process_id, todo.date)
-            todo.add_child(made)
-        todo.effort = float(effort) if effort else None
-        todo.set_conditions(self.conn, conditions)
-        todo.set_blockers(self.conn, blockers)
-        todo.set_enables(self.conn, enables)
-        todo.set_disables(self.conn, disables)
-        todo.is_done = is_done
-        todo.calendarize = calendarize
-        todo.comment = comment
-        todo.save(self.conn)
-        return f'/todo?id={todo.id_}'
+        # pylint: disable=too-many-branches
+        # pylint: disable=too-many-statements
+        assert todo.id_ is not None
+        adoptees = [(id_, todo.id_) for id_ in self._form.get_all_int('adopt')]
+        to_make = {'full': [(id_, todo.id_)
+                            for id_ in self._form.get_all_int('make_full')],
+                   'empty': [(id_, todo.id_)
+                             for id_ in self._form.get_all_int('make_empty')]}
+        step_fillers_to = self._form.get_all_of_key_prefixed('step_filler_to_')
+        to_update: dict[str, Any] = {
+            'comment': self._form.get_str_or_fail('comment', '')}
+        for k in ('is_done', 'calendarize'):
+            v = self._form.get_bool_or_none(k)
+            if v is not None:
+                to_update[k] = v
+        cond_rels = [self._form.get_all_int(name) for name in
+                     ['conditions', 'blockers', 'enables', 'disables']]
+        effort_or_not = self._form.get_str('effort')
+        if effort_or_not is not None:
+            if effort_or_not == '':
+                to_update['effort'] = None
+            else:
+                try:
+                    to_update['effort'] = float(effort_or_not)
+                except ValueError as e:
+                    msg = 'cannot float form field value for key: effort'
+                    raise BadFormatException(msg) from e
+        for k, fillers in step_fillers_to.items():
+            try:
+                parent_id = int(k)
+            except ValueError as e:
+                msg = f'bad step_filler_to_ key: {k}'
+                raise BadFormatException(msg) from e
+            for filler in [f for f in fillers if f != 'ignore']:
+                target_id: int
+                prefix = 'make_'
+                to_int = filler[5:] if filler.startswith(prefix) else filler
+                try:
+                    target_id = int(to_int)
+                except ValueError as e:
+                    msg = f'bad fill_for target: {filler}'
+                    raise BadFormatException(msg) from e
+                if filler.startswith(prefix):
+                    to_make['empty'] += [(target_id, parent_id)]
+                else:
+                    adoptees += [(target_id, parent_id)]
+        #
+        todo.set_condition_relations(self._conn, *cond_rels)
+        for parent in [Todo.by_id(self._conn, a[1])
+                       for a in adoptees] + [todo]:
+            for child in parent.children:
+                if child not in [t[0] for t in adoptees
+                                 if t[0] == child.id_ and t[1] == parent.id_]:
+                    parent.remove_child(child)
+                    parent.save(self._conn)
+        for child_id, parent_id in adoptees:
+            parent = Todo.by_id(self._conn, parent_id)
+            if child_id not in [c.id_ for c in parent.children]:
+                parent.add_child(Todo.by_id(self._conn, child_id))
+                parent.save(self._conn)
+        todo.update_attrs(**to_update)
+        for approach, make_data in to_make.items():
+            for process_id, parent_id in make_data:
+                parent = Todo.by_id(self._conn, parent_id)
+                process = Process.by_id(self._conn, process_id)
+                made = Todo(None, process, False, todo.date)
+                made.save(self._conn)
+                if 'full' == approach:
+                    made.ensure_children(self._conn)
+                parent.add_child(made)
+                parent.save(self._conn)
+        # todo.save() may destroy Todo if .effort < 0, so retrieve .id_ early
+        url = f'/todo?id={todo.id_}'
+        todo.save(self._conn)
+        return url
 
     def do_POST_process_descriptions(self) -> str:
         """Update history timestamps for Process.description."""
@@ -667,71 +723,53 @@ class TaskHandler(BaseHTTPRequestHandler):
     def do_POST_process(self, process: Process) -> str:
         """Update or insert Process of ?id= and fields defined in postvars."""
         # pylint: disable=too-many-locals
-        # pylint: disable=too-many-statements
-        title = self._form_data.get_str('title')
-        description = self._form_data.get_str('description')
-        effort = self._form_data.get_float('effort')
-        conditions = self._form_data.get_all_int('conditions')
-        blockers = self._form_data.get_all_int('blockers')
-        enables = self._form_data.get_all_int('enables')
-        disables = self._form_data.get_all_int('disables')
-        calendarize = self._form_data.get_all_str('calendarize') != []
-        suppresses = self._form_data.get_all_int('suppresses')
-        step_of = self._form_data.get_all_str('step_of')
-        keep_steps = self._form_data.get_all_int('keep_step')
-        step_ids = self._form_data.get_all_int('steps')
-        new_top_steps = self._form_data.get_all_str('new_top_step')
-        step_process_id_to = {}
-        step_parent_id_to = {}
+
+        def id_or_title(l_id_or_title: list[str]) -> tuple[str, list[int]]:
+            l_ids, title = [], ''
+            for id_or_title in l_id_or_title:
+                try:
+                    l_ids += [int(id_or_title)]
+                except ValueError:
+                    title = id_or_title
+            return title, l_ids
+
+        versioned = {'title': self._form.get_str_or_fail('title'),
+                     'description': self._form.get_str_or_fail('description'),
+                     'effort': self._form.get_float_or_fail('effort')}
+        cond_rels = [self._form.get_all_int(s) for s
+                     in ['conditions', 'blockers', 'enables', 'disables']]
+        calendarize = self._form.get_bool_or_none('calendarize')
+        step_of = self._form.get_all_str('step_of')
+        suppressions = self._form.get_all_int('suppresses')
+        kept_steps = self._form.get_all_int('kept_steps')
+        new_top_step_procs = self._form.get_all_str('new_top_step')
         new_steps_to = {}
-        for step_id in step_ids:
+        for step_id in kept_steps:
             name = f'new_step_to_{step_id}'
-            new_steps_to[step_id] = self._form_data.get_all_int(name)
-        for step_id in keep_steps:
-            name = f'step_{step_id}_process_id'
-            step_process_id_to[step_id] = self._form_data.get_int(name)
-            name = f'step_{step_id}_parent_id'
-            step_parent_id_to[step_id] = self._form_data.get_int_or_none(name)
-        process.title.set(title)
-        process.description.set(description)
-        process.effort.set(effort)
-        process.set_conditions(self.conn, conditions)
-        process.set_blockers(self.conn, blockers)
-        process.set_enables(self.conn, enables)
-        process.set_disables(self.conn, disables)
-        process.calendarize = calendarize
-        process.save(self.conn)
+            new_steps_to[step_id] = self._form.get_all_int(name)
+        new_owner_title, owners_to_set = id_or_title(step_of)
+        new_step_title, new_top_step_proc_ids = id_or_title(new_top_step_procs)
+        #
+        for k, v in versioned.items():
+            getattr(process, k).set(v)
+        if calendarize is not None:
+            process.calendarize = calendarize
+        process.save(self._conn)
         assert isinstance(process.id_, int)
-        new_step_title = None
-        steps: list[ProcessStep] = []
-        for step_id in keep_steps:
-            if step_id not in step_ids:
-                raise BadFormatException('trying to keep unknown step')
-            step = ProcessStep(step_id, process.id_,
-                               step_process_id_to[step_id],
-                               step_parent_id_to[step_id])
-            steps += [step]
-        for step_id in step_ids:
-            new = [ProcessStep(None, process.id_, step_process_id, step_id)
-                   for step_process_id in new_steps_to[step_id]]
-            steps += new
-        for step_identifier in new_top_steps:
-            try:
-                step_process_id = int(step_identifier)
-                step = ProcessStep(None, process.id_, step_process_id, None)
-                steps += [step]
-            except ValueError:
-                new_step_title = step_identifier
-        process.set_steps(self.conn, steps)
-        process.set_step_suppressions(self.conn, suppresses)
-        owners_to_set = []
-        new_owner_title = None
-        for owner_identifier in step_of:
-            try:
-                owners_to_set += [int(owner_identifier)]
-            except ValueError:
-                new_owner_title = owner_identifier
-        process.set_owners(self.conn, owners_to_set)
+        # set relations to Conditions and ProcessSteps / other Processes
+        process.set_condition_relations(self._conn, *cond_rels)
+        owned_steps = []
+        for step_id in kept_steps:
+            owned_steps += [ProcessStep.by_id(self._conn, step_id)]
+            owned_steps += [  # new sub-steps
+                    ProcessStep(None, process.id_, step_process_id, step_id)
+                    for step_process_id in new_steps_to[step_id]]
+        for step_process_id in new_top_step_proc_ids:
+            owned_steps += [ProcessStep(None, process.id_, step_process_id,
+                                        None)]
+        process.set_step_relations(self._conn, owners_to_set, suppressions,
+                                   owned_steps)
+        # encode titles for potential newly-to-create Processes up or down
         params = f'id={process.id_}'
         if new_step_title:
             title_b64_encoded = b64encode(new_step_title.encode()).decode()
@@ -739,7 +777,7 @@ class TaskHandler(BaseHTTPRequestHandler):
         elif new_owner_title:
             title_b64_encoded = b64encode(new_owner_title.encode()).decode()
             params = f'has_step={process.id_}&title_b64={title_b64_encoded}'
-        process.save(self.conn)
+        process.save(self._conn)
         return f'/process?{params}'
 
     def do_POST_condition_descriptions(self) -> str:
@@ -753,11 +791,13 @@ class TaskHandler(BaseHTTPRequestHandler):
     @_delete_or_post(Condition, '/conditions')
     def do_POST_condition(self, condition: Condition) -> str:
         """Update/insert Condition of ?id= and fields defined in postvars."""
-        is_active = self._form_data.get_str('is_active') == 'True'
-        title = self._form_data.get_str('title')
-        description = self._form_data.get_str('description')
-        condition.is_active = is_active
+        title = self._form.get_str_or_fail('title')
+        description = self._form.get_str_or_fail('description')
+        is_active = self._form.get_bool_or_none('is_active')
+        #
+        if is_active is not None:
+            condition.is_active = is_active
         condition.title.set(title)
         condition.description.set(description)
-        condition.save(self.conn)
+        condition.save(self._conn)
         return f'/condition?id={condition.id_}'