From: Christian Heller <c.heller@plomlompom.de>
Date: Tue, 17 Feb 2015 05:23:49 +0000 (+0100)
Subject: Server/py: Add tokenizer function.
X-Git-Tag: tce~521
X-Git-Url: https://plomlompom.com/repos/%7B%7Bprefix%7D%7D/static/%7B%7Byoutube_prefix%7D%7D%7B%7Bvideo_data.id_%7D%7D?a=commitdiff_plain;h=56867ba7a05f616174db6496ebc4115592ad6e6c;p=plomrogue

Server/py: Add tokenizer function.
---

diff --git a/plomrogue-server.py b/plomrogue-server.py
index a1717a7..286d3eb 100755
--- a/plomrogue-server.py
+++ b/plomrogue-server.py
@@ -60,6 +60,44 @@ def obey(msg):
     print("Input: " + msg)
 
 
+def tokenize(string):
+    """Divide string by " ", \t and quotes (that also group). Escape with \."""
+    charlist_A = list(string)
+    i = 0
+    for c in charlist_A:
+        if "\\" == c and i < len(charlist_A) - 1:
+            charlist_A[i] = "remove"
+            charlist_A[i + 1] = charlist_A[i + 1] + "_escaped"
+        i = i + 1
+    charlist_B = []
+    for c in charlist_A:
+        if "remove" != c:
+            charlist_B.append(c)
+    in_quotes = 0
+    i = 0
+    for c in charlist_B:
+        if "\"" == c:
+            in_quotes = 0 if in_quotes else 1
+            if i < len(charlist_B) - 1:
+                charlist_B[i] = "separator"
+        elif (not in_quotes) and (" " == c or "\t" == c):
+            charlist_B[i] = "separator"
+        i = i + 1
+    list_of_charlists = [[]]
+    i = 0
+    for c in charlist_B:
+        if "separator" == c:
+            if [] != list_of_charlists[-1]:
+                list_of_charlists.append([])
+                i = i + 1
+        else:
+            list_of_charlists[i].append(c[0])
+    tokens = []
+    for charlist in list_of_charlists:
+        tokens.append("".join(charlist))
+    return tokens
+
+
 io_db = {}
 try:
     parser = argparse.ArgumentParser()