From: Christian Heller Date: Tue, 17 Feb 2015 05:23:49 +0000 (+0100) Subject: Server/py: Add tokenizer function. X-Git-Tag: tce~521 X-Git-Url: https://plomlompom.com/repos/template?a=commitdiff_plain;h=56867ba7a05f616174db6496ebc4115592ad6e6c;p=plomrogue Server/py: Add tokenizer function. --- diff --git a/plomrogue-server.py b/plomrogue-server.py index a1717a7..286d3eb 100755 --- a/plomrogue-server.py +++ b/plomrogue-server.py @@ -60,6 +60,44 @@ def obey(msg): print("Input: " + msg) +def tokenize(string): + """Divide string by " ", \t and quotes (that also group). Escape with \.""" + charlist_A = list(string) + i = 0 + for c in charlist_A: + if "\\" == c and i < len(charlist_A) - 1: + charlist_A[i] = "remove" + charlist_A[i + 1] = charlist_A[i + 1] + "_escaped" + i = i + 1 + charlist_B = [] + for c in charlist_A: + if "remove" != c: + charlist_B.append(c) + in_quotes = 0 + i = 0 + for c in charlist_B: + if "\"" == c: + in_quotes = 0 if in_quotes else 1 + if i < len(charlist_B) - 1: + charlist_B[i] = "separator" + elif (not in_quotes) and (" " == c or "\t" == c): + charlist_B[i] = "separator" + i = i + 1 + list_of_charlists = [[]] + i = 0 + for c in charlist_B: + if "separator" == c: + if [] != list_of_charlists[-1]: + list_of_charlists.append([]) + i = i + 1 + else: + list_of_charlists[i].append(c[0]) + tokens = [] + for charlist in list_of_charlists: + tokens.append("".join(charlist)) + return tokens + + io_db = {} try: parser = argparse.ArgumentParser()