Truncate words to dictionary resolution before lookup
This commit is contained in:
parent
6e62ca203b
commit
bbd70e8577
1 changed files with 5 additions and 1 deletions
|
|
@ -124,9 +124,13 @@ class ZLexer:
|
||||||
|
|
||||||
token_list = self._tokenise_string(string, separators)
|
token_list = self._tokenise_string(string, separators)
|
||||||
|
|
||||||
|
# Truncate to dictionary resolution (6 chars V1-3, 9 chars V4+)
|
||||||
|
max_word_len = 6 if self._memory.version <= 3 else 9
|
||||||
|
|
||||||
final_list = []
|
final_list = []
|
||||||
for word in token_list:
|
for word in token_list:
|
||||||
byte_addr = dict.get(word, 0)
|
lookup_key = word[:max_word_len]
|
||||||
|
byte_addr = dict.get(lookup_key, 0)
|
||||||
final_list.append([word, byte_addr])
|
final_list.append([word, byte_addr])
|
||||||
|
|
||||||
return final_list
|
return final_list
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue