13: def scan_tokens tokens, options
14:
15: opened_tokens = []
16:
17: until eos?
18:
19: kind = nil
20: match = nil
21:
22: if scan(/\s+/)
23: tokens << [matched, :space]
24: next
25:
26: elsif scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \) /x)
27: kind = self[1].to_sym
28: match = self[2].gsub(/\\(.)/, '\1')
29:
30: elsif scan(/ (\w+) < /x)
31: kind = self[1].to_sym
32: opened_tokens << kind
33: match = :open
34:
35: elsif !opened_tokens.empty? && scan(/ > /x)
36: kind = opened_tokens.pop || :error
37: match = :close
38:
39: else
40: kind = :error
41: getch
42:
43: end
44:
45: match ||= matched
46: if $CODERAY_DEBUG and not kind
47: raise_inspect 'Error token %p in line %d' %
48: [[match, kind], line], tokens
49: end
50: raise_inspect 'Empty token', tokens unless match
51:
52: tokens << [match, kind]
53:
54: end
55:
56: tokens
57: end