CodeRay::Scanners::Delphi

     # File lib/coderay/scanners/delphi.rb, line 41
 41:     def scan_tokens tokens, options
 42: 
 43:       state = :initial
 44:       last_token = ''
 45: 
 46:       until eos?
 47: 
 48:         kind = nil
 49:         match = nil
 50: 
 51:         if state == :initial
 52:           
 53:           if scan(/ \s+ /)
 54:             tokens << [matched, :space]
 55:             next
 56:             
 57:           elsif scan(% \{ \$ [^}]* \}? | \(\* \$ (?: .*? \*\) | .* ) !x)
 58:             tokens << [matched, :preprocessor]
 59:             next
 60:             
 61:           elsif scan(% // [^\n]* | \{ [^}]* \}? | \(\* (?: .*? \*\) | .* ) !x)
 62:             tokens << [matched, :comment]
 63:             next
 64:             
 65:           elsif match = scan(/ <[>=]? | >=? | :=? | [-+=*\/;,@\^|\(\)\[\]] | \.\. /)
 66:             kind = :operator
 67:           
 68:           elsif match = scan(/\./)
 69:             kind = :operator
 70:             if last_token == 'end'
 71:               tokens << [match, kind]
 72:               next
 73:             end
 74:             
 75:           elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /)
 76:             kind = NAME_FOLLOWS[last_token] ? :ident : IDENT_KIND[match]
 77:             
 78:           elsif match = scan(/ ' ( [^\n']|'' ) (?:'|$) /)
 79:             tokens << [:open, :char]
 80:             tokens << ["'", :delimiter]
 81:             tokens << [self[1], :content]
 82:             tokens << ["'", :delimiter]
 83:             tokens << [:close, :char]
 84:             next
 85:             
 86:           elsif match = scan(/ ' /)
 87:             tokens << [:open, :string]
 88:             state = :string
 89:             kind = :delimiter
 90:             
 91:           elsif scan(/ \# (?: \d+ | \$[0-9A-Fa-f]+ ) /)
 92:             kind = :char
 93:             
 94:           elsif scan(/ \$ [0-9A-Fa-f]+ /)
 95:             kind = :hex
 96:             
 97:           elsif scan(/ (?: \d+ ) (?![eE]|\.[^.]) /)
 98:             kind = :integer
 99:             
100:           elsif scan(/ \d+ (?: \.\d+ (?: [eE][+-]? \d+ )? | [eE][+-]? \d+ ) /)
101:             kind = :float
102: 
103:           else
104:             kind = :error
105:             getch
106: 
107:           end
108:           
109:         elsif state == :string
110:           if scan(/[^\n']+/)
111:             kind = :content
112:           elsif scan(/''/)
113:             kind = :char
114:           elsif scan(/'/)
115:             tokens << ["'", :delimiter]
116:             tokens << [:close, :string]
117:             state = :initial
118:             next
119:           elsif scan(/\n/)
120:             tokens << [:close, :string]
121:             kind = :error
122:             state = :initial
123:           else
124:             raise "else case \' reached; %p not handled." % peek(1), tokens
125:           end
126:           
127:         else
128:           raise 'else-case reached', tokens
129:           
130:         end
131:         
132:         match ||= matched
133:         if $CODERAY_DEBUG and not kind
134:           raise_inspect 'Error token %p in line %d' %
135:             [[match, kind], line], tokens, state
136:         end
137:         raise_inspect 'Empty token', tokens unless match
138: 
139:         last_token = match
140:         tokens << [match, kind]
141:         
142:       end
143:       
144:       tokens
145:     end