CodeRay::Scanners::CPlusPlus

     # File lib/coderay/scanners/cpp.rb, line 51
 51:     def scan_tokens tokens, options
 52: 
 53:       state = :initial
 54:       label_expected = true
 55:       case_expected = false
 56:       label_expected_before_preproc_line = nil
 57:       in_preproc_line = false
 58: 
 59:       until eos?
 60: 
 61:         kind = nil
 62:         match = nil
 63:         
 64:         case state
 65: 
 66:         when :initial
 67: 
 68:           if match = scan(/ \s+ | \\\n /)
 69:             if in_preproc_line && match != "\\\n" && match.index(\n\)
 70:               in_preproc_line = false
 71:               label_expected = label_expected_before_preproc_line
 72:             end
 73:             tokens << [match, :space]
 74:             next
 75: 
 76:           elsif scan(% // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !x)
 77:             kind = :comment
 78: 
 79:           elsif match = scan(/ \# \s* if \s* 0 /)
 80:             match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /m) unless eos?
 81:             kind = :comment
 82: 
 83:           elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /)
 84:             label_expected = match =~ /[;\{\}]/
 85:             if case_expected
 86:               label_expected = true if match == ':'
 87:               case_expected = false
 88:             end
 89:             kind = :operator
 90: 
 91:           elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /)
 92:             kind = IDENT_KIND[match]
 93:             if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/)
 94:               kind = :label
 95:               match << matched
 96:             else
 97:               label_expected = false
 98:               if kind == :reserved
 99:                 case match
100:                 when 'class'
101:                   state = :class_name_expected
102:                 when 'case', 'default'
103:                   case_expected = true
104:                 end
105:               end
106:             end
107: 
108:           elsif scan(/\$/)
109:             kind = :ident
110:           
111:           elsif match = scan(/L?"/)
112:             tokens << [:open, :string]
113:             if match[0] == LL
114:               tokens << ['L', :modifier]
115:               match = '"'
116:             end
117:             state = :string
118:             kind = :delimiter
119: 
120:           elsif scan(/#[ \t]*(\w*)/)
121:             kind = :preprocessor
122:             in_preproc_line = true
123:             label_expected_before_preproc_line = label_expected
124:             state = :include_expected if self[1] == 'include'
125: 
126:           elsif scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /x)
127:             label_expected = false
128:             kind = :char
129: 
130:           elsif scan(/0[xX][0-9A-Fa-f]+/)
131:             label_expected = false
132:             kind = :hex
133: 
134:           elsif scan(/(?:0[0-7]+)(?![89.eEfF])/)
135:             label_expected = false
136:             kind = :oct
137: 
138:           elsif scan(/(?:\d+)(?![.eEfF])L?L?/)
139:             label_expected = false
140:             kind = :integer
141: 
142:           elsif scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
143:             label_expected = false
144:             kind = :float
145: 
146:           else
147:             getch
148:             kind = :error
149: 
150:           end
151: 
152:         when :string
153:           if scan(/[^\\"]+/)
154:             kind = :content
155:           elsif scan(/"/)
156:             tokens << ['"', :delimiter]
157:             tokens << [:close, :string]
158:             state = :initial
159:             label_expected = false
160:             next
161:           elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /ox)
162:             kind = :char
163:           elsif scan(/ \\ | $ /)
164:             tokens << [:close, :string]
165:             kind = :error
166:             state = :initial
167:             label_expected = false
168:           else
169:             raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
170:           end
171: 
172:         when :include_expected
173:           if scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/)
174:             kind = :include
175:             state = :initial
176: 
177:           elsif match = scan(/\s+/)
178:             kind = :space
179:             state = :initial if match.index \n\
180: 
181:           else
182:             state = :initial
183:             next
184: 
185:           end
186:         
187:         when :class_name_expected
188:           if scan(/ [A-Za-z_][A-Za-z_0-9]* /)
189:             kind = :class
190:             state = :initial
191: 
192:           elsif match = scan(/\s+/)
193:             kind = :space
194: 
195:           else
196:             getch
197:             kind = :error
198:             state = :initial
199: 
200:           end
201:           
202:         else
203:           raise_inspect 'Unknown state', tokens
204: 
205:         end
206: 
207:         match ||= matched
208:         if $CODERAY_DEBUG and not kind
209:           raise_inspect 'Error token %p in line %d' %
210:             [[match, kind], line], tokens
211:         end
212:         raise_inspect 'Empty token', tokens unless match
213: 
214:         tokens << [match, kind]
215: 
216:       end
217: 
218:       if state == :string
219:         tokens << [:close, :string]
220:       end
221: 
222:       tokens
223:     end