CodeRay::Scanners::JavaScript

     # File lib/coderay/scanners/java_script.rb, line 54
 54:     def scan_tokens tokens, options
 55: 
 56:       state = :initial
 57:       string_delimiter = nil
 58:       value_expected = true
 59:       key_expected = false
 60:       function_expected = false
 61: 
 62:       until eos?
 63: 
 64:         kind = nil
 65:         match = nil
 66:         
 67:         case state
 68: 
 69:         when :initial
 70: 
 71:           if match = scan(/ \s+ | \\\n /)
 72:             value_expected = true if !value_expected && match.index(\n\)
 73:             tokens << [match, :space]
 74:             next
 75: 
 76:           elsif scan(% // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !x)
 77:             value_expected = true
 78:             kind = :comment
 79: 
 80:           elsif check(/\.?\d/)
 81:             key_expected = value_expected = false
 82:             if scan(/0[xX][0-9A-Fa-f]+/)
 83:               kind = :hex
 84:             elsif scan(/(?>0[0-7]+)(?![89.eEfF])/)
 85:               kind = :oct
 86:             elsif scan(/\d+[fF]|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
 87:               kind = :float
 88:             elsif scan(/\d+/)
 89:               kind = :integer
 90:             end
 91:           
 92:           elsif value_expected && match = scan(/<([[:alpha:]]\w*) (?: [^\/>]*\/> | .*?<\/\11>>)/im)
 93:             # FIXME: scan over nested tags
 94:             xml_scanner.tokenize match
 95:             value_expected = false
 96:             next
 97:             
 98:           elsif match = scan(/ [-+*=<>?:;,!&^|(\[{~%]+ | \.(?!\d) /)
 99:             value_expected = true
100:             last_operator = match[1]
101:             key_expected = (last_operator == {{) || (last_operator == ,,)
102:             function_expected = false
103:             kind = :operator
104: 
105:           elsif scan(/ [)\]}]+ /)
106:             function_expected = key_expected = value_expected = false
107:             kind = :operator
108: 
109:           elsif match = scan(/ [$a-zA-Z_][A-Za-z_0-9$]* /)
110:             kind = IDENT_KIND[match]
111:             value_expected = (kind == :keyword) && KEYWORDS_EXPECTING_VALUE[match]
112:             # TODO: labels
113:             if kind == :ident
114:               if match.index($$)  # $ allowed inside an identifier
115:                 kind = :predefined
116:               elsif function_expected
117:                 kind = :function
118:               elsif check(/\s*[=:]\s*function\b/)
119:                 kind = :function
120:               elsif key_expected && check(/\s*:/)
121:                 kind = :key
122:               end
123:             end
124:             function_expected = (kind == :keyword) && (match == 'function')
125:             key_expected = false
126:           
127:           elsif match = scan(/["']/)
128:             if key_expected && check(KEY_CHECK_PATTERN[match])
129:               state = :key
130:             else
131:               state = :string
132:             end
133:             tokens << [:open, state]
134:             string_delimiter = match
135:             kind = :delimiter
136: 
137:           elsif value_expected && (match = scan(/\/(?=\S)/))
138:             tokens << [:open, :regexp]
139:             state = :regexp
140:             string_delimiter = '/'
141:             kind = :delimiter
142: 
143:           elsif scan(/ \/ /)
144:             value_expected = true
145:             key_expected = false
146:             kind = :operator
147: 
148:           else
149:             getch
150:             kind = :error
151: 
152:           end
153: 
154:         when :string, :regexp, :key
155:           if scan(STRING_CONTENT_PATTERN[string_delimiter])
156:             kind = :content
157:           elsif match = scan(/["'\/]/)
158:             tokens << [match, :delimiter]
159:             if state == :regexp
160:               modifiers = scan(/[gim]+/)
161:               tokens << [modifiers, :modifier] if modifiers && !modifiers.empty?
162:             end
163:             tokens << [:close, state]
164:             string_delimiter = nil
165:             key_expected = value_expected = false
166:             state = :initial
167:             next
168:           elsif state != :regexp && (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /ox))
169:             if string_delimiter == "'" && !(match == "\\\\" || match == "\\'")
170:               kind = :content
171:             else
172:               kind = :char
173:             end
174:           elsif state == :regexp && scan(/ \\ (?: #{ESCAPE} | #{REGEXP_ESCAPE} | #{UNICODE_ESCAPE} ) /ox)
175:             kind = :char
176:           elsif scan(/\\./)
177:             kind = :content
178:           elsif scan(/ \\ | $ /)
179:             tokens << [:close, state]
180:             kind = :error
181:             key_expected = value_expected = false
182:             state = :initial
183:           else
184:             raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
185:           end
186: 
187:         else
188:           raise_inspect 'Unknown state', tokens
189: 
190:         end
191: 
192:         match ||= matched
193:         if $CODERAY_DEBUG and not kind
194:           raise_inspect 'Error token %p in line %d' %
195:             [[match, kind], line], tokens
196:         end
197:         raise_inspect 'Empty token', tokens unless match
198:         
199:         tokens << [match, kind]
200: 
201:       end
202: 
203:       if [:string, :regexp].include? state
204:         tokens << [:close, state]
205:       end
206: 
207:       tokens
208:     end

     # File lib/coderay/scanners/java_script.rb, line 212
212:     def reset_instance
213:       super
214:       @xml_scanner.reset if defined? @xml_scanner
215:     end

     # File lib/coderay/scanners/java_script.rb, line 217
217:     def xml_scanner
218:       @xml_scanner ||= CodeRay.scanner :xml, :tokens => @tokens, :keep_tokens => true, :keep_state => false
219:     end