CodeRay::Scanners::Scheme

     # File lib/coderay/scanners/scheme.rb, line 73
 73:       def scan_tokens tokens,options
 74:         
 75:         state = :initial
 76:         ident_kind = IDENT_KIND
 77:         
 78:         until eos?
 79:           kind = match = nil
 80:           
 81:           case state
 82:           when :initial
 83:             if scan(/ \s+ | \\\n /)
 84:               kind = :space
 85:             elsif scan(/['\(\[\)\]]|#\(/)
 86:               kind = :operator_fat
 87:             elsif scan(/;.*/)
 88:               kind = :comment
 89:             elsif scan(/#\\(?:newline|space|.?)/)
 90:               kind = :char
 91:             elsif scan(/#[ft]/)
 92:               kind = :pre_constant
 93:             elsif scan(/#{IDENTIFIER}/)
 94:               kind = ident_kind[matched]
 95:             elsif scan(/\./)
 96:               kind = :operator
 97:             elsif scan(/"/)
 98:               tokens << [:open, :string]
 99:               state = :string
100:               tokens << ['"', :delimiter]
101:               next
102:             elsif scan(/#{NUM}/) and not matched.empty?
103:               kind = :integer
104:             elsif getch
105:               kind = :error
106:             end
107:             
108:           when :string
109:             if scan(/[^"\\]+/) or scan(/\\.?/)
110:               kind = :content
111:             elsif scan(/"/)
112:               tokens << ['"', :delimiter]
113:               tokens << [:close, :string]
114:               state = :initial
115:               next
116:             else
117:               raise_inspect "else case \" reached; %p not handled." % peek(1),
118:                 tokens, state
119:             end
120:             
121:           else
122:             raise "else case reached"
123:           end
124:           
125:           match ||= matched
126:           if $CODERAY_DEBUG and not kind
127:             raise_inspect 'Error token %p in line %d' %
128:             [[match, kind], line], tokens
129:           end
130:           raise_inspect 'Empty token', tokens, state unless match
131:           
132:           tokens << [match, kind]
133:           
134:         end  # until eos
135:         
136:         if state == :string
137:           tokens << [:close, :string]
138:         end
139:         
140:         tokens
141:         
142:       end