# File parsers/parse_rb.rb, line 1387
1387:     def initialize(top_level, file_name, content, options, stats)
1388:       @options = options
1389:       @stats   = stats
1390:       @size = 0
1391:       @token_listeners = nil
1392:       @input_file_name = file_name
1393:       @scanner = RubyLex.new(content)
1394:       @scanner.exception_on_syntax_error = false
1395:       @top_level = top_level
1396:       @progress = $stderr unless options.quiet
1397:     end

      # File parsers/parse_rb.rb, line 1456
1456:     def add_token_listener(obj)
1457:       @token_listeners ||= []
1458:       @token_listeners << obj
1459:     end

      # File parsers/parse_rb.rb, line 1546
1546:     def collect_first_comment
1547:       skip_tkspace
1548:       res = ''
1549:       first_line = true
1550: 
1551:       tk = get_tk
1552:       while tk.kind_of?(TkCOMMENT)
1553:         if first_line && tk.text[0,2] == "#!"
1554:           skip_tkspace
1555:           tk = get_tk
1556:         else
1557:           res << tk.text << "\n"
1558:           tk = get_tk
1559:           if tk.kind_of? TkNL
1560:             skip_tkspace(false)
1561:             tk = get_tk
1562:           end
1563:         end
1564:         first_line = false
1565:       end
1566:       unget_tk(tk)
1567:       res
1568:     end

      # File parsers/parse_rb.rb, line 1443
1443:     def error(msg)
1444:       msg = make_message msg
1445:       $stderr.puts msg
1446:       exit(1)
1447:     end

      # File parsers/parse_rb.rb, line 2446
2446:     def get_bool
2447:       skip_tkspace
2448:       tk = get_tk
2449:       case tk
2450:       when TkTRUE
2451:         true
2452:       when TkFALSE, TkNIL
2453:         false
2454:       else
2455:         unget_tk tk
2456:         true
2457:       end
2458:     end

      # File parsers/parse_rb.rb, line 1801
1801:     def get_class_or_module(container)
1802:       skip_tkspace
1803:       name_t = get_tk
1804: 
1805:       # class ::A -> A is in the top level
1806:       if name_t.kind_of?(TkCOLON2)
1807:         name_t = get_tk
1808:         container = @top_level
1809:       end
1810: 
1811:       skip_tkspace(false)
1812: 
1813:       while peek_tk.kind_of?(TkCOLON2)
1814:         prev_container = container
1815:         container = container.find_module_named(name_t.name)
1816:         if !container
1817: #          warn("Couldn't find module #{name_t.name}")
1818:           container = prev_container.add_module(NormalModule, name_t.name)
1819:         end
1820:         get_tk
1821:         name_t = get_tk
1822:       end
1823:       skip_tkspace(false)
1824:       return [container, name_t]
1825:     end

      # File parsers/parse_rb.rb, line 2130
2130:     def get_class_specification
2131:       tk = get_tk
2132:       return "self" if tk.kind_of?(TkSELF)
2133:         
2134:       res = ""
2135:       while tk.kind_of?(TkCOLON2) ||
2136:           tk.kind_of?(TkCOLON3)   ||
2137:           tk.kind_of?(TkCONSTANT)   
2138:         
2139:         res += tk.text
2140:         tk = get_tk
2141:       end
2142: 
2143:       unget_tk(tk)
2144:       skip_tkspace(false)
2145: 
2146:       get_tkread # empty out read buffer
2147: 
2148:       tk = get_tk
2149: 
2150:       case tk
2151:       when TkNL, TkCOMMENT, TkSEMICOLON
2152:         unget_tk(tk)
2153:         return res
2154:       end
2155: 
2156:       res += parse_call_parameters(tk)
2157:       res
2158:     end

      # File parsers/parse_rb.rb, line 2202
2202:     def get_constant
2203:       res = ""
2204:       skip_tkspace(false)
2205:       tk = get_tk
2206: 
2207:       while tk.kind_of?(TkCOLON2) ||
2208:           tk.kind_of?(TkCOLON3)   ||
2209:           tk.kind_of?(TkCONSTANT)          
2210:         
2211:         res += tk.text
2212:         tk = get_tk
2213:       end
2214: 
2215: #      if res.empty?
2216: #        warn("Unexpected token #{tk} in constant")
2217: #      end 
2218:       unget_tk(tk)
2219:       res
2220:     end

      # File parsers/parse_rb.rb, line 2224
2224:     def get_constant_with_optional_parens
2225:       skip_tkspace(false)
2226:       nest = 0
2227:       while (tk = peek_tk).kind_of?(TkLPAREN)  || tk.kind_of?(TkfLPAREN)
2228:         get_tk
2229:         skip_tkspace(true)
2230:         nest += 1
2231:       end
2232: 
2233:       name = get_constant
2234: 
2235:       while nest > 0
2236:         skip_tkspace(true)
2237:         tk = get_tk
2238:         nest -= 1 if tk.kind_of?(TkRPAREN)
2239:       end
2240:       name
2241:     end

      # File parsers/parse_rb.rb, line 2360
2360:     def get_symbol_or_name
2361:       tk = get_tk
2362:       case tk
2363:       when  TkSYMBOL
2364:         tk.text.sub(/^:/, '')
2365:       when TkId, TkOp
2366:         tk.name
2367:       when TkSTRING
2368:         tk.text
2369:       else
2370:         raise "Name or symbol expected (got #{tk})"
2371:       end
2372:     end

      # File parsers/parse_rb.rb, line 1465
1465:     def get_tk
1466:       tk = nil
1467:       if @tokens.empty?
1468:         tk = @scanner.token
1469:         @read.push @scanner.get_read
1470:         puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG
1471:       else
1472:         @read.push @unget_read.shift
1473:         tk = @tokens.shift
1474:         puts "get_tk2 => #{tk.inspect}" if $TOKEN_DEBUG
1475:       end
1476: 
1477:       if tk.kind_of?(TkSYMBEG)
1478:         set_token_position(tk.line_no, tk.char_no)
1479:         tk1 = get_tk
1480:         if tk1.kind_of?(TkId) || tk1.kind_of?(TkOp) || tk1.kind_of?(TkSTRING)
1481:           if tk1.respond_to?(:name)
1482:             tk = Token(TkSYMBOL).set_text(":" + tk1.name)
1483:           else
1484:             tk = Token(TkSYMBOL).set_text(":" + tk1.text)
1485:           end
1486:           # remove the identifier we just read (we're about to
1487:           # replace it with a symbol)
1488:           @token_listeners.each do |obj|
1489:             obj.pop_token
1490:           end if @token_listeners
1491:         else
1492:           warn("':' not followed by identifier or operator")
1493:           tk = tk1
1494:         end
1495:       end
1496: 
1497:       # inform any listeners of our shiny new token
1498:       @token_listeners.each do |obj|
1499:         obj.add_token(tk)
1500:       end if @token_listeners
1501: 
1502:       tk
1503:     end

      # File parsers/parse_rb.rb, line 1530
1530:     def get_tkread
1531:       read = @read.join("")
1532:       @read = []
1533:       read
1534:     end

      # File parsers/parse_rb.rb, line 2309
2309:     def look_for_directives_in(context, comment)
2310: 
2311:       preprocess = SM::PreProcess.new(@input_file_name,
2312:                                       @options.rdoc_include)
2313: 
2314:       preprocess.handle(comment) do |directive, param|
2315:         case directive
2316:         when "stopdoc"
2317:           context.stop_doc
2318:           ""
2319:         when "startdoc"
2320:           context.start_doc
2321:           context.force_documentation = true
2322:           ""
2323: 
2324:         when "enddoc"
2325:           #context.done_documenting = true
2326:           #""
2327:           throw :enddoc
2328: 
2329:         when "main"
2330:           options = Options.instance
2331:           options.main_page = param
2332:           ""
2333: 
2334:         when "title"
2335:           options = Options.instance
2336:           options.title = param
2337:           ""
2338: 
2339:         when "section"
2340:           context.set_current_section(param, comment)
2341: #          comment.clear # ruby 1.9 feature
2342:           comment.replace("") # 1.8 doesn't support #clear
2343:           break 
2344:         else
2345:           warn "Unrecognized directive '#{directive}'"
2346:           break
2347:         end
2348:       end
2349: 
2350:       remove_private_comments(comment)
2351:     end

      # File parsers/parse_rb.rb, line 1429
1429:     def make_message(msg)
1430:       prefix = "\n" + @input_file_name + ":"
1431:       if @scanner
1432:         prefix << "#{@scanner.line_no}:#{@scanner.char_no}: "
1433:       end
1434:       return prefix + msg
1435:     end

      # File parsers/parse_rb.rb, line 2374
2374:     def parse_alias(context, single, tk, comment)
2375:       skip_tkspace
2376:       if (peek_tk.kind_of? TkLPAREN)
2377:         get_tk
2378:         skip_tkspace
2379:       end
2380:       new_name = get_symbol_or_name
2381:       @scanner.instance_eval{@lex_state = EXPR_FNAME}
2382:       skip_tkspace
2383:       if (peek_tk.kind_of? TkCOMMA)
2384:         get_tk
2385:         skip_tkspace
2386:       end
2387:       old_name = get_symbol_or_name
2388: 
2389:       al = Alias.new(get_tkread, old_name, new_name, comment)
2390:       read_documentation_modifiers(al, ATTR_MODIFIERS)
2391:       if al.document_self
2392:         context.add_alias(al)
2393:       end
2394:     end

      # File parsers/parse_rb.rb, line 2460
2460:     def parse_attr(context, single, tk, comment)
2461:       args = parse_symbol_arg(1)
2462:       if args.size > 0
2463:         name = args[0]
2464:         rw = "R"
2465:         skip_tkspace(false)
2466:         tk = get_tk
2467:         if tk.kind_of? TkCOMMA
2468:           rw = "RW" if get_bool
2469:         else
2470:           unget_tk tk
2471:         end
2472:         att = Attr.new(get_tkread, name, rw, comment)
2473:         read_documentation_modifiers(att, ATTR_MODIFIERS)
2474:         if att.document_self
2475:           context.add_attribute(att)
2476:         end
2477:       else
2478:         warn("'attr' ignored - looks like a variable")
2479:       end    
2480: 
2481:     end

      # File parsers/parse_rb.rb, line 2514
2514:     def parse_attr_accessor(context, single, tk, comment)
2515:       args = parse_symbol_arg
2516:       read = get_tkread
2517:       rw = "?"
2518: 
2519:       # If nodoc is given, don't document any of them
2520: 
2521:       tmp = CodeObject.new
2522:       read_documentation_modifiers(tmp, ATTR_MODIFIERS)
2523:       return unless tmp.document_self
2524: 
2525:       case tk.name
2526:       when "attr_reader"   then rw = "R"
2527:       when "attr_writer"   then rw = "W"
2528:       when "attr_accessor" then rw = "RW"
2529:       else
2530:         rw = @options.extra_accessor_flags[tk.name]
2531:       end
2532:       
2533:       for name in args
2534:         att = Attr.new(get_tkread, name, rw, comment)
2535:         context.add_attribute(att)
2536:       end    
2537:     end

      # File parsers/parse_rb.rb, line 2160
2160:     def parse_call_parameters(tk)
2161: 
2162:       end_token = case tk
2163:                   when TkLPAREN, TkfLPAREN
2164:                     TkRPAREN
2165:                   when TkRPAREN
2166:                     return ""
2167:                   else
2168:                     TkNL
2169:                   end
2170:       nest = 0
2171: 
2172:       loop do
2173:         puts("Call param: #{tk}, #{@scanner.continue} " +
2174:           "#{@scanner.lex_state} #{nest}") if $DEBUG
2175:         case tk
2176:         when TkSEMICOLON
2177:           break
2178:         when TkLPAREN, TkfLPAREN
2179:           nest += 1
2180:         when end_token
2181:           if end_token == TkRPAREN
2182:             nest -= 1
2183:             break if @scanner.lex_state == EXPR_END and nest <= 0
2184:           else
2185:             break unless @scanner.continue
2186:           end
2187:         when TkCOMMENT
2188:           unget_tk(tk)
2189:           break
2190:         end
2191:         tk = get_tk
2192:       end
2193:       res = get_tkread.tr("\n", " ").strip
2194:       res = "" if res == ";"
2195:       res
2196:     end

      # File parsers/parse_rb.rb, line 1733
1733:     def parse_class(container, single, tk, comment, &block)
1734:       progress("c")
1735: 
1736:       @stats.num_classes += 1
1737: 
1738:       container, name_t = get_class_or_module(container)
1739: 
1740:       case name_t
1741:       when TkCONSTANT
1742:         name = name_t.name
1743:         superclass = "Object"
1744: 
1745:         if peek_tk.kind_of?(TkLT)
1746:           get_tk
1747:           skip_tkspace(true)
1748:           superclass = get_class_specification
1749:           superclass = "<unknown>" if superclass.empty?
1750:         end
1751: 
1752:         if single == SINGLE
1753:           cls_type = SingleClass
1754:         else
1755:           cls_type = NormalClass
1756:         end
1757: 
1758:         cls = container.add_class(cls_type, name, superclass)
1759:         read_documentation_modifiers(cls, CLASS_MODIFIERS)
1760:         cls.record_location(@top_level)
1761:         parse_statements(cls)
1762:         cls.comment = comment
1763: 
1764:       when TkLSHFT
1765:         case name = get_class_specification
1766:         when "self", container.name
1767:           parse_statements(container, SINGLE, &block)
1768:         else
1769:           other = TopLevel.find_class_named(name)
1770:           unless other
1771: #            other = @top_level.add_class(NormalClass, name, nil)
1772: #            other.record_location(@top_level)
1773: #            other.comment = comment
1774:             other = NormalClass.new("Dummy", nil)
1775:           end
1776:           read_documentation_modifiers(other, CLASS_MODIFIERS)
1777:           parse_statements(other, SINGLE, &block)
1778:         end
1779: 
1780:       else
1781:         warn("Expected class name or '<<'. Got #{name_t.class}: #{name_t.text.inspect}")
1782:       end
1783:     end

      # File parsers/parse_rb.rb, line 1827
1827:     def parse_constant(container, single, tk, comment)
1828:       name = tk.name
1829:       skip_tkspace(false)
1830:       eq_tk = get_tk
1831: 
1832:       unless eq_tk.kind_of?(TkASSIGN)
1833:         unget_tk(eq_tk)
1834:         return
1835:       end
1836: 
1837: 
1838:       nest = 0
1839:       get_tkread
1840: 
1841:       tk = get_tk
1842:       if tk.kind_of? TkGT
1843:         unget_tk(tk)
1844:         unget_tk(eq_tk)
1845:         return
1846:       end
1847: 
1848:       loop do
1849:         puts("Param: #{tk}, #{@scanner.continue} " +
1850:           "#{@scanner.lex_state} #{nest}")  if $DEBUG
1851: 
1852:         case tk
1853:         when TkSEMICOLON
1854:           break
1855:         when TkLPAREN, TkfLPAREN
1856:           nest += 1
1857:         when TkRPAREN
1858:           nest -= 1
1859:         when TkCOMMENT
1860:           if nest <= 0 && @scanner.lex_state == EXPR_END
1861:             unget_tk(tk)
1862:             break
1863:           end
1864:         when TkNL
1865:           if (@scanner.lex_state == EXPR_END and nest <= 0) || !@scanner.continue
1866:             unget_tk(tk)
1867:             break
1868:           end
1869:         end
1870:         tk = get_tk
1871:       end
1872: 
1873:       res = get_tkread.tr("\n", " ").strip
1874:       res = "" if res == ";"
1875:       con = Constant.new(name, res, comment)
1876:       read_documentation_modifiers(con, CONSTANT_MODIFIERS)
1877:       if con.document_self
1878:         container.add_constant(con)
1879:       end
1880:     end

      # File parsers/parse_rb.rb, line 2434
2434:   def parse_include(context, comment)
2435:     loop do
2436:       skip_tkspace_comment
2437:       name = get_constant_with_optional_parens
2438:       unless name.empty?
2439:         context.add_include(Include.new(name, comment))
2440:       end
2441:       return unless peek_tk.kind_of?(TkCOMMA)
2442:       get_tk
2443:     end
2444:   end

      # File parsers/parse_rb.rb, line 1882
1882:     def parse_method(container, single, tk, comment)
1883:       progress(".")
1884:       @stats.num_methods += 1
1885:       line_no = tk.line_no
1886:       column  = tk.char_no
1887:       
1888:       start_collecting_tokens
1889:       add_token(tk)
1890:       add_token_listener(self)
1891:       
1892:       @scanner.instance_eval{@lex_state = EXPR_FNAME}
1893:       skip_tkspace(false)
1894:       name_t = get_tk
1895:       back_tk = skip_tkspace
1896:       meth = nil
1897:       added_container = false
1898: 
1899:       dot = get_tk
1900:       if dot.kind_of?(TkDOT) or dot.kind_of?(TkCOLON2)
1901:         @scanner.instance_eval{@lex_state = EXPR_FNAME}
1902:         skip_tkspace
1903:         name_t2 = get_tk
1904:         case name_t
1905:         when TkSELF
1906:           name = name_t2.name
1907:         when TkCONSTANT
1908:           name = name_t2.name
1909:           prev_container = container
1910:           container = container.find_module_named(name_t.name)
1911:           if !container
1912:             added_container = true
1913:             obj = name_t.name.split("::").inject(Object) do |state, item|
1914:               state.const_get(item)
1915:             end rescue nil
1916: 
1917:             type = obj.class == Class ? NormalClass : NormalModule
1918:             if not [Class, Module].include?(obj.class)
1919:               warn("Couldn't find #{name_t.name}. Assuming it's a module")
1920:             end
1921: 
1922:             if type == NormalClass then
1923:               container = prev_container.add_class(type, name_t.name, obj.superclass.name)
1924:             else
1925:               container = prev_container.add_module(type, name_t.name)
1926:             end
1927:           end
1928:         else
1929:           # warn("Unexpected token '#{name_t2.inspect}'")
1930:           # break
1931:           skip_method(container)
1932:           return
1933:         end
1934:         meth =  AnyMethod.new(get_tkread, name)
1935:         meth.singleton = true
1936:       else
1937:         unget_tk dot
1938:         back_tk.reverse_each do
1939:           |tk|
1940:           unget_tk tk
1941:         end
1942:         name = name_t.name
1943: 
1944:         meth =  AnyMethod.new(get_tkread, name)
1945:         meth.singleton = (single == SINGLE)
1946:       end
1947: 
1948:       remove_token_listener(self)
1949: 
1950:       meth.start_collecting_tokens
1951:       indent = TkSPACE.new(1,1)
1952:       indent.set_text(" " * column)
1953: 
1954:       meth.add_tokens([TkCOMMENT.new(line_no,
1955:                                      1,
1956:                                      "# File #{@top_level.file_absolute_name}, line #{line_no}"),
1957:                         NEWLINE_TOKEN,
1958:                         indent])
1959: 
1960:       meth.add_tokens(@token_stream)
1961: 
1962:       add_token_listener(meth)
1963: 
1964:       @scanner.instance_eval{@continue = false}
1965:       parse_method_parameters(meth)
1966: 
1967:       if meth.document_self
1968:         container.add_method(meth)
1969:       elsif added_container
1970:         container.document_self = false
1971:       end
1972: 
1973:       # Having now read the method parameters and documentation modifiers, we
1974:       # now know whether we have to rename #initialize to ::new
1975: 
1976:       if name == "initialize" && !meth.singleton
1977:         if meth.dont_rename_initialize
1978:           meth.visibility = :protected
1979:         else
1980:           meth.singleton = true
1981:           meth.name = "new"
1982:           meth.visibility = :public
1983:         end
1984:       end
1985:       
1986:       parse_statements(container, single, meth)
1987:       
1988:       remove_token_listener(meth)
1989: 
1990:       # Look for a 'call-seq' in the comment, and override the
1991:       # normal parameter stuff
1992: 
1993:       if comment.sub!(/:?call-seq:(.*?)^\s*\#?\s*$/m, '')
1994:         seq = $1
1995:         seq.gsub!(/^\s*\#\s*/, '')
1996:         meth.call_seq = seq
1997:       end
1998:       
1999:       meth.comment = comment
2000: 
2001:     end

      # File parsers/parse_rb.rb, line 2026
2026:     def parse_method_or_yield_parameters(method=nil, modifiers=METHOD_MODIFIERS)
2027:       skip_tkspace(false)
2028:       tk = get_tk
2029: 
2030:       # Little hack going on here. In the statement
2031:       #  f = 2*(1+yield)
2032:       # We see the RPAREN as the next token, so we need
2033:       # to exit early. This still won't catch all cases
2034:       # (such as "a = yield + 1"
2035:       end_token = case tk
2036:                   when TkLPAREN, TkfLPAREN
2037:                     TkRPAREN
2038:                   when TkRPAREN
2039:                     return ""
2040:                   else
2041:                     TkNL
2042:                   end
2043:       nest = 0
2044: 
2045:       loop do
2046:         puts("Param: #{tk.inspect}, #{@scanner.continue} " +
2047:           "#{@scanner.lex_state} #{nest}")  if $DEBUG
2048:         case tk
2049:         when TkSEMICOLON
2050:           break
2051:         when TkLBRACE
2052:           nest += 1
2053:         when TkRBRACE
2054:           # we might have a.each {|i| yield i }
2055:           unget_tk(tk) if nest.zero?
2056:           nest -= 1
2057:           break if nest <= 0
2058:         when TkLPAREN, TkfLPAREN
2059:           nest += 1
2060:         when end_token
2061:           if end_token == TkRPAREN
2062:             nest -= 1
2063:             break if @scanner.lex_state == EXPR_END and nest <= 0
2064:           else
2065:             break unless @scanner.continue
2066:           end
2067:         when method && method.block_params.nil? && TkCOMMENT
2068:           unget_tk(tk)
2069:           read_documentation_modifiers(method, modifiers)
2070:         end
2071:         tk = get_tk
2072:       end
2073:       res = get_tkread.tr("\n", " ").strip
2074:       res = "" if res == ";"
2075:       res
2076:     end

      # File parsers/parse_rb.rb, line 2016
2016:     def parse_method_parameters(method)
2017:       res = parse_method_or_yield_parameters(method)
2018:       res = "(" + res + ")" unless res[0] == ?(
2019:       method.params = res unless method.params
2020:       if method.block_params.nil?
2021:           skip_tkspace(false)
2022:           read_documentation_modifiers(method, METHOD_MODIFIERS)
2023:       end
2024:     end

      # File parsers/parse_rb.rb, line 1785
1785:     def parse_module(container, single, tk, comment)
1786:       progress("m")
1787:       @stats.num_modules += 1
1788:       container, name_t  = get_class_or_module(container)
1789: #      skip_tkspace
1790:       name = name_t.name
1791:       mod = container.add_module(NormalModule, name)
1792:       mod.record_location(@top_level)
1793:       read_documentation_modifiers(mod, CLASS_MODIFIERS)
1794:       parse_statements(mod)
1795:       mod.comment = comment
1796:     end

      # File parsers/parse_rb.rb, line 2408
2408:   def parse_require(context, comment)
2409:     skip_tkspace_comment
2410:     tk = get_tk
2411:     if tk.kind_of? TkLPAREN
2412:       skip_tkspace_comment
2413:       tk = get_tk
2414:     end
2415: 
2416:     name = nil
2417:     case tk
2418:     when TkSTRING
2419:       name = tk.text
2420: #    when TkCONSTANT, TkIDENTIFIER, TkIVAR, TkGVAR
2421: #      name = tk.name
2422:     when TkDSTRING
2423:       warn "Skipping require of dynamic string: #{tk.text}"
2424:  #   else
2425:  #     warn "'require' used as variable"
2426:     end
2427:     if name
2428:       context.add_require(Require.new(name, comment))
2429:     else
2430:       unget_tk(tk)
2431:     end
2432:   end

      # File parsers/parse_rb.rb, line 1577
1577:     def parse_statements(container, single=NORMAL, current_method=nil, comment='')
1578:       nest = 1
1579:       save_visibility = container.visibility
1580:       
1581: #      if container.kind_of?(TopLevel)
1582: #      else
1583: #        comment = ''
1584: #      end
1585: 
1586:       non_comment_seen = true
1587:       
1588:       while tk = get_tk
1589:         
1590:         keep_comment = false
1591:         
1592:         non_comment_seen = true unless tk.kind_of?(TkCOMMENT)
1593:         
1594:         case tk
1595: 
1596:         when TkNL
1597:           skip_tkspace(true)   # Skip blanks and newlines
1598:           tk = get_tk
1599:           if tk.kind_of?(TkCOMMENT)
1600:             if non_comment_seen
1601:               comment = ''
1602:               non_comment_seen = false
1603:             end
1604:             while tk.kind_of?(TkCOMMENT)
1605:               comment << tk.text << "\n"
1606:               tk = get_tk          # this is the newline 
1607:               skip_tkspace(false)  # leading spaces
1608:               tk = get_tk
1609:             end
1610:             unless comment.empty?
1611:               look_for_directives_in(container, comment) 
1612:               if container.done_documenting
1613:                 container.ongoing_visibility = save_visibility
1614: #                return
1615:               end
1616:             end
1617:             keep_comment = true
1618:           else
1619:             non_comment_seen = true
1620:           end
1621:           unget_tk(tk)
1622:           keep_comment = true
1623: 
1624: 
1625:         when TkCLASS
1626:           if container.document_children
1627:             parse_class(container, single, tk, comment)
1628:           else
1629:             nest += 1
1630:           end
1631: 
1632:         when TkMODULE
1633:           if container.document_children
1634:             parse_module(container, single, tk, comment)
1635:           else
1636:             nest += 1
1637:           end
1638: 
1639:         when TkDEF
1640:           if container.document_self
1641:             parse_method(container, single, tk, comment)
1642:           else
1643:             nest += 1
1644:           end
1645: 
1646:         when TkCONSTANT
1647:           if container.document_self
1648:             parse_constant(container, single, tk, comment)
1649:           end
1650: 
1651:         when TkALIAS
1652:           if container.document_self
1653:             parse_alias(container, single, tk, comment)
1654:           end
1655: 
1656:         when TkYIELD
1657:           if current_method.nil?
1658:             warn("Warning: yield outside of method") if container.document_self
1659:           else
1660:             parse_yield(container, single, tk, current_method)
1661:           end
1662: 
1663:           # Until and While can have a 'do', which shouldn't increas
1664:           # the nesting. We can't solve the general case, but we can
1665:           # handle most occurrences by ignoring a do at the end of a line
1666: 
1667:         when  TkUNTIL, TkWHILE
1668:           nest += 1
1669:           puts "FOUND #{tk.class} in #{container.name}, nest = #{nest}, " +
1670:             "line #{tk.line_no}" if $DEBUG
1671:           skip_optional_do_after_expression
1672: 
1673:           # 'for' is trickier
1674:         when TkFOR
1675:           nest += 1
1676:           puts "FOUND #{tk.class} in #{container.name}, nest = #{nest}, " +
1677:             "line #{tk.line_no}" if $DEBUG
1678:           skip_for_variable
1679:           skip_optional_do_after_expression
1680: 
1681:         when TkCASE, TkDO, TkIF, TkUNLESS, TkBEGIN
1682:           nest += 1
1683:           puts "Found #{tk.class} in #{container.name}, nest = #{nest}, " +
1684:             "line #{tk.line_no}" if $DEBUG
1685: 
1686:         when TkIDENTIFIER
1687:           if nest == 1 and current_method.nil?
1688:             case tk.name
1689:             when "private", "protected", "public",
1690:                  "private_class_method", "public_class_method"
1691:               parse_visibility(container, single, tk)
1692:               keep_comment = true
1693:             when "attr"
1694:               parse_attr(container, single, tk, comment)
1695:             when /^attr_(reader|writer|accessor)$/, @options.extra_accessors
1696:               parse_attr_accessor(container, single, tk, comment)
1697:             when "alias_method"
1698:               if container.document_self
1699:                 parse_alias(container, single, tk, comment)
1700:               end
1701:             end
1702:           end
1703:           
1704:           case tk.name
1705:           when "require"
1706:             parse_require(container, comment)
1707:           when "include"
1708:             parse_include(container, comment)
1709:           end
1710: 
1711: 
1712:         when TkEND
1713:           nest -= 1
1714:           puts "Found 'end' in #{container.name}, nest = #{nest}, line #{tk.line_no}" if $DEBUG
1715:           puts "Method = #{current_method.name}" if $DEBUG and current_method
1716:           if nest == 0
1717:             read_documentation_modifiers(container, CLASS_MODIFIERS)
1718:             container.ongoing_visibility = save_visibility
1719:             return
1720:           end
1721: 
1722:         end
1723: 
1724:         comment = '' unless keep_comment
1725:         begin
1726:           get_tkread
1727:           skip_tkspace(false)
1728:         end while peek_tk == TkNL
1729: 
1730:       end
1731:     end

      # File parsers/parse_rb.rb, line 2547
2547:     def parse_symbol_arg(no = nil)
2548: 
2549:       args = []
2550:       skip_tkspace_comment
2551:       case tk = get_tk
2552:       when TkLPAREN
2553:         loop do
2554:           skip_tkspace_comment
2555:           if tk1 = parse_symbol_in_arg
2556:             args.push tk1
2557:             break if no and args.size >= no
2558:           end
2559:           
2560:           skip_tkspace_comment
2561:           case tk2 = get_tk
2562:           when TkRPAREN
2563:             break
2564:           when TkCOMMA
2565:           else
2566:            warn("unexpected token: '#{tk2.inspect}'") if $DEBUG
2567:             break
2568:           end
2569:         end
2570:       else
2571:         unget_tk tk
2572:         if tk = parse_symbol_in_arg
2573:           args.push tk
2574:           return args if no and args.size >= no
2575:         end
2576: 
2577:         loop do
2578: #         skip_tkspace_comment(false)
2579:           skip_tkspace(false)
2580: 
2581:           tk1 = get_tk
2582:           unless tk1.kind_of?(TkCOMMA) 
2583:             unget_tk tk1
2584:             break
2585:           end
2586:           
2587:           skip_tkspace_comment
2588:           if tk = parse_symbol_in_arg
2589:             args.push tk
2590:             break if no and args.size >= no
2591:           end
2592:         end
2593:       end
2594:       args
2595:     end

      # File parsers/parse_rb.rb, line 2597
2597:     def parse_symbol_in_arg
2598:       case tk = get_tk
2599:       when TkSYMBOL
2600:         tk.text.sub(/^:/, '')
2601:       when TkSTRING
2602:         eval @read[-1]
2603:       else
2604:         warn("Expected symbol or string, got #{tk.inspect}") if $DEBUG
2605:         nil
2606:       end
2607:     end

      # File parsers/parse_rb.rb, line 1570
1570:     def parse_toplevel_statements(container)
1571:       comment = collect_first_comment
1572:       look_for_directives_in(container, comment)
1573:       container.comment = comment unless comment.empty?
1574:       parse_statements(container, NORMAL, nil, comment)
1575:     end

      # File parsers/parse_rb.rb, line 2483
2483:     def parse_visibility(container, single, tk)
2484:       singleton = (single == SINGLE)
2485:       vis = case tk.name
2486:             when "private"   then :private
2487:             when "protected" then :protected
2488:             when "public"    then :public
2489:             when "private_class_method"
2490:               singleton = true
2491:               :private
2492:             when "public_class_method"
2493:               singleton = true
2494:               :public
2495:             else raise "Invalid visibility: #{tk.name}"
2496:             end
2497:             
2498:       skip_tkspace_comment(false)
2499:       case peek_tk
2500:         # Ryan Davis suggested the extension to ignore modifiers, because he
2501:         # often writes
2502:         #
2503:         #   protected unless $TESTING
2504:         #
2505:       when TkNL, TkUNLESS_MOD, TkIF_MOD
2506: #        error("Missing argument") if singleton        
2507:         container.ongoing_visibility = vis
2508:       else
2509:         args = parse_symbol_arg
2510:         container.set_visibility_for(args, vis, singleton)
2511:       end
2512:     end

      # File parsers/parse_rb.rb, line 2400
2400:   def parse_yield(context, single, tk, method)
2401:     if method.block_params.nil?
2402:       get_tkread
2403:       @scanner.instance_eval{@continue = false}
2404:       method.block_params = parse_yield_parameters
2405:     end
2406:   end

      # File parsers/parse_rb.rb, line 2396
2396:     def parse_yield_parameters
2397:       parse_method_or_yield_parameters
2398:     end

      # File parsers/parse_rb.rb, line 1536
1536:     def peek_read
1537:       @read.join('')
1538:     end

      # File parsers/parse_rb.rb, line 1505
1505:     def peek_tk
1506:       unget_tk(tk = get_tk)
1507:       tk
1508:     end

      # File parsers/parse_rb.rb, line 1449
1449:     def progress(char)
1450:       unless @options.quiet
1451:         @progress.print(char)
1452:         @progress.flush
1453:       end
1454:     end

      # File parsers/parse_rb.rb, line 2254
2254:     def read_directive(allowed)
2255:       tk = get_tk
2256:       puts "directive: #{tk.inspect}" if $DEBUG
2257:       result = nil
2258:       if tk.kind_of?(TkCOMMENT) 
2259:         if tk.text =~ /\s*:?(\w+):\s*(.*)/
2260:           directive = $1.downcase
2261:           if allowed.include?(directive)
2262:             result = [directive, $2]
2263:           end
2264:         end
2265:       else
2266:         unget_tk(tk)
2267:       end
2268:       result
2269:     end

      # File parsers/parse_rb.rb, line 2272
2272:     def read_documentation_modifiers(context, allow)
2273:       dir = read_directive(allow)
2274: 
2275:       case dir[0]
2276: 
2277:       when "notnew", "not_new", "not-new"
2278:         context.dont_rename_initialize = true
2279: 
2280:       when "nodoc"
2281:         context.document_self = false
2282:         if dir[1].downcase == "all"
2283:           context.document_children = false
2284:         end
2285: 
2286:       when "doc"
2287:         context.document_self = true
2288:         context.force_documentation = true
2289: 
2290:       when "yield", "yields"
2291:         unless context.params.nil?
2292:           context.params.sub!(/(,|)\s*&\w+/,'') # remove parameter &proc
2293:         end
2294:         context.block_params = dir[1]
2295: 
2296:       when "arg", "args"
2297:         context.params = dir[1]
2298:       end if dir
2299:     end

      # File parsers/parse_rb.rb, line 2353
2353:     def remove_private_comments(comment)
2354:       comment.gsub!(/^#--.*?^#\+\+/m, '')
2355:       comment.sub!(/^#--.*/m, '')
2356:     end

      # File parsers/parse_rb.rb, line 1461
1461:     def remove_token_listener(obj)
1462:       @token_listeners.delete(obj)
1463:     end

      # File parsers/parse_rb.rb, line 1399
1399:     def scan
1400:       @tokens = []
1401:       @unget_read = []
1402:       @read = []
1403:       catch(:eof) do
1404:         catch(:enddoc) do
1405:           begin
1406:             parse_toplevel_statements(@top_level)
1407:           rescue Exception => e
1408:             $stderr.puts "\n\n"
1409:             $stderr.puts "RDoc failure in #@input_file_name at or around " +
1410:                          "line #{@scanner.line_no} column #{@scanner.char_no}"
1411:             $stderr.puts 
1412:             $stderr.puts "Before reporting this, could you check that the file"
1413:             $stderr.puts "you're documenting compiles cleanly--RDoc is not a"
1414:             $stderr.puts "full Ruby parser, and gets confused easily if fed"
1415:             $stderr.puts "invalid programs."
1416:             $stderr.puts
1417:             $stderr.puts "The internal error was:\n\n"
1418:             
1419:             e.set_backtrace(e.backtrace[0,4])
1420:             raise
1421:           end
1422:         end
1423:       end
1424:       @top_level
1425:     end

      # File parsers/parse_rb.rb, line 2079
2079:     def skip_for_variable
2080:       skip_tkspace(false)
2081:       tk = get_tk
2082:       skip_tkspace(false)
2083:       tk = get_tk
2084:       unget_tk(tk) unless tk.kind_of?(TkIN)
2085:     end

      # File parsers/parse_rb.rb, line 2003
2003:     def skip_method(container)
2004:       meth =  AnyMethod.new("", "anon")
2005:       parse_method_parameters(meth)
2006:       parse_statements(container, false, meth)
2007:     end

      # File parsers/parse_rb.rb, line 2088
2088:     def skip_optional_do_after_expression
2089:       skip_tkspace(false)
2090:       tk = get_tk
2091:       case tk
2092:       when TkLPAREN, TkfLPAREN
2093:         end_token = TkRPAREN
2094:       else
2095:         end_token = TkNL
2096:       end
2097: 
2098:       nest = 0
2099:       @scanner.instance_eval{@continue = false}
2100: 
2101:       loop do
2102:         puts("\nWhile: #{tk}, #{@scanner.continue} " +
2103:           "#{@scanner.lex_state} #{nest}") if $DEBUG
2104:         case tk
2105:         when TkSEMICOLON
2106:           break
2107:         when TkLPAREN, TkfLPAREN
2108:           nest += 1
2109:         when TkDO
2110:           break if nest.zero?
2111:         when end_token
2112:           if end_token == TkRPAREN
2113:             nest -= 1
2114:             break if @scanner.lex_state == EXPR_END and nest.zero?
2115:           else
2116:             break unless @scanner.continue
2117:           end
2118:         end
2119:         tk = get_tk
2120:       end
2121:       skip_tkspace(false)
2122:       if peek_tk.kind_of? TkDO
2123:         get_tk
2124:       end
2125:     end

      # File parsers/parse_rb.rb, line 1520
1520:     def skip_tkspace(skip_nl = true)
1521:       tokens = []
1522:       while ((tk = get_tk).kind_of?(TkSPACE) ||
1523:              (skip_nl && tk.kind_of?(TkNL)))
1524:         tokens.push tk
1525:       end
1526:       unget_tk(tk)
1527:       tokens
1528:     end

      # File parsers/parse_rb.rb, line 2539
2539:     def skip_tkspace_comment(skip_nl = true)
2540:       loop do
2541:         skip_tkspace(skip_nl)
2542:         return unless peek_tk.kind_of? TkCOMMENT
2543:         get_tk
2544:       end
2545:     end

      # File parsers/parse_rb.rb, line 1510
1510:     def unget_tk(tk)
1511:       @tokens.unshift tk
1512:       @unget_read.unshift @read.pop
1513: 
1514:       # Remove this token from any listeners
1515:       @token_listeners.each do |obj|
1516:         obj.pop_token
1517:       end if @token_listeners
1518:     end

      # File parsers/parse_rb.rb, line 1437
1437:     def warn(msg)
1438:       return if @options.quiet
1439:       msg = make_message msg
1440:       $stderr.puts msg
1441:     end