Rewrite the Apache builtin generator to use SVN

Gemfile

@@ -42,4 +42,7 @@ group :development do
 
   gem 'ostruct'
   gem 'reline'
+  gem 'nokogiri'
+
+  gem 'svn-downloader'
 end

lib/rouge/lexers/apache.rb

@@ -17,6 +17,8 @@ class Apache < RegexLexer
       end
 
       def name_for_token(token, tktype)
+        token = token.downcase
+
         if SECTIONS.include? token
           tktype
         elsif DIRECTIVES.include? token
@@ -37,12 +39,12 @@ def name_for_token(token, tktype)
         mixin :whitespace
 
         rule %r/(<\/?)(\w+)/ do |m|
-          groups Punctuation, name_for_token(m[2].downcase, Name::Label)
+          groups Punctuation, name_for_token(m[2], Name::Label)
           push :section
         end
 
         rule %r/\w+/ do |m|
-          token name_for_token(m[0].downcase, Name::Class)
+          token name_for_token(m[0], Name::Class)
           push :directive
         end
       end
@@ -64,8 +66,22 @@ def name_for_token(token, tktype)
         mixin :whitespace
 
         rule %r/\S+/ do |m|
-          token name_for_token(m[0].downcase, Literal::String::Symbol)
+          if VALUES.include?(m[0].downcase)
+            token Literal::String::Symbol
+          else
+            fallthrough!
+          end
         end
+
+        rule(%r/(?=\S)/) { push :value }
+      end
+
+      state :value do
+        rule %r/[ \t]+/, Text, :pop!
+        rule %r/[^\s%]+/, Text
+        rule %r/%{.*?}/, Name::Variable
+        rule %r/[%]/, Text
+        rule(/(?=\n)/) { pop! }
       end
     end
   end

tasks/builtins/apache.rake

@@ -1,124 +1,110 @@
 # -*- coding: utf-8 -*- #
 # frozen_string_literal: true
 
-require 'open-uri'
+require_relative '../task_helper'
+
+# backcompat for ancient net/dav lib
+def File.exists?(p)
+  File.exist?(p)
+end
+
+# And it still warns a lot.
+silence_warnings { require 'net/dav' }
+
+require 'nokogiri'
+require 'svn/downloader'
 
 APACHE_DOCS_URI = "https://downloads.apache.org/httpd/docs/"
 APACHE_KEYWORDS_FILE = "./lib/rouge/lexers/apache/keywords.rb"
 
 namespace :builtins do
   task :apache do
-    STDERR.puts "SKIP: rewrite the apache fetcher as the source appears to have been removed."
-    next
-    generator = Rouge::Tasks::Builtins::Apache.new
+    ApacheBuiltins.process(APACHE_KEYWORDS_FILE)
+  end
+end
 
-    input = URI.open(APACHE_DOCS_URI) { |f| f.read }
-    files = generator.download_docs(input)
-    list  = files.delete("directives.html")
-    mods  = files.values
+class ApacheBuiltins < BuiltinsGenerator
+  APACHE_MOD_DOCS = 'https://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/manual/mod/'
 
-    keywords = generator.extract_keywords(list)
-    values   = generator.extract_values(mods)
+  BASE_PATH = File.expand_path("#{__dir__}/../../tmp/apache-mods")
 
-    output = generator.render_output(keywords, values)
+  def fetch
+    SVN::Downloader.download(APACHE_MOD_DOCS, BASE_PATH)
 
-    File.write(APACHE_KEYWORDS_FILE, output)
+    files = {}
+
+    Dir.chdir BASE_PATH do
+      Dir.glob("{directives,core,prefork,mod_*,mpm*}.xml*").each do |f|
+        files[File.basename(f)] = File.read(f)
+      end
+    end
+
+    files
   end
-end
 
-module Rouge
-  module Tasks
-    module Builtins
-      class Apache
-        def download_docs(input)
-          files = Hash.new
-
-          name, ext = input.match(/href="(.+\.en)(\.zip)"/) { |m| [m[1], m[2]] }
-          docs_zip = APACHE_DOCS_URI + name + ext
-
-          system "mkdir -p /tmp/rouge"
-          Dir.chdir "/tmp/rouge" do
-            system "wget -q #{docs_zip}"
-            system "unzip -oq #{name + ext}"
-            Dir.chdir "./#{name}/mod/" do
-              Dir.glob("./{directives,core,mod_*,mpm*}.html").each do |f|
-                files[File.basename(f)] = File.read(f)
-              end
-            end
-          end
-
-          files
-        end
+  WORD_VALUE = /\A\w[\w*-]+\z/o
 
-        def extract_keywords(input)
-          keywords = Hash.new { |h,k| h[k] = Array.new }
+  def parse
+    out = Hash.new { |h, k| h[k] = Set.new }
 
-          input.each_line do |line|
-            if line.scrub =~ %r(<li><a.*?>(&lt;)?(.*?)(&gt;)?</a></li>)
-              next unless $2
+    @input.each do |name, content|
+      xml = Nokogiri::XML(content)
 
-              if $1 && $3
-                key = "sections"
-              else
-                key = "directives"
-              end
+      xml.css('directivesynopsis').each do |synopsis|
+        name = synopsis.css('name').inner_text.strip
+        syntax = synopsis.css('syntax')
 
-              keywords[key].push $2.downcase
-            end
-          end
+        # anything in a <var> or <em> tag is a variable description, and not a constant
+        syntax.css('var').remove
+        syntax.css('em').remove
 
-          keywords
-        end
+        # detect constants in command arguments
+
+        values = syntax.inner_text.strip
+          .gsub(/\A\w+\s*/, '')
+          .split(/[|\s]+/)
+          .grep(WORD_VALUE)
 
-        def extract_values(inputs)
-          values = Set.new
-
-          inputs.each do |input|
-            input = input.scrub
-            input.scan(%r[Syntax:.*?<code>(.*?)</code>]m) do |m|
-              m[0].split(/[\s|]/).
-                drop(1).
-                filter { |v| v.index(/^\w[\w*-]+$/) }.
-                each { |v| values.add(v.downcase) }
-            end
-
-            input.scan(%r[<dt>\s*(?:<.+?>\s*)*(.*?)<]m) do |m|
-              m[0].split(/[=\[\]]/).
-                filter { |v| v.index(/^\w[\w*-]+$/) }.
-                each { |v| values.add(v.downcase) }
-            end
-          end
-
-          values.to_a.sort
+        values.each do |value|
+          out[:values] << value.strip.downcase
         end
 
-        def render_output(keywords, values, &b)
-          return enum_for(:render_output, keywords, values).to_a.join("\n") unless b
-
-          yield   "# -*- coding: utf-8 -*- #"
-          yield   "# frozen_string_literal: true"
-          yield   ""
-          yield   "# DO NOT EDIT"
-          yield   "# This file is automatically generated by `rake builtins:apache`."
-          yield   "# See tasks/builtins/apache.rake for more info."
-          yield   ""
-          yield   "module Rouge"
-          yield   "  module Lexers"
-          yield   "    class Apache"
-          keywords.each do |k,v|
-            yield "      def self.#{k}"
-            yield "        @#{k} ||= Set.new #{v.inspect}"
-            yield "      end"
-            yield ""
-          end
-          yield   "      def self.values"
-          yield   "        @values ||= Set.new #{values.inspect}"
-          yield   "      end"
-          yield   "    end"
-          yield   "  end"
-          yield   "end"
+        name = name.downcase
+
+        if synopsis.attr('type') == "section"
+          out[:sections] << name
+        else
+          out[:directives] << name
         end
       end
+
+      xml.css('usage dt code, section dt code').each do |el|
+        el.children.reject(&:text?).each(&:remove)
+        value = el.inner_text.strip.downcase
+
+        value.split(/[=\[\]]+/).grep(WORD_VALUE).each { |v| out[:values] << v }
+      end
     end
+
+    out
+  end
+
+  def generate
+    yield "# -*- coding: utf-8 -*- #"
+    yield "# frozen_string_literal: true"
+    yield ""
+    yield "# DO NOT EDIT"
+    yield "# This file is automatically generated by `rake builtins:apache`."
+    yield "# See tasks/builtins/apache.rake for more info."
+    yield ""
+    yield "module Rouge"
+    yield "  module Lexers"
+    yield "    class Apache"
+    yield "      DIRECTIVES = Set#{@keywords[:directives].sort.inspect}"
+    yield "      VALUES = Set#{@keywords[:values].sort.inspect}"
+    yield "      SECTIONS = Set#{@keywords[:sections].sort.inspect}"
+    yield "    end"
+    yield "  end"
+    yield "end"
   end
 end

tasks/task_helper.rb

@@ -1,5 +1,12 @@
 require 'open-uri'
 
+def silence_warnings
+  old_verbose, $VERBOSE = $VERBOSE, false
+  yield
+ensure
+  $VERBOSE = old_verbose
+end
+
 class BuiltinsGenerator
   def self.process(fname, *a)
     new(*a).process(fname)