Finish Falcon scraper

Thibaut Courouble committed 9 years ago

Commit 899aee81 parent f42425610 files · +27 −15Browse files

Showing the changes to lib/docs (2 of 10 files). Show all

Unified · Split

2 files

lib/docs/filters/falcon/entries.rb +17 −11
lib/docs/scrapers/falcon.rb +6 −6

lib/docs/filters/falcon/entries.rb+17−11

       def get_type
         case slug.split('/').first
-        when 'community'
-          'Community Guide'
         when 'user'
-          'User Guide'
+          'Guide'
         when 'api'
-          'Classes and Functions'
+          'API'
-        else
-          'Other'
         end
       end
         entries = []
         css('.class').each do |node|
+          namespace = node.at_css('.descclassname').content.strip.remove(/\.\z/)
           class_name = node.at_css('dt > .descname').content
           class_id = node.at_css('dt[id]')['id']
-          entries << [class_name, class_id]
+          entries << ["#{namespace}.#{class_name}", class_id, namespace]
+           node.css('.attribute').each do |n|
+            next unless n.at_css('dt[id]')
+            name = n.at_css('.descname').content
+            name = "#{namespace}.#{class_name}.#{name}"
+            id = n.at_css('dt[id]')['id']
+            entries << [name, id, namespace]
+          end
           node.css('.method').each do |n|
             next unless n.at_css('dt[id]')
             name = n.at_css('.descname').content
-            name = "#{class_name}.#{name}()"
+            name = "#{namespace}.#{class_name}.#{name}()"
             id = n.at_css('dt[id]')['id']
-            entries << [name, id]
+            entries << [name, id, namespace]
           end
         end
         css('.function').each do |node|
-          name = "#{node.at_css('.descname').content}()"
+          namespace = node.at_css('.descclassname').content.strip.remove(/\.\z/)
+          name = "#{namespace}.#{node.at_css('.descname').content}()"
           id = node.at_css('dt[id]')['id']
-          entries << [name, id]
+          entries << [name, id, namespace]
         end
         entries

lib/docs/scrapers/falcon.rb+6−6

     options[:container] = '.body'
-    options[:skip_patterns] = [/\Achanges/, /\A_modules/]
+    options[:skip] = %w(user/index.html api/index.html)
+    options[:skip_patterns] = [/\Achanges/, /\A_modules/, /\Acommunity/]
     options[:attribution] = <<-HTML
-      &copy; 2016 Falcon Contributors<br>
+      &copy; 2012&ndash;2016 by Rackspace Hosting, Inc. and other contributors<br>
-      Licensed under the Apache 2 License.
+      Licensed under the Apache License, Version 2.0.
     HTML
-    version '1.2.0' do
+    version '1.2' do
       self.release = '1.2.0'
-      self.base_url = "https://falcon.readthedocs.io/en/#{self.version}/"
+      self.base_url = "https://falcon.readthedocs.io/en/#{self.release}/"
     end
   end
 end