Finish Bluebird scraper

Thibaut Courouble committed 8 years ago

Commit dac41921 parent b399a0c10 files · +47 −30Browse files

Showing the changes to lib/docs (3 of 10 files). Show all

Unified · Split

3 files

@@ -2,8 +2,33 @@ module Docs · 1 unmodified line
2	class Bluebird	2	class Bluebird
3	class CleanHtmlFilter < Filter	3	class CleanHtmlFilter < Filter
4	def call	4	def call
5	css('.post-content > p:first').remove	5	@doc = at_css('.post')
6	css('pre').attr('data-language', 'javascript')	6
		7	css('hr').remove
		8
		9	css('.api-code-section').each do \|node\|
		10	node.previous_element.remove
		11	end
		12
		13	css('.post-header', '.post-content', '.api-reference-menu', '.api-code-section', 'markdown', '.highlight', 'code code').each do \|node\|
		14	node.before(node.children).remove
		15	end
		16
		17	at_css('> h2:first-child').name = 'h1' unless at_css('h1')
		18
		19	css('.header-anchor[name]').each do \|node\|
		20	node.parent['id'] = node['name']
		21	end
		22
		23	css('pre').each do \|node\|
		24	node.content = node.content
		25	node['data-language'] = 'javascript'
		26	end
		27
		28	css('.info-box').each do \|node\|
		29	node.name = 'blockquote'
		30	end
		31
7	doc	32	doc
8	end	33	end
9	end	34	end

           done promise.config suppressunhandledrejections
           promise.onpossiblyunhandledrejection promise.bind
           promise.onunhandledrejectionhandled),
-        'Progression migration': %(),
-        'Deferred migration': %(),
-        'Environment variables': %(),
-        "Beginner's Guide": %w(),
-        'Error management configuration': %w(),
-        'Anti-patterns': %w(),
-        'Deprecated APIs': %w()
       }
       def get_name
-        name = at_css('h1.post-title')
+        name = at_css('h1').content.strip
-        if name.nil?
+        name << '()' if doc.to_html.include?("#{name}(")
-          name = at_css('.post-content h2')
+        name
-        end
-        name.text
       end
       def get_type
-        type = nil
+        if slug.start_with?('api')
-        TYPE_MAP.each do |k,v|
+          TYPE_MAP.each do |key, value|
-          if k.to_s.casecmp(name.strip) == 0
+            return key.to_s if value.include?(slug.remove('api/'))
-            type = k
-            break
-          else
-            slug_end = slug.sub(%r(^docs/api/), '')
-            if v.include?(slug_end.downcase)
-              type = k
-              break
-            end
           end
+        else
+          'Guides'
         end
-        type.to_s
       end
     end
   end
 end

 module Docs
   class Bluebird < UrlScraper
-    self.type = 'bluebird'
+    self.type = 'simple'
-    self.base_url = 'http://bluebirdjs.com'
+    self.release = '3.5.1'
-    self.root_path = '/docs/api-reference.html'
+    self.base_url = 'http://bluebirdjs.com/docs/'
-    self.release = '3.5.0'
+    self.root_path = 'api-reference.html'
+    self.force_gzip = true
     self.links = {
       home: 'http://bluebirdjs.com/',
       code: 'https://github.com/petkaantonov/bluebird/'
     html_filters.push 'bluebird/clean_html', 'bluebird/entries'
-    options[:container] = 'body .post'
+    options[:skip] = %w(support.html download-api-reference.html contribute.html)
     options[:attribution] = <<-HTML
-      &copy; Petka Antonov<br/>
+      &copy; 2013&ndash;2017 Petka Antonov<br>
       Licensed under the MIT License.
     HTML
   end