History

Finish webpack scraper

Thibaut committed 11 years ago

Commit f0cb4b61 parent ba165ae15 files · +25 −89Browse files

Showing the changes to lib/docs (3 of 15 files). Show all

Unified · Split

3 files
  1. lib/docs/filters/webpack/clean_html.rb +6 −18
  2. lib/docs/filters/webpack/entries.rb +2 −12
  3. lib/docs/scrapers/webpack.rb +11 −4
lib/docs/filters/webpack/clean_html.rb+6−18
@@ -1,5 +1,5 @@
11 module Docs
2− class Webpack
2+ class Webpack
33 class CleanHtmlFilter < Filter
44 def call
55 root_page? ? root : other
@@ -14,32 +14,20 @@ module Docs · 8 unmodified lines
1414 # in devdocs.
1515 hr_index = doc.children.find_index { |node| node.name == "hr" }
1616 doc.children[0..hr_index].each(&:remove)
17−
18− # Add a page header :)
19− page_header_node = Nokogiri::XML::Node.new "h1", @doc
20− page_header_node.content = "Webpack"
21− @doc.children.first.add_previous_sibling page_header_node
2217 end
2318
2419
2520 def other
2621 # Re-create the header element
27− page_header = at_css("#wikititle").content.upcase
28− page_header_node = Nokogiri::XML::Node.new "h1", @doc
29− page_header_node.content = page_header
22+ at_css("#wiki").child.before("<h1>#{at_css("#wikititle").content}</h1>")
3023
31− # Change the scope of the page
3224 @doc = at_css("#wiki")
3325
34− # Add the page header
35− @doc.children.first.add_previous_sibling page_header_node
36−
37− # Remove the sidebar links in each page
38− css(".contents").remove
39−
40− # Remove all anchors
41− css("a.anchor").remove
26+ css('.contents', 'a.anchor', 'hr').remove
4227
28+ css('pre').each do |node|
29+ node.content = node.content
30+ end
4331 end
4432 end
4533 end
lib/docs/filters/webpack/entries.rb+2−12
@@ -2,7 +2,7 @@ module Docs · 1 unmodified line
22 class Webpack
33 class EntriesFilter < Docs::EntriesFilter
44 def get_name
5− entry_link.content.strip.titleize
5+ entry_link.content
66 end
77
88 def get_type
@@ -10,7 +10,7 @@ module Docs · 1 unmodified line
1010 type_links_list = link_li.parent
1111 current_type = type_links_list.parent
1212
13− # current type is a
13+ # current type is a
1414 # <li>
1515 # TYPE
1616 # <li> <ul> .. links .. </ul> </li>
@@ -20,16 +20,6 @@ module Docs · 3 unmodified lines
2020 current_type.children.first.content.strip.titleize
2121 end
2222
23−
24− def additional_entries
25− return [] unless root_page?
26−
27− css('.sidebar a').each_with_object [] do |link, entries|
28− name = link.content.strip
29− entries << [name, link['href']]
30− end
31− end
32−
3323 private
3424
3525 def entry_link
lib/docs/scrapers/webpack.rb+11−4
@@ -1,13 +1,20 @@
11 module Docs
22 class Webpack < UrlScraper
3− self.name = 'Webpack'
3+ self.name = 'webpack'
44 self.type = 'webpack'
5− self.version = '1.9'
5+ self.version = '1.10'
66 self.base_url = 'https://webpack.github.io/docs/'
7− self.root_path = 'index.html'
7+ self.links = {
8+ home: 'https://webpack.github.io/',
9+ code: 'https://github.com/webpack/webpack'
10+ }
811
9− html_filters.push 'webpack/entries', 'webpack/clean_html'
12+ html_filters.push 'webpack/entries', 'webpack/clean_html', 'title'
1013
14+ options[:title] = false
15+ options[:root_title] = 'webpack'
16+
17+ options[:skip] = %w(list-of-tutorials.html examples.html changelog.html ideas.html roadmap.html)
1118
1219 options[:attribution] = <<-HTML
1320 &copy; 2012&ndash;2015 Tobias Koppers<br>