Finish webpack scraper

Thibaut committed 11 years ago

Commit f0cb4b61 parent ba165ae15 files · +25 −89Browse files

Unified · Split

15 files

Binary file not shown.

Binary file not shown.

@@ -1,7 +1,7 @@
1	[	1	[
2	[	2	[
3	"2015-07-05",	3	"2015-07-05",
4	"New documentation: <a href=\"/drupal/\">Drupal</a>"	4	"New documentations: <a href=\"/drupal/\">Drupal</a> and <a href=\"/webpack/\">webpack</a>"
5	], [	5	], [
6	"2015-05-24",	6	"2015-05-24",
7	"New <a href=\"/rust/\">Rust</a> documentation"	7	"New <a href=\"/rust/\">Rust</a> documentation"

     '2009-2015 Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors',
     'MIT',
     'https://raw.github.com/jashkenas/underscore/master/LICENSE'
+  ], [
+    'Webpack',
+    '2012-2015 Tobias Koppers',
+    'MIT',
+    'https://raw.githubusercontent.com/webpack/webpack/master/LICENSE'
   ], [
     'Yii',
     '2008-2015 by Yii Software LLC',

@@ -28,6 +28,7 @@ app.views.NodePage = · 27 unmodified lines
28	app.views.RethinkdbPage =	28	app.views.RethinkdbPage =
29	app.views.SinonPage =	29	app.views.SinonPage =
30	app.views.UnderscorePage =	30	app.views.UnderscorePage =
		31	app.views.WebpackPage =
31	app.views.JavascriptPage	32	app.views.JavascriptPage
32		33
33	app.views.RequirejsPage =	34	app.views.RequirejsPage =

@@ -102,3 +102,4 @@ · 101 unmodified lines
102	._icon-npm:before { background-position: -8rem -7rem; }	102	._icon-npm:before { background-position: -8rem -7rem; }
103	._icon-apache_http_server:before { background-position: -9rem -7rem; }	103	._icon-apache_http_server:before { background-position: -9rem -7rem; }
104	._icon-drupal:before { background-position: 0 -8rem; }	104	._icon-drupal:before { background-position: 0 -8rem; }
		105	._icon-webpack:before { background-position: -1rem -8rem; @extend %darkIconFix !optional; }

@@ -1,55 +0,0 @@
1	._webpack {
2	> h2 { @extend %block-heading; }
3	> h3 { @extend %block-label, %label-blue; }
4	> h4 { font-size: 1em; }
5
6	/*
7	Copied from the original webpack css
8	https://webpack.github.io/docs/css/doc.css
9	*/
10	code .comment {
11	color: #998;
12	font-style: italic
13	}
14
15	code .keyword {
16	color: #333;
17	font-weight: 700
18	}
19
20	code .number {
21	color: #099
22	}
23
24	code .string {
25	color: #d14
26	}
27
28	code .tag {
29	color: navy;
30	font-weight: 400
31	}
32
33	code .attribute {
34	color: teal
35	}
36
37	code .regexp {
38	color: #009926
39	}
40
41	code .preprocessor {
42	color: #999;
43	font-weight: 700
44	}
45
46	code .deletion {
47	background: #fdd;
48	display: block
49	}
50
51	code .addition {
52	background: #dfd;
53	display: block
54	}
55	}

 module Docs
   class Webpack
     class CleanHtmlFilter < Filter
       def call
         root_page? ? root : other
         # in devdocs.
         hr_index = doc.children.find_index { |node| node.name == "hr" }
         doc.children[0..hr_index].each(&:remove)
-        # Add a page header :)
-        page_header_node = Nokogiri::XML::Node.new "h1", @doc
-        page_header_node.content = "Webpack"
-        @doc.children.first.add_previous_sibling page_header_node
       end
       def other
         # Re-create the header element
-        page_header = at_css("#wikititle").content.upcase
+        at_css("#wiki").child.before("<h1>#{at_css("#wikititle").content}</h1>")
-        page_header_node = Nokogiri::XML::Node.new "h1", @doc
-        page_header_node.content = page_header
-        # Change the scope of the page
         @doc = at_css("#wiki")
-        # Add the page header
+        css('.contents', 'a.anchor', 'hr').remove
-        @doc.children.first.add_previous_sibling page_header_node
-        # Remove the sidebar links in each page
-        css(".contents").remove
-        # Remove all anchors
-        css("a.anchor").remove
+        css('pre').each do |node|
+          node.content = node.content
+        end
       end
     end
   end

   class Webpack
     class EntriesFilter < Docs::EntriesFilter
       def get_name
-        entry_link.content.strip.titleize
+        entry_link.content
       end
       def get_type
         type_links_list = link_li.parent
         current_type = type_links_list.parent
         # current type is a
         # <li>
         #   TYPE
         #   <li> <ul> .. links .. </ul> </li>
         current_type.children.first.content.strip.titleize
       end
-      def additional_entries
-        return [] unless root_page?
-        css('.sidebar a').each_with_object [] do |link, entries|
-          name = link.content.strip
-          entries << [name, link['href']]
-        end
-      end
       private
       def entry_link

 module Docs
   class Webpack < UrlScraper
-    self.name = 'Webpack'
+    self.name = 'webpack'
     self.type = 'webpack'
-    self.version = '1.9'
+    self.version = '1.10'
     self.base_url = 'https://webpack.github.io/docs/'
-    self.root_path = 'index.html'
+    self.links = {
+      home: 'https://webpack.github.io/',
+      code: 'https://github.com/webpack/webpack'
+    }
-    html_filters.push 'webpack/entries', 'webpack/clean_html'
+    html_filters.push 'webpack/entries', 'webpack/clean_html', 'title'
+    options[:title] = false
+    options[:root_title] = 'webpack'
+    options[:skip] = %w(list-of-tutorials.html examples.html changelog.html ideas.html roadmap.html)
     options[:attribution] = <<-HTML
       &copy; 2012&ndash;2015 Tobias Koppers<br>

Binary file not shown.

Binary file not shown.