Finish Statsmodels scraper

Thibaut Courouble committed 10 years ago

Commit 48479171 parent 06f5fae12 files · +14 −8Browse files

Showing the changes to lib/docs (5 of 12 files). Show all

Unified · Split

5 files

@@ -13,8 +13,6 @@ module Docs · 12 unmodified lines
13	node.before(node.children).remove	13	node.before(node.children).remove
14	end	14	end
15		15
16	css('.anchor-link').remove
17
18	doc	16	doc
19	end	17	end
20	end	18	end

@@ -2,7 +2,7 @@ module Docs · 1 unmodified line
2	class Sphinx	2	class Sphinx
3	class CleanHtmlFilter < Filter	3	class CleanHtmlFilter < Filter
4	def call	4	def call
5	css('.headerlink', 'hr', '#contents .topic-title', '#topics .topic-title', 'colgroup', '.line-block').remove	5	css('.headerlink', 'hr', '#contents .topic-title', '#topics .topic-title', 'colgroup', '.line-block', '.anchor-link').remove
6		6
7	css('.contents > ul:first-child:last-child.simple > li:first-child:last-child').each do \|node\|	7	css('.contents > ul:first-child:last-child.simple > li:first-child:last-child').each do \|node\|
8	node.parent.before(node.at_css('> ul')) if node.at_css('> ul')	8	node.parent.before(node.at_css('> ul')) if node.at_css('> ul')

@@ -5,10 +5,10 @@ module Docs · 4 unmodified lines
5	@doc = at_css('.body')	5	@doc = at_css('.body')
6		6
7	if root_page?	7	if root_page?
8	at_css('h1').content = 'Statsmodels'	8	at_css('h1').content = 'Statsmodels'
9	at_css('#basic-documentation').remove	9	at_css('#basic-documentation').remove
10	at_css('#table-of-contents').remove	10	at_css('#table-of-contents').remove
11	at_css('#indices-and-tables').remove	11	at_css('#indices-and-tables').remove
12	end	12	end
13		13
14	doc	14	doc

           name = at_css('dt').content.strip
           name.sub! %r{\(.*}, '()' # Remove method arguments
           name.remove! %r{[\=\[].*} # Remove "[source]"
-          name.remove! %r{\A(class(method)?) (statsmodels\.)?}
+          name.remove! %r{\A(class(method)?) }
+          name.remove! %r{\Astatsmodels\.}
         else
           name = at_css('h1').content.strip
+          name.prepend 'Manual: ' if type == 'Manual'
+          name.prepend 'Example: ' if type == 'Examples'
         end
         name.remove! "\u{00B6}" # Remove ¶
         name

 module Docs
   class Statsmodels < UrlScraper
-    self.name = 'Statsmodels'
     self.type = 'sphinx'
     self.release = '0.6.1'
-    self.base_url = "http://statsmodels.sourceforge.net/stable/"
+    self.base_url = 'http://statsmodels.sourceforge.net/stable/'
     self.root_path = 'index.html'
     self.links = {
       home: 'http://statsmodels.sourceforge.net/',
     html_filters.push 'statsmodels/entries', 'statsmodels/clean_html', 'sphinx/clean_html'
-    options[:skip] = %w(about.html)
+    options[:skip] = %w(about.html search.html genindex.html)
     options[:skip_patterns] = [/\Arelease/, /\Adev/, /\A_modules/, /\Adatasets/]
     options[:attribution] = <<-HTML