Finish Statsmodels scraper

Thibaut Courouble committed 10 years ago

Commit 48479171 parent 06f5fae12 files · +14 −8Browse files

Unified · Split

12 files

Binary file not shown.

Binary file not shown.

@@ -1,7 +1,7 @@
1	1	[
2	2	[
3	3	"2016-10-10",
4		− "New documentation: <a href=\"/scikit_learn/\">scikit-learn</a>"
	4	+ "New documentations: <a href=\"/scikit_learn/\">scikit-learn</a> and <a href=\"/statsmodels/\">Statsmodels</a>"
5	5	], [
6	6	"2016-09-18",
7	7	"New documentations: <a href=\"/pandas/\">pandas</a> and <a href=\"/twig/\">Twig</a>"

@@ -519,6 +519,11 @@ credits = [ · 518 unmodified lines
519	519	'2014-2015 Automattic',
520	520	'MIT',
521	521	'https://raw.githubusercontent.com/Automattic/socket.io/master/LICENSE'
	522	+ ], [
	523	+ 'Statsmodels',
	524	+ '2009-2012 Statsmodels Developers<br>© 2006-2008 Scipy Developers<br>© 2006 Jonathan E. Taylor',
	525	+ 'BSD',
	526	+ 'https://raw.githubusercontent.com/statsmodels/statsmodels/master/LICENSE.txt'
522	527	], [
523	528	'Symfony',
524	529	'2004-2016 Fabien Potencier',

@@ -88,6 +88,7 @@ · 87 unmodified lines
88	88	._icon-knockout:before { background-position: -5rem -3rem; }
89	89	._icon-moment:before { background-position: -6rem -3rem; @extend %darkIconFix !optional; }
90	90	._icon-c:before { background-position: -7rem -3rem; }
	91	+._icon-statsmodels:before { background-position: -8rem -3rem; }
91	92	._icon-yii:before,
92	93	._icon-yii1:before { background-position: -9rem -3rem; }
93	94	._icon-cpp:before { background-position: 0 -4rem; }

@@ -13,8 +13,6 @@ module Docs · 12 unmodified lines
13	13	node.before(node.children).remove
14	14	end
15	15
16		− css('.anchor-link').remove
17		−
18	16	doc
19	17	end
20	18	end

@@ -2,7 +2,7 @@ module Docs · 1 unmodified line
2	2	class Sphinx
3	3	class CleanHtmlFilter < Filter
4	4	def call
5		− css('.headerlink', 'hr', '#contents .topic-title', '#topics .topic-title', 'colgroup', '.line-block').remove
	5	+ css('.headerlink', 'hr', '#contents .topic-title', '#topics .topic-title', 'colgroup', '.line-block', '.anchor-link').remove
6	6
7	7	css('.contents > ul:first-child:last-child.simple > li:first-child:last-child').each do \|node\|
8	8	node.parent.before(node.at_css('> ul')) if node.at_css('> ul')

@@ -5,10 +5,10 @@ module Docs · 4 unmodified lines
5	5	@doc = at_css('.body')
6	6
7	7	if root_page?
8		− at_css('h1').content = 'Statsmodels'
9		− at_css('#basic-documentation').remove
10		− at_css('#table-of-contents').remove
11		− at_css('#indices-and-tables').remove
	8	+ at_css('h1').content = 'Statsmodels'
	9	+ at_css('#basic-documentation').remove
	10	+ at_css('#table-of-contents').remove
	11	+ at_css('#indices-and-tables').remove
12	12	end
13	13
14	14	doc

@@ -6,9 +6,12 @@ module Docs · 5 unmodified lines
6	6	name = at_css('dt').content.strip
7	7	name.sub! %r{\(.*}, '()' # Remove method arguments
8	8	name.remove! %r{[\=\[].*} # Remove "[source]"
9		− name.remove! %r{\A(class(method)?) (statsmodels\.)?}
	9	+ name.remove! %r{\A(class(method)?) }
	10	+ name.remove! %r{\Astatsmodels\.}
10	11	else
11	12	name = at_css('h1').content.strip
	13	+ name.prepend 'Manual: ' if type == 'Manual'
	14	+ name.prepend 'Example: ' if type == 'Examples'
12	15	end
13	16	name.remove! "\u{00B6}" # Remove ¶
14	17	name

@@ -1,9 +1,8 @@
1	1	module Docs
2	2	class Statsmodels < UrlScraper
3		− self.name = 'Statsmodels'
4	3	self.type = 'sphinx'
5	4	self.release = '0.6.1'
6		− self.base_url = "http://statsmodels.sourceforge.net/stable/"
	5	+ self.base_url = 'http://statsmodels.sourceforge.net/stable/'
7	6	self.root_path = 'index.html'
8	7	self.links = {
9	8	home: 'http://statsmodels.sourceforge.net/',
@@ -12,7 +11,7 @@ module Docs · 2 unmodified lines
12	11
13	12	html_filters.push 'statsmodels/entries', 'statsmodels/clean_html', 'sphinx/clean_html'
14	13
15		− options[:skip] = %w(about.html)
	14	+ options[:skip] = %w(about.html search.html genindex.html)
16	15	options[:skip_patterns] = [/\Arelease/, /\Adev/, /\A_modules/, /\Adatasets/]
17	16
18	17	options[:attribution] = <<-HTML

Binary file not shown.

Binary file not shown.