History

Finish Statsmodels scraper

Thibaut Courouble committed 10 years ago

Commit 48479171 parent 06f5fae12 files · +14 −8Browse files

Unified · Split

12 files
  1. assets/images/docs.png +0 −0
  2. assets/images/docs@2x.png +0 −0
  3. assets/javascripts/news.json +1 −1
  4. assets/javascripts/templates/pages/about_tmpl.coffee +5 −0
  5. assets/stylesheets/global/_icons.scss +1 −0
  6. lib/docs/filters/pandas/clean_html.rb +0 −2
  7. lib/docs/filters/sphinx/clean_html.rb +1 −1
  8. lib/docs/filters/statsmodels/clean_html.rb +4 −4
  9. lib/docs/filters/statsmodels/entries.rb +4 −1
  10. lib/docs/scrapers/statsmodels.rb +2 −3
  11. public/icons/docs/statsmodels/16.png +0 −0
  12. public/icons/docs/statsmodels/16@2x.png +0 −0
assets/images/docs.png+0−0

Binary file not shown.

assets/images/docs@2x.png+0−0

Binary file not shown.

assets/javascripts/news.json+1−1
@@ -1,7 +1,7 @@
1[1[
2 [2 [
3 "2016-10-10",3 "2016-10-10",
4 "New documentation: <a href=\"/scikit_learn/\">scikit-learn</a>"4 "New documentations: <a href=\"/scikit_learn/\">scikit-learn</a> and <a href=\"/statsmodels/\">Statsmodels</a>"
5 ], [5 ], [
6 "2016-09-18",6 "2016-09-18",
7 "New documentations: <a href=\"/pandas/\">pandas</a> and <a href=\"/twig/\">Twig</a>"7 "New documentations: <a href=\"/pandas/\">pandas</a> and <a href=\"/twig/\">Twig</a>"
assets/javascripts/templates/pages/about_tmpl.coffee+5−0
@@ -519,6 +519,11 @@ credits = [ · 518 unmodified lines
519 '2014-2015 Automattic',519 '2014-2015 Automattic',
520 'MIT',520 'MIT',
521 'https://raw.githubusercontent.com/Automattic/socket.io/master/LICENSE'521 'https://raw.githubusercontent.com/Automattic/socket.io/master/LICENSE'
522 ], [
523 'Statsmodels',
524 '2009-2012 Statsmodels Developers<br>&copy; 2006-2008 Scipy Developers<br>&copy; 2006 Jonathan E. Taylor',
525 'BSD',
526 'https://raw.githubusercontent.com/statsmodels/statsmodels/master/LICENSE.txt'
522 ], [527 ], [
523 'Symfony',528 'Symfony',
524 '2004-2016 Fabien Potencier',529 '2004-2016 Fabien Potencier',
assets/stylesheets/global/_icons.scss+1−0
@@ -88,6 +88,7 @@ · 87 unmodified lines
88._icon-knockout:before { background-position: -5rem -3rem; }88._icon-knockout:before { background-position: -5rem -3rem; }
89._icon-moment:before { background-position: -6rem -3rem; @extend %darkIconFix !optional; }89._icon-moment:before { background-position: -6rem -3rem; @extend %darkIconFix !optional; }
90._icon-c:before { background-position: -7rem -3rem; }90._icon-c:before { background-position: -7rem -3rem; }
91._icon-statsmodels:before { background-position: -8rem -3rem; }
91._icon-yii:before,92._icon-yii:before,
92._icon-yii1:before { background-position: -9rem -3rem; }93._icon-yii1:before { background-position: -9rem -3rem; }
93._icon-cpp:before { background-position: 0 -4rem; }94._icon-cpp:before { background-position: 0 -4rem; }
lib/docs/filters/pandas/clean_html.rb+0−2
@@ -13,8 +13,6 @@ module Docs · 12 unmodified lines
13 node.before(node.children).remove13 node.before(node.children).remove
14 end14 end
15 15
16 css('.anchor-link').remove
17
18 doc16 doc
19 end17 end
20 end18 end
lib/docs/filters/sphinx/clean_html.rb+1−1
@@ -2,7 +2,7 @@ module Docs · 1 unmodified line
2 class Sphinx2 class Sphinx
3 class CleanHtmlFilter < Filter3 class CleanHtmlFilter < Filter
4 def call4 def call
5 css('.headerlink', 'hr', '#contents .topic-title', '#topics .topic-title', 'colgroup', '.line-block').remove5 css('.headerlink', 'hr', '#contents .topic-title', '#topics .topic-title', 'colgroup', '.line-block', '.anchor-link').remove
6 6
7 css('.contents > ul:first-child:last-child.simple > li:first-child:last-child').each do |node|7 css('.contents > ul:first-child:last-child.simple > li:first-child:last-child').each do |node|
8 node.parent.before(node.at_css('> ul')) if node.at_css('> ul')8 node.parent.before(node.at_css('> ul')) if node.at_css('> ul')
lib/docs/filters/statsmodels/clean_html.rb+4−4
@@ -5,10 +5,10 @@ module Docs · 4 unmodified lines
5 @doc = at_css('.body')5 @doc = at_css('.body')
6 6
7 if root_page?7 if root_page?
8 at_css('h1').content = 'Statsmodels'8 at_css('h1').content = 'Statsmodels'
9 at_css('#basic-documentation').remove9 at_css('#basic-documentation').remove
10 at_css('#table-of-contents').remove10 at_css('#table-of-contents').remove
11 at_css('#indices-and-tables').remove11 at_css('#indices-and-tables').remove
12 end12 end
13 13
14 doc14 doc
lib/docs/filters/statsmodels/entries.rb+4−1
@@ -6,9 +6,12 @@ module Docs · 5 unmodified lines
6 name = at_css('dt').content.strip6 name = at_css('dt').content.strip
7 name.sub! %r{\(.*}, '()' # Remove method arguments7 name.sub! %r{\(.*}, '()' # Remove method arguments
8 name.remove! %r{[\=\[].*} # Remove "[source]"8 name.remove! %r{[\=\[].*} # Remove "[source]"
9 name.remove! %r{\A(class(method)?) (statsmodels\.)?}9 name.remove! %r{\A(class(method)?) }
10 name.remove! %r{\Astatsmodels\.}
10 else11 else
11 name = at_css('h1').content.strip12 name = at_css('h1').content.strip
13 name.prepend 'Manual: ' if type == 'Manual'
14 name.prepend 'Example: ' if type == 'Examples'
12 end15 end
13 name.remove! "\u{00B6}" # Remove ¶16 name.remove! "\u{00B6}" # Remove ¶
14 name17 name
lib/docs/scrapers/statsmodels.rb+2−3
@@ -1,9 +1,8 @@
1module Docs1module Docs
2 class Statsmodels < UrlScraper2 class Statsmodels < UrlScraper
3 self.name = 'Statsmodels'
4 self.type = 'sphinx'3 self.type = 'sphinx'
5 self.release = '0.6.1'4 self.release = '0.6.1'
6 self.base_url = "http://statsmodels.sourceforge.net/stable/"5 self.base_url = 'http://statsmodels.sourceforge.net/stable/'
7 self.root_path = 'index.html'6 self.root_path = 'index.html'
8 self.links = {7 self.links = {
9 home: 'http://statsmodels.sourceforge.net/',8 home: 'http://statsmodels.sourceforge.net/',
@@ -12,7 +11,7 @@ module Docs · 2 unmodified lines
12 11
13 html_filters.push 'statsmodels/entries', 'statsmodels/clean_html', 'sphinx/clean_html'12 html_filters.push 'statsmodels/entries', 'statsmodels/clean_html', 'sphinx/clean_html'
14 13
15 options[:skip] = %w(about.html)14 options[:skip] = %w(about.html search.html genindex.html)
16 options[:skip_patterns] = [/\Arelease/, /\Adev/, /\A_modules/, /\Adatasets/]15 options[:skip_patterns] = [/\Arelease/, /\Adev/, /\A_modules/, /\Adatasets/]
17 16
18 options[:attribution] = <<-HTML17 options[:attribution] = <<-HTML
public/icons/docs/statsmodels/16.png+0−0

Binary file not shown.

public/icons/docs/statsmodels/16@2x.png+0−0

Binary file not shown.