History

Finish pandas scraper

Thibaut Courouble committed 10 years ago

Commit 659cf941 parent 194168712 files · +32 −8Browse files

Unified · Split

12 files
  1. assets/images/docs.png +0 −0
  2. assets/images/docs@2x.png +0 −0
  3. assets/javascripts/news.json +1 −1
  4. assets/javascripts/templates/pages/about_tmpl.coffee +5 −0
  5. assets/stylesheets/global/_icons.scss +1 −0
  6. assets/stylesheets/pages/_sphinx.scss +2 −1
  7. lib/docs/filters/pandas/clean_html.rb +11 −0
  8. lib/docs/filters/pandas/entries.rb +9 −4
  9. lib/docs/filters/sphinx/clean_html.rb +1 −1
  10. lib/docs/scrapers/pandas.rb +6 −5
  11. public/icons/docs/pandas/16.png +0 −0
  12. public/icons/docs/pandas/16@2x.png +0 −0
assets/images/docs.png+0−0

Binary file not shown.

assets/images/docs@2x.png+0−0

Binary file not shown.

assets/javascripts/news.json+1−1
@@ -1,7 +1,7 @@
1[1[
2 [2 [
3 "2016-09-18",3 "2016-09-18",
4 "New documentation: <a href=\"/twig/\">Twig</a>"4 "New documentations: <a href=\"/pandas/\">pandas</a> and <a href=\"/twig/\">Twig</a>"
5 ], [5 ], [
6 "2016-09-05",6 "2016-09-05",
7 "New documentations: <a href=\"/fish/\">Fish</a>, <a href=\"/bottle/\">Bottle</a> and <a href=\"/scikit_image/\">scikit-image</a>"7 "New documentations: <a href=\"/fish/\">Fish</a>, <a href=\"/bottle/\">Bottle</a> and <a href=\"/scikit_image/\">scikit-image</a>"
assets/javascripts/templates/pages/about_tmpl.coffee+5−0
@@ -399,6 +399,11 @@ credits = [ · 398 unmodified lines
399 '2010-2016 Padrino',399 '2010-2016 Padrino',
400 'MIT',400 'MIT',
401 'https://raw.githubusercontent.com/padrino/padrino-framework/master/padrino/LICENSE.txt'401 'https://raw.githubusercontent.com/padrino/padrino-framework/master/padrino/LICENSE.txt'
402 ], [
403 'pandas',
404 '2011-2012 Lambda Foundry, Inc. and PyData Development Team<br>&copy; 2008-2011 AQR Capital Management, LLC<br>&copy; 2008-2014 the pandas development team',
405 'BSD',
406 'https://raw.githubusercontent.com/pydata/pandas/master/LICENSE'
402 ], [407 ], [
403 'Perl',408 'Perl',
404 '1993-2016 Larry Wall and others',409 '1993-2016 Larry Wall and others',
assets/stylesheets/global/_icons.scss+1−0
@@ -119,6 +119,7 @@ · 118 unmodified lines
119._icon-fish:before { background-position: -5rem -6rem; @extend %darkIconFix !optional; }119._icon-fish:before { background-position: -5rem -6rem; @extend %darkIconFix !optional; }
120._icon-scikit_image:before { background-position: -6rem -6rem; }120._icon-scikit_image:before { background-position: -6rem -6rem; }
121._icon-twig:before { background-position: -7rem -6rem; }121._icon-twig:before { background-position: -7rem -6rem; }
122._icon-pandas:before { background-position: -8rem -6rem; }
122._icon-bottle:before { background-position: 0 -7rem; }123._icon-bottle:before { background-position: 0 -7rem; }
123._icon-docker:before { background-position: -1rem -7rem; }124._icon-docker:before { background-position: -1rem -7rem; }
124._icon-cakephp:before { background-position: -2rem -7rem; }125._icon-cakephp:before { background-position: -2rem -7rem; }
assets/stylesheets/pages/_sphinx.scss+2−1
@@ -1,5 +1,6 @@
1%sphinx {1%sphinx {
2 h2, h3 { @extend %block-heading; }2 h2 { @extend %block-heading; }
3 h3 { @extend %block-label; }
3 h4 { font-size: 1em; }4 h4 { font-size: 1em; }
4 > dl:not(.docutils) > dt { @extend %block-label, %label-blue; }5 > dl:not(.docutils) > dt { @extend %block-label, %label-blue; }
5 dd > dl:not(.docutils) > dt { @extend %block-label; }6 dd > dl:not(.docutils) > dt { @extend %block-label; }
lib/docs/filters/pandas/clean_html.rb+11−0
@@ -4,6 +4,17 @@ module Docs · 3 unmodified lines
4 def call4 def call
5 @doc = at_css('.body')5 @doc = at_css('.body')
6 6
7 if root_page?
8 css('a[href$=".zip"]', 'a[href$=".pdf"]', '.toctree-wrapper').remove
9 at_css('h1').content = 'pandas'
10 end
11
12 css('h2 > a.reference', 'h3 > a.reference').each do |node|
13 node.before(node.children).remove
14 end
15
16 css('.anchor-link').remove
17
7 doc18 doc
8 end19 end
9 end20 end
lib/docs/filters/pandas/entries.rb+9−4
@@ -2,20 +2,25 @@ module Docs · 1 unmodified line
2 class Pandas2 class Pandas
3 class EntriesFilter < Docs::EntriesFilter3 class EntriesFilter < Docs::EntriesFilter
4 def get_name4 def get_name
5 if dt = at_css('dt')5 if subpath.start_with?('generated')
6 name = dt.content.strip6 name = at_css('dt').content.strip
7 name.sub! %r{\(.*}, '()'7 name.sub! %r{\(.*}, '()'
8 name.remove! %r{\s=.*}8 name.remove! %r{\s=.*}
9 name.remove! %r{\A(class(method)?) }9 name.remove! %r{\A(class(method)?) (pandas\.)?}
10 else10 else
11 name = at_css('h1').content.strip11 name = at_css('h1').content.strip
12 name.prepend "#{css('.toctree-l1 > a:not([href^="http"])').to_a.index(at_css('.toctree-l1.current > a')) + 1}. "
12 end13 end
13 name.remove! "\u{00B6}"14 name.remove! "\u{00B6}"
14 name15 name
15 end16 end
16 17
17 def get_type18 def get_type
18 css(".toctree-l2.current > a").last.content19 if subpath.start_with?('generated')
20 css('.toctree-l2.current > a').last.content
21 else
22 'Manual'
23 end
19 end24 end
20 end25 end
21 end26 end
lib/docs/filters/sphinx/clean_html.rb+1−1
@@ -49,7 +49,7 @@ module Docs · 48 unmodified lines
49 end49 end
50 50
51 css('dt').each do |node|51 css('dt').each do |node|
52 next unless node['id'] || node.at_css('code')52 next unless node['id'] || node.at_css('code, .classifier')
53 links = []53 links = []
54 links << node.children.last.remove while node.children.last.try(:name) == 'a'54 links << node.children.last.remove while node.children.last.try(:name) == 'a'
55 node.inner_html = "<code>#{node.content.strip}</code> "55 node.inner_html = "<code>#{node.content.strip}</code> "
lib/docs/scrapers/pandas.rb+6−5
@@ -2,7 +2,7 @@ module Docs · 1 unmodified line
2 class Pandas < UrlScraper2 class Pandas < UrlScraper
3 self.name = 'pandas'3 self.name = 'pandas'
4 self.type = 'sphinx'4 self.type = 'sphinx'
5 self.root_path = 'api.html'5 self.root_path = 'index.html'
6 self.links = {6 self.links = {
7 home: 'http://pandas.pydata.org/',7 home: 'http://pandas.pydata.org/',
8 code: 'https://github.com/pydata/pandas'8 code: 'https://github.com/pydata/pandas'
@@ -13,12 +13,13 @@ module Docs · 4 unmodified lines
13 # Cannot take only the body, as the sidebar gives info about the type.13 # Cannot take only the body, as the sidebar gives info about the type.
14 options[:container] = '.document'14 options[:container] = '.document'
15 15
16 # Using the above container, leads to tons of anchors. Only keep the generated/ pages.16 options[:skip] = %w(internals.html release.html contributing.html whatsnew.html)
17 options[:only_patterns] = [/\Agenerated\//]
18 17
19 options[:attribution] = <<-HTML18 options[:attribution] = <<-HTML
20 &copy; 2008&ndash;2014, the pandas development team.<br>19 &copy; 2011&ndash;2012 Lambda Foundry, Inc. and PyData Development Team<br>
21 Licensed under the BSD license.20 &copy; 2008&ndash;2011 AQR Capital Management, LLC<br>
21 &copy; 2008&ndash;2014 the pandas development team<br>
22 Licensed under the 3-clause BSD License.
22 HTML23 HTML
23 24
24 version '0.18' do25 version '0.18' do
public/icons/docs/pandas/16.png+0−0

Binary file not shown.

public/icons/docs/pandas/16@2x.png+0−0

Binary file not shown.