Finish pandas scraper

Thibaut Courouble committed 10 years ago

Commit 659cf941 parent 194168712 files · +32 −8Browse files

Unified · Split

12 files

Binary file not shown.

Binary file not shown.

@@ -1,7 +1,7 @@
1	1	[
2	2	[
3	3	"2016-09-18",
4		− "New documentation: <a href=\"/twig/\">Twig</a>"
	4	+ "New documentations: <a href=\"/pandas/\">pandas</a> and <a href=\"/twig/\">Twig</a>"
5	5	], [
6	6	"2016-09-05",
7	7	"New documentations: <a href=\"/fish/\">Fish</a>, <a href=\"/bottle/\">Bottle</a> and <a href=\"/scikit_image/\">scikit-image</a>"

@@ -399,6 +399,11 @@ credits = [ · 398 unmodified lines
399	399	'2010-2016 Padrino',
400	400	'MIT',
401	401	'https://raw.githubusercontent.com/padrino/padrino-framework/master/padrino/LICENSE.txt'
	402	+ ], [
	403	+ 'pandas',
	404	+ '2011-2012 Lambda Foundry, Inc. and PyData Development Team<br>© 2008-2011 AQR Capital Management, LLC<br>© 2008-2014 the pandas development team',
	405	+ 'BSD',
	406	+ 'https://raw.githubusercontent.com/pydata/pandas/master/LICENSE'
402	407	], [
403	408	'Perl',
404	409	'1993-2016 Larry Wall and others',

@@ -119,6 +119,7 @@ · 118 unmodified lines
119	119	._icon-fish:before { background-position: -5rem -6rem; @extend %darkIconFix !optional; }
120	120	._icon-scikit_image:before { background-position: -6rem -6rem; }
121	121	._icon-twig:before { background-position: -7rem -6rem; }
	122	+._icon-pandas:before { background-position: -8rem -6rem; }
122	123	._icon-bottle:before { background-position: 0 -7rem; }
123	124	._icon-docker:before { background-position: -1rem -7rem; }
124	125	._icon-cakephp:before { background-position: -2rem -7rem; }

@@ -1,5 +1,6 @@
1	1	%sphinx {
2		− h2, h3 { @extend %block-heading; }
	2	+ h2 { @extend %block-heading; }
	3	+ h3 { @extend %block-label; }
3	4	h4 { font-size: 1em; }
4	5	> dl:not(.docutils) > dt { @extend %block-label, %label-blue; }
5	6	dd > dl:not(.docutils) > dt { @extend %block-label; }

@@ -4,6 +4,17 @@ module Docs · 3 unmodified lines
4	4	def call
5	5	@doc = at_css('.body')
6	6
	7	+ if root_page?
	8	+ css('a[href$=".zip"]', 'a[href$=".pdf"]', '.toctree-wrapper').remove
	9	+ at_css('h1').content = 'pandas'
	10	+ end
	11	+
	12	+ css('h2 > a.reference', 'h3 > a.reference').each do \|node\|
	13	+ node.before(node.children).remove
	14	+ end
	15	+
	16	+ css('.anchor-link').remove
	17	+
7	18	doc
8	19	end
9	20	end

@@ -2,20 +2,25 @@ module Docs · 1 unmodified line
2	2	class Pandas
3	3	class EntriesFilter < Docs::EntriesFilter
4	4	def get_name
5		− if dt = at_css('dt')
6		− name = dt.content.strip
	5	+ if subpath.start_with?('generated')
	6	+ name = at_css('dt').content.strip
7	7	name.sub! %r{\(.*}, '()'
8	8	name.remove! %r{\s=.*}
9		− name.remove! %r{\A(class(method)?) }
	9	+ name.remove! %r{\A(class(method)?) (pandas\.)?}
10	10	else
11	11	name = at_css('h1').content.strip
	12	+ name.prepend "#{css('.toctree-l1 > a:not([href^="http"])').to_a.index(at_css('.toctree-l1.current > a')) + 1}. "
12	13	end
13	14	name.remove! "\u{00B6}"
14	15	name
15	16	end
16	17
17	18	def get_type
18		− css(".toctree-l2.current > a").last.content
	19	+ if subpath.start_with?('generated')
	20	+ css('.toctree-l2.current > a').last.content
	21	+ else
	22	+ 'Manual'
	23	+ end
19	24	end
20	25	end
21	26	end

@@ -49,7 +49,7 @@ module Docs · 48 unmodified lines
49	49	end
50	50
51	51	css('dt').each do \|node\|
52		− next unless node['id'] \|\| node.at_css('code')
	52	+ next unless node['id'] \|\| node.at_css('code, .classifier')
53	53	links = []
54	54	links << node.children.last.remove while node.children.last.try(:name) == 'a'
55	55	node.inner_html = "<code>#{node.content.strip}</code> "

@@ -2,7 +2,7 @@ module Docs · 1 unmodified line
2	2	class Pandas < UrlScraper
3	3	self.name = 'pandas'
4	4	self.type = 'sphinx'
5		− self.root_path = 'api.html'
	5	+ self.root_path = 'index.html'
6	6	self.links = {
7	7	home: 'http://pandas.pydata.org/',
8	8	code: 'https://github.com/pydata/pandas'
@@ -13,12 +13,13 @@ module Docs · 4 unmodified lines
13	13	# Cannot take only the body, as the sidebar gives info about the type.
14	14	options[:container] = '.document'
15	15
16		− # Using the above container, leads to tons of anchors. Only keep the generated/ pages.
17		− options[:only_patterns] = [/\Agenerated\//]
	16	+ options[:skip] = %w(internals.html release.html contributing.html whatsnew.html)
18	17
19	18	options[:attribution] = <<-HTML
20		− © 2008–2014, the pandas development team.<br>
21		− Licensed under the BSD license.
	19	+ © 2011–2012 Lambda Foundry, Inc. and PyData Development Team<br>
	20	+ © 2008–2011 AQR Capital Management, LLC<br>
	21	+ © 2008–2014 the pandas development team<br>
	22	+ Licensed under the 3-clause BSD License.
22	23	HTML
23	24
24	25	version '0.18' do

Binary file not shown.

Binary file not shown.