Finish Pygame scraper

Thibaut Courouble committed 8 years ago

Commit 34fe8331 parent 6f4f34812 files · +26 −31Browse files

Unified · Split

12 files

assets/images/docs-2.png +0 −0
assets/images/docs-2@2x.png +0 −0
assets/javascripts/news.json +3 −0
assets/javascripts/templates/pages/about_tmpl.coffee +5 −0
assets/stylesheets/global/_icons.scss +1 −0
lib/docs/filters/pygame/clean_html.rb +9 −11
lib/docs/filters/pygame/entries.rb +11 −17
lib/docs/scrapers/pygame.rb +3 −9
public/icons/docs-2.pxm +0 −0
public/icons/docs-2@2x.pxm +0 −0
public/icons/docs/pygame/16.png +0 −0
public/icons/docs/pygame/16@2x.png +0 −0

assets/images/docs-2.png+0−0

Binary file not shown.

assets/images/docs-2@2x.png+0−0

Binary file not shown.

assets/javascripts/news.json+3−0

@@ -1,5 +1,8 @@
1	1	[
2	2	[
	3	+ "2018-07-29",
	4	+ "New documentation: <a href=\"/pygame/\">Pygame</a>"
	5	+ ], [
3	6	"2018-07-08",
4	7	"New documentations: <a href=\"/leaflet/\">Leaflet</a>, <a href=\"/terraform/\">Terraform</a> and <a href=\"/koa/\">Koa</a>"
5	8	], [

assets/javascripts/templates/pages/about_tmpl.coffee+5−0

@@ -545,6 +545,11 @@ credits = [ · 544 unmodified lines
545	545	'1996-2018 The PostgreSQL Global Development Group<br>© 1994 The Regents of the University of California',
546	546	'PostgreSQL',
547	547	'https://www.postgresql.org/about/licence/'
	548	+ ], [
	549	+ 'Pygame',
	550	+ 'Pygame Developpers',
	551	+ 'LGPLv2.1',
	552	+ 'https://raw.githubusercontent.com/pygame/pygame/master/LICENSE'
548	553	], [
549	554	'Python',
550	555	'2001-2018 Python Software Foundation<br>Python is a trademark of the Python Software Foundation.',

assets/stylesheets/global/_icons.scss+1−0

@@ -173,3 +173,4 @@ · 172 unmodified lines
173	173	._icon-babel:before { background-position: -2rem -3rem; @extend %doc-icon-2; }
174	174	._icon-leaflet:before { background-position: -3rem -3rem; @extend %doc-icon-2; }
175	175	._icon-terraform:before { background-position: -4rem -3rem; @extend %doc-icon-2; }
	176	+._icon-pygame:before { background-position: -5rem -3rem; @extend %doc-icon-2; }

lib/docs/filters/pygame/clean_html.rb+9−11

@@ -2,23 +2,21 @@ module Docs · 1 unmodified line
2	2	class Pygame
3	3	class CleanHtmlFilter < Filter
4	4	def call
5		−
6	5	@doc = at_css '.body'
7	6
8	7	if root_page?
9	8	# remove unneeded stuff
10	9	at_css('.modindex-jumpbox').remove
11		− css('[role="navigation"],.pcap, .cap, .footer').each do \|node\|
12		− node.remove
13		− end
	10	+ css('[role="navigation"],.pcap, .cap, .footer').remove
14	11	# table -> list
15	12	list = at_css('table')
16	13	list.replace(list.children)
17	14	list.name = 'ul'
18	15	css('tr').each do \|row\|
19	16	row.name = 'li'
20		− row['class'] = ''
	17	+ row.remove_attribute('class')
21	18	end
	19	+ at_css('h1').content = 'Pygame'
22	20	return doc
23	21	end
24	22
@@ -26,11 +24,7 @@ module Docs · 1 unmodified line
26	24	# .headerlink => ¶ after links
27	25	# .toc => table of content
28	26	# .tooltip-content => tooltips after links to functions
29		− if toremove = css('table.toc.docutils, .headerlink, .tooltip-content')
30		− toremove.each do \|node\|
31		− node.remove
32		− end
33		− end
	27	+ css('table.toc.docutils, .headerlink, .tooltip-content').remove
34	28
35	29	# Remove wrapper .section
36	30	section = at_css('.section')
@@ -43,7 +37,7 @@ module Docs · 6 unmodified lines
43	37	pre = node.at_css('pre')
44	38	node.replace(pre)
45	39	# gets rid of the already existing syntax highlighting
46		− pre.inner_html = pre.inner_text
	40	+ pre.content = pre.content
47	41	pre['class'] = 'language-python'
48	42	pre['data-language'] = "python"
49	43	end
@@ -97,6 +91,10 @@ module Docs · 47 unmodified lines
97	91	end
98	92	end
99	93
	94	+ css('> dl', '> dl > dd', 'h1 code').each do \|node\|
	95	+ node.before(node.children).remove
	96	+ end
	97	+
100	98	doc
101	99	end
102	100	end

lib/docs/filters/pygame/entries.rb+11−17

@@ -2,33 +2,27 @@ module Docs · 1 unmodified line
2	2	class Pygame
3	3	class EntriesFilter < Docs::EntriesFilter
4	4	def get_name
5		− return 'pygame'
	5	+ at_css('h1').content.remove('pygame.')
6	6	end
7	7
8	8	def get_type
9	9	at_css('h1').content
10	10	end
11	11
12		− def include_default_entry?
13		− return false
14		− end
15		−
16	12	def additional_entries
17		− return ['pygame'] if root_page?
	13	+ return [] if root_page?
	14	+
	15	+ css('h1, h2, h3').each_with_object [] do \|node, entries\|
	16	+ name = node['id'] \|\| node['data-name']
18	17
19		− entries = []
20		− css('h1,h2,h3').each do \|node\|
21		− parentclass = node.parent['class']
22		− name = node['id']
23		− if not name
24		− name = node['data-name']
25		− elsif parentclass.include?('function') or parentclass.include?('method')
26		− name += '()'
	18	+ if node.parent['class'].include?('function') or node.parent['class'].include?('method')
	19	+ name << '()'
27	20	end
28		− name = name.sub('pygame.', '')
29		− entries << [name, node['id'], nil]
	21	+
	22	+ name.remove!('pygame.')
	23	+
	24	+ entries << [name, node['id']] unless name == self.name
30	25	end
31		− entries
32	26	end
33	27	end
34	28	end

lib/docs/scrapers/pygame.rb+3−9

@@ -1,15 +1,8 @@
1	1	module Docs
2	2	class Pygame < UrlScraper
3		−
4	3	self.type = 'simple'
5		−
6		− self.release = 'v1.9.4.dev0'
7		−
8		− self.initial_paths = ['py-modindex.html']
9		− self.base_url = 'https://www.pygame.org/docs/'
	4	+ self.release = '1.9.4'
10	5	self.root_path = 'py-modindex.html'
11		− self.initial_paths = []
12		−
13	6	self.links = {
14	7	home: 'https://www.pygame.org/',
15	8	code: 'https://github.com/pygame/pygame'
@@ -20,7 +13,8 @@ module Docs · 4 unmodified lines
20	13	options[:only_patterns] = [/ref\//]
21	14
22	15	options[:attribution] = <<-HTML
23		− © Pygame Developpers.
	16	+ © Pygame Developpers.<br>
	17	+ Licensed under the GNU LGPL License version 2.1.
24	18	HTML
25	19	end
26	20	end

public/icons/docs-2.pxm+0−0

Binary file not shown.

public/icons/docs-2@2x.pxm+0−0

Binary file not shown.

public/icons/docs/pygame/16.png+0−0

Binary file not shown.

public/icons/docs/pygame/16@2x.png+0−0

Binary file not shown.