History

Finish Pygame scraper

Thibaut Courouble committed 8 years ago

Commit 34fe8331 parent 6f4f34812 files · +26 −31Browse files

Unified · Split

12 files
  1. assets/images/docs-2.png +0 −0
  2. assets/images/docs-2@2x.png +0 −0
  3. assets/javascripts/news.json +3 −0
  4. assets/javascripts/templates/pages/about_tmpl.coffee +5 −0
  5. assets/stylesheets/global/_icons.scss +1 −0
  6. lib/docs/filters/pygame/clean_html.rb +9 −11
  7. lib/docs/filters/pygame/entries.rb +11 −17
  8. lib/docs/scrapers/pygame.rb +3 −9
  9. public/icons/docs-2.pxm +0 −0
  10. public/icons/docs-2@2x.pxm +0 −0
  11. public/icons/docs/pygame/16.png +0 −0
  12. public/icons/docs/pygame/16@2x.png +0 −0
assets/images/docs-2.png+0−0

Binary file not shown.

assets/images/docs-2@2x.png+0−0

Binary file not shown.

assets/javascripts/news.json+3−0
@@ -1,5 +1,8 @@
1[1[
2 [2 [
3 "2018-07-29",
4 "New documentation: <a href=\"/pygame/\">Pygame</a>"
5 ], [
3 "2018-07-08",6 "2018-07-08",
4 "New documentations: <a href=\"/leaflet/\">Leaflet</a>, <a href=\"/terraform/\">Terraform</a> and <a href=\"/koa/\">Koa</a>"7 "New documentations: <a href=\"/leaflet/\">Leaflet</a>, <a href=\"/terraform/\">Terraform</a> and <a href=\"/koa/\">Koa</a>"
5 ], [8 ], [
assets/javascripts/templates/pages/about_tmpl.coffee+5−0
@@ -545,6 +545,11 @@ credits = [ · 544 unmodified lines
545 '1996-2018 The PostgreSQL Global Development Group<br>&copy; 1994 The Regents of the University of California',545 '1996-2018 The PostgreSQL Global Development Group<br>&copy; 1994 The Regents of the University of California',
546 'PostgreSQL',546 'PostgreSQL',
547 'https://www.postgresql.org/about/licence/'547 'https://www.postgresql.org/about/licence/'
548 ], [
549 'Pygame',
550 'Pygame Developpers',
551 'LGPLv2.1',
552 'https://raw.githubusercontent.com/pygame/pygame/master/LICENSE'
548 ], [553 ], [
549 'Python',554 'Python',
550 '2001-2018 Python Software Foundation<br>Python is a trademark of the Python Software Foundation.',555 '2001-2018 Python Software Foundation<br>Python is a trademark of the Python Software Foundation.',
assets/stylesheets/global/_icons.scss+1−0
@@ -173,3 +173,4 @@ · 172 unmodified lines
173._icon-babel:before { background-position: -2rem -3rem; @extend %doc-icon-2; }173._icon-babel:before { background-position: -2rem -3rem; @extend %doc-icon-2; }
174._icon-leaflet:before { background-position: -3rem -3rem; @extend %doc-icon-2; }174._icon-leaflet:before { background-position: -3rem -3rem; @extend %doc-icon-2; }
175._icon-terraform:before { background-position: -4rem -3rem; @extend %doc-icon-2; }175._icon-terraform:before { background-position: -4rem -3rem; @extend %doc-icon-2; }
176._icon-pygame:before { background-position: -5rem -3rem; @extend %doc-icon-2; }
lib/docs/filters/pygame/clean_html.rb+9−11
@@ -2,23 +2,21 @@ module Docs · 1 unmodified line
2 class Pygame2 class Pygame
3 class CleanHtmlFilter < Filter3 class CleanHtmlFilter < Filter
4 def call4 def call
5
6 @doc = at_css '.body'5 @doc = at_css '.body'
7 6
8 if root_page?7 if root_page?
9 # remove unneeded stuff8 # remove unneeded stuff
10 at_css('.modindex-jumpbox').remove9 at_css('.modindex-jumpbox').remove
11 css('[role="navigation"],.pcap, .cap, .footer').each do |node|10 css('[role="navigation"],.pcap, .cap, .footer').remove
12 node.remove
13 end
14 # table -> list11 # table -> list
15 list = at_css('table')12 list = at_css('table')
16 list.replace(list.children)13 list.replace(list.children)
17 list.name = 'ul'14 list.name = 'ul'
18 css('tr').each do |row|15 css('tr').each do |row|
19 row.name = 'li'16 row.name = 'li'
20 row['class'] = ''17 row.remove_attribute('class')
21 end18 end
19 at_css('h1').content = 'Pygame'
22 return doc20 return doc
23 end21 end
24 22
@@ -26,11 +24,7 @@ module Docs · 1 unmodified line
26 # .headerlink => ¶ after links24 # .headerlink => ¶ after links
27 # .toc => table of content25 # .toc => table of content
28 # .tooltip-content => tooltips after links to functions26 # .tooltip-content => tooltips after links to functions
29 if toremove = css('table.toc.docutils, .headerlink, .tooltip-content')27 css('table.toc.docutils, .headerlink, .tooltip-content').remove
30 toremove.each do |node|
31 node.remove
32 end
33 end
34 28
35 # Remove wrapper .section29 # Remove wrapper .section
36 section = at_css('.section')30 section = at_css('.section')
@@ -43,7 +37,7 @@ module Docs · 6 unmodified lines
43 pre = node.at_css('pre')37 pre = node.at_css('pre')
44 node.replace(pre)38 node.replace(pre)
45 # gets rid of the already existing syntax highlighting39 # gets rid of the already existing syntax highlighting
46 pre.inner_html = pre.inner_text40 pre.content = pre.content
47 pre['class'] = 'language-python'41 pre['class'] = 'language-python'
48 pre['data-language'] = "python"42 pre['data-language'] = "python"
49 end43 end
@@ -97,6 +91,10 @@ module Docs · 47 unmodified lines
97 end91 end
98 end92 end
99 93
94 css('> dl', '> dl > dd', 'h1 code').each do |node|
95 node.before(node.children).remove
96 end
97
100 doc98 doc
101 end99 end
102 end100 end
lib/docs/filters/pygame/entries.rb+11−17
@@ -2,33 +2,27 @@ module Docs · 1 unmodified line
2 class Pygame2 class Pygame
3 class EntriesFilter < Docs::EntriesFilter3 class EntriesFilter < Docs::EntriesFilter
4 def get_name4 def get_name
5 return 'pygame'5 at_css('h1').content.remove('pygame.')
6 end6 end
7 7
8 def get_type8 def get_type
9 at_css('h1').content9 at_css('h1').content
10 end10 end
11 11
12 def include_default_entry?
13 return false
14 end
15
16 def additional_entries12 def additional_entries
17 return ['pygame'] if root_page?13 return [] if root_page?
14
15 css('h1, h2, h3').each_with_object [] do |node, entries|
16 name = node['id'] || node['data-name']
18 17
19 entries = []18 if node.parent['class'].include?('function') or node.parent['class'].include?('method')
20 css('h1,h2,h3').each do |node|19 name << '()'
21 parentclass = node.parent['class']
22 name = node['id']
23 if not name
24 name = node['data-name']
25 elsif parentclass.include?('function') or parentclass.include?('method')
26 name += '()'
27 end20 end
28 name = name.sub('pygame.', '')21
29 entries << [name, node['id'], nil]22 name.remove!('pygame.')
23
24 entries << [name, node['id']] unless name == self.name
30 end25 end
31 entries
32 end26 end
33 end27 end
34 end28 end
lib/docs/scrapers/pygame.rb+3−9
@@ -1,15 +1,8 @@
1module Docs1module Docs
2 class Pygame < UrlScraper2 class Pygame < UrlScraper
3
4 self.type = 'simple'3 self.type = 'simple'
5 4 self.release = '1.9.4'
6 self.release = 'v1.9.4.dev0'
7
8 self.initial_paths = ['py-modindex.html']
9 self.base_url = 'https://www.pygame.org/docs/'
10 self.root_path = 'py-modindex.html'5 self.root_path = 'py-modindex.html'
11 self.initial_paths = []
12
13 self.links = {6 self.links = {
14 home: 'https://www.pygame.org/',7 home: 'https://www.pygame.org/',
15 code: 'https://github.com/pygame/pygame'8 code: 'https://github.com/pygame/pygame'
@@ -20,7 +13,8 @@ module Docs · 4 unmodified lines
20 options[:only_patterns] = [/ref\//]13 options[:only_patterns] = [/ref\//]
21 14
22 options[:attribution] = <<-HTML15 options[:attribution] = <<-HTML
23 &copy; Pygame Developpers.16 &copy; Pygame Developpers.<br>
17 Licensed under the GNU LGPL License version 2.1.
24 HTML18 HTML
25 end19 end
26end20end
public/icons/docs-2.pxm+0−0

Binary file not shown.

public/icons/docs-2@2x.pxm+0−0

Binary file not shown.

public/icons/docs/pygame/16.png+0−0

Binary file not shown.

public/icons/docs/pygame/16@2x.png+0−0

Binary file not shown.