Finish Nim scraper

Thibaut Courouble committed 9 years ago

Commit a4ba32e1 parent 09eea6617 files · +151 −135Browse files

Showing the changes to lib/docs (3 of 17 files). Show all

Unified · Split

3 files

lib/docs/filters/nim/clean_html.rb +46 −12
lib/docs/filters/nim/entries.rb +44 −31
lib/docs/scrapers/nim.rb +8 −8

lib/docs/filters/nim/clean_html.rb+46−12

@@ -4,26 +4,60 @@ module Docs · 3 unmodified lines
4	def call	4	def call
5	@doc = at_css('#documentId .container')	5	@doc = at_css('#documentId .container')
6		6
7	css('.docinfo').remove	7	css('.docinfo', '.footer', 'blockquote > p:empty', '.link-seesrc').remove
8		8
9	content = at_css('#content')	9	css('h1:not(.title), h2, h3, h4').each do \|node\|
10	if content != nil	10	node.name = node.name.sub(/\d/) { \|i\| i.to_i + 1 }
11	at_css('#content').remove_attribute('class')
12	@doc.add_child(at_css('#content').inner_html)
13	end	11	end
14		12
15	css('> div.row').remove	13	if content = at_css('#content')
		14	content.prepend_child at_css('h1.title')
		15	@doc = content
		16	end
16		17
17	css('pre').each do \|node\|	18	if root_page?
18	node['data-language'] = 'nim'	19	at_css('h1').content = 'Nim Documentation'
19	end	20	end
20		21
21	# remove link from headers
22	css('h1 > a', 'h2 > a', 'h3 > a', 'h4 > a').each do \|node\|	22	css('h1 > a', 'h2 > a', 'h3 > a', 'h4 > a').each do \|node\|
23	node.parent['id'] = node['id']	23	node.parent['id'] = node['id'] if node['id']
24	node.parent.content = node.content	24	node.before(node.children).remove
		25	end
		26
		27	css('a[name]').each do \|node\|
		28	node.next_element['id'] = node['name']
		29	node.remove
		30	end
		31
		32	css('pre').each do \|node\|
		33	node.content = node.content.strip
		34	node['data-language'] = 'nim' unless node.content =~ /\A[\w\-\_\:\=\ ]+\z/
25	end	35	end
26		36
		37	css('tt').each do \|node\|
		38	node.name = 'code'
		39	end
		40
		41	css('cite').each do \|node\|
		42	node.name = 'em'
		43	end
		44
		45	css('.section').each do \|node\|
		46	node.first_element_child['id'] = node['id'] if node['id']
		47	node.before(node.children).remove
		48	end
		49
		50	css('span.pre').each do \|node\|
		51	node.before(node.children).remove
		52	end
		53
		54	css('blockquote > pre:only-child', 'blockquote > dl:only-child', 'blockquote > table').each do \|node\|
		55	node.parent.before(node.parent.children).remove
		56	end
		57
		58	css('a', 'dl', 'table', 'code').remove_attr('class')
		59	css('table').remove_attr('border')
		60
27	doc	61	doc
28	end	62	end
29	end	63	end

lib/docs/filters/nim/entries.rb+44−31

@@ -1,49 +1,62 @@
1	module Docs	1	module Docs
2	class Nim	2	class Nim
3	class EntriesFilter < Docs::EntriesFilter	3	class EntriesFilter < Docs::EntriesFilter
4	def get_type	4	def get_name
5	at_css('h1').content	5	name = at_css('h1').content
		6	name.remove! 'Module '
		7	name.remove! ' User Guide'
		8	name.remove! ' User\'s manual'
		9	name.remove! %r{ \-.*}
		10	name.strip!
		11	name
6	end	12	end
7		13
8	def get_name	14	def get_type
9	at_css('h1').content	15	if name.include?('Tutorial')
		16	'Tutorial'
		17	elsif slug == 'manual'
		18	'Manual'
		19	elsif at_css('h1').content.include?('Module ')
		20	name
		21	else
		22	'Reference'
		23	end
10	end	24	end
11		25
12	def additional_entries	26	def additional_entries
13	entries = []	27	entries = []
14	if get_name.start_with? 'Module '	28
15	module_name = get_name[7..-1]	29	if at_css('h1').content.include?('Module ')
16	css('div .section').map do \|node\|	30	css('#toc-list > li > .simple-toc-section').each do \|node\|
17	section_node = node.at_css('h1 a')	31	type = node.previous_element.content.strip
18	if section_node != nil	32
19	section_name = section_node.content.strip	33	node.css('a.reference:not(.reference-toplevel)').each do \|n\|
20	items_node = node.at_css('dl.item')	34	n.css('span').remove
21	if items_node != nil	35	name = n.content.strip
22	items_node.css('dt a').map do \|item_node\|	36	name << '()' if (type == 'Procs' \|\| type == 'Templates') && !name.include?('`')
23	item_name = item_node['name']	37	name.remove! '`'
24	if item_name.include? ','	38	name.prepend "#{self.name}."
25	item_name = item_name.sub(',', '(') + ')'	39	id = n['href'].remove('#')
26	end	40	entries << [name, id] unless entries.any? { \|e\| e[0] == name }
27	entries << [module_name + '.' + item_name, item_node.parent['id']]
28	end
29	end
30	end	41	end
31	end	42	end
32	else	43	elsif slug == 'manual'
33	css('h1', 'h2', 'h3').map do \|node\|	44	css('#toc-list > li > a').each do \|node\|
34	id = node['id']
35	name = node.content.strip	45	name = node.content.strip
36	if id != nil	46	next if name.start_with?('About')
37	entries << [name, id]	47	id = node['href'].remove('#')
38	else	48	entries << [name, id]
39	a = node.at_css('a')	49	end
40	if a != nil	50
41	id = a['id']	51	css('#toc-list > ul').each do \|node\|
42	entries << [name, id]	52	type = node.previous_element.content.strip
43	end	53
		54	node.css('> li > a').each do \|n\|
		55	entries << [n.content.strip, n['href'].remove('#'), "Manual: #{type}"]
44	end	56	end
45	end	57	end
46	end	58	end
		59
47	entries	60	entries
48	end	61	end
49	end	62	end

lib/docs/scrapers/nim.rb+8−8

   class Nim < UrlScraper
     self.type = 'nim'
     self.release = '0.17.0'
+    self.base_url = 'https://nim-lang.org/docs/'
+    self.root_path = 'overview.html'
     self.links = {
       home: 'https://nim-lang.org/',
       code: 'https://github.com/nim-lang/Nim'
     }
-    self.base_url = 'https://nim-lang.org/'
-    self.root_path = 'docs/overview.html'
     html_filters.push 'nim/entries', 'nim/clean_html'
-    options[:skip] = %w(cdn-cgi/l/email-protection docs/theindex.html docs/docgen.txt)
+    options[:skip] = %w(theindex.html docgen.txt)
     options[:attribution] = <<-HTML
       &copy; 2006&ndash;2017 Andreas Rumpf<br>
-      All rights reserved. Licensed under the MIT License.
+      Licensed under the MIT License.
     HTML
   end
 end