Improve D scraper

Thibaut Courouble committed 9 years ago

Commit 8225e1d1 parent 3e0193c3 files · +31 −10Browse files

Unified · Split

3 files

assets/stylesheets/pages/_d.scss +2 −0
lib/docs/filters/d/clean_html.rb +28 −5
lib/docs/filters/d/entries.rb +1 −5

@@ -2,6 +2,8 @@ · 1 unmodified line
2	h2 { @extend %block-heading; }	2	h2 { @extend %block-heading; }
3	h3, .d_decl { @extend %block-label, %label-blue; }	3	h3, .d_decl { @extend %block-label, %label-blue; }
4	.d_decl { @extend %code; }	4	.d_decl { @extend %code; }
		5	.d_decl > small { color: $textColorLight; }
		6	.d_decl > strong { font-weight: $bolderFontWeight; }
5		7
6	p > code, li > code, td > code, dd > code { @extend %label; }	8	p > code, li > code, td > code, dd > code { @extend %label; }
7		9

lib/docs/filters/d/clean_html.rb+28−5

@@ -28,17 +28,40 @@ module Docs · 27 unmodified lines
28	node.replace("<dl><dt>#{dt}</dt><dd>#{dd}</dd></dl>")	28	node.replace("<dl><dt>#{dt}</dt><dd>#{dd}</dd></dl>")
29	end	29	end
30		30
		31	css('.description > .blankline:first-child + .quickindex').each do \|node\|
		32	node.next_element.remove if node.next_element && node.next_element['class'] == 'blankline'
		33	node.previous_element.remove
		34	node.parent.before(node)
		35	end
		36
31	css('div.summary', 'div.description').each do \|node\|	37	css('div.summary', 'div.description').each do \|node\|
32	node.name = 'p' unless node.at_css('p')	38	node.name = 'p' unless node.at_css('p')
33	node.css('.blankline').each { \|n\| n.replace('<br><br>') }	39	node.css('.blankline').each { \|n\| n.replace('<br><br>') }
34	end	40	end
35		41
36	css('.d_decl').each do \|node\|	42	css('.d_decl').each do \|node\|
37	node['id'] = node.at_css('.def-anchor')['id'].remove(/\A\./)	43	node['id'] \|\|= node.at_css('.quickindex[id]')['id'].remove('quickindex.')
38	constraints = node.css('.constraint').remove	44
39	node.content = node.content.strip	45	node.css('.def-anchor[id]').each do \|n\|
40	node.inner_html = node.inner_html.gsub(/;\s*/, '<br>').remove(/<br>\z/)	46	n.next_element['id'] \|\|= n['id']
41	node << "<br><br> Constraints:<br> #{constraints.map(&:content).join('<br> ')}" unless constraints.empty?	47	end
		48
		49	node.css('.constraint').each do \|n\|
		50	n.content = " Constraints: #{n.content}#{n.next.remove.content if n.next.text?}"
		51	n.name = 'small'
		52	n.remove_attribute('class')
		53	end
		54
		55	node.css('code[id]').each do \|n\|
		56	n.name = 'strong'
		57	n.remove_attribute('class')
		58	end
		59
		60	node.css('*').each do \|n\|
		61	n.before(n.children).remove unless n.name == 'br' \|\| n.name == 'small' \|\| n.name == 'strong'
		62	end
		63
		64	node.inner_html = node.inner_html.remove(/<br>\z/)
42	end	65	end
43		66
44	css('pre').each do \|node\|	67	css('pre').each do \|node\|

lib/docs/filters/d/entries.rb+1−5

         entries = []
-        css('.book > tr > td > a').each do |node|
-          entries << ["#{self.name}.#{node.content}", node['href'].remove(/\A#/).remove(/\A\./)]
-        end
         if entries.empty?
           css('.quickindex[id]').each do |node|
             name = node['id'].remove(/quickindex\.?/)
-            next if name.empty? || name =~ /\.\d+\z/
+            next if name.empty? || name =~ /\.\d+\z/ || name =~ /\A([^\.]+)\.\1\z/
             entries << ["#{self.name}.#{name}", name]
           end
         end

Improve D scraper

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code