History

Improve D scraper

Thibaut Courouble committed 9 years ago

Commit 8225e1d1 parent 3e0193c3 files · +31 −10Browse files

Unified · Split

3 files
  1. assets/stylesheets/pages/_d.scss +2 −0
  2. lib/docs/filters/d/clean_html.rb +28 −5
  3. lib/docs/filters/d/entries.rb +1 −5
assets/stylesheets/pages/_d.scss+2−0
@@ -2,6 +2,8 @@ · 1 unmodified line
22 h2 { @extend %block-heading; }
33 h3, .d_decl { @extend %block-label, %label-blue; }
44 .d_decl { @extend %code; }
5+ .d_decl > small { color: $textColorLight; }
6+ .d_decl > strong { font-weight: $bolderFontWeight; }
57
68 p > code, li > code, td > code, dd > code { @extend %label; }
79
lib/docs/filters/d/clean_html.rb+28−5
@@ -28,17 +28,40 @@ module Docs · 27 unmodified lines
2828 node.replace("<dl><dt>#{dt}</dt><dd>#{dd}</dd></dl>")
2929 end
3030
31+ css('.description > .blankline:first-child + .quickindex').each do |node|
32+ node.next_element.remove if node.next_element && node.next_element['class'] == 'blankline'
33+ node.previous_element.remove
34+ node.parent.before(node)
35+ end
36+
3137 css('div.summary', 'div.description').each do |node|
3238 node.name = 'p' unless node.at_css('p')
3339 node.css('.blankline').each { |n| n.replace('<br><br>') }
3440 end
3541
3642 css('.d_decl').each do |node|
37− node['id'] = node.at_css('.def-anchor')['id'].remove(/\A\./)
38− constraints = node.css('.constraint').remove
39− node.content = node.content.strip
40− node.inner_html = node.inner_html.gsub(/;\s*/, '<br>').remove(/<br>\z/)
41− node << "<br><br> Constraints:<br> #{constraints.map(&:content).join('<br> ')}" unless constraints.empty?
43+ node['id'] ||= node.at_css('.quickindex[id]')['id'].remove('quickindex.')
44+
45+ node.css('.def-anchor[id]').each do |n|
46+ n.next_element['id'] ||= n['id']
47+ end
48+
49+ node.css('.constraint').each do |n|
50+ n.content = " Constraints: #{n.content}#{n.next.remove.content if n.next.text?}"
51+ n.name = 'small'
52+ n.remove_attribute('class')
53+ end
54+
55+ node.css('code[id]').each do |n|
56+ n.name = 'strong'
57+ n.remove_attribute('class')
58+ end
59+
60+ node.css('*').each do |n|
61+ n.before(n.children).remove unless n.name == 'br' || n.name == 'small' || n.name == 'strong'
62+ end
63+
64+ node.inner_html = node.inner_html.remove(/<br>\z/)
4265 end
4366
4467 css('pre').each do |node|
lib/docs/filters/d/entries.rb+1−5
@@ -29,14 +29,10 @@ module Docs · 28 unmodified lines
2929
3030 entries = []
3131
32− css('.book > tr > td > a').each do |node|
33− entries << ["#{self.name}.#{node.content}", node['href'].remove(/\A#/).remove(/\A\./)]
34− end
35−
3632 if entries.empty?
3733 css('.quickindex[id]').each do |node|
3834 name = node['id'].remove(/quickindex\.?/)
39− next if name.empty? || name =~ /\.\d+\z/
35+ next if name.empty? || name =~ /\.\d+\z/ || name =~ /\A([^\.]+)\.\1\z/
4036 entries << ["#{self.name}.#{name}", name]
4137 end
4238 end

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t