History

Improve Ember.js scraper

Thibaut Courouble committed 10 years ago

Commit 9386a2d1 parent 3a7cc9c6 files · +157 −97Browse files

Unified · Split

6 files
  1. assets/stylesheets/global/_classes.scss +15 −0
  2. assets/stylesheets/pages/_bootstrap.scss +2 −10
  3. assets/stylesheets/pages/_ember.scss +10 −41
  4. lib/docs/filters/ember/clean_html.rb +56 −19
  5. lib/docs/filters/ember/entries.rb +35 −23
  6. lib/docs/scrapers/ember.rb +39 −4
assets/stylesheets/global/_classes.scss+15−0
@@ -47,6 +47,21 @@ · 46 unmodified lines
4747 @extend %heading-box;
4848 }
4949
50+%pre-heading {
51+ padding: .375rem .625rem;
52+ line-height: 1.5;
53+ border-bottom-left-radius: 0;
54+ border-bottom-right-radius: 0;
55+ @extend %heading-box;
56+
57+ + pre {
58+ border-top-left-radius: 0;
59+ border-top-right-radius: 0;
60+ border-top: 0;
61+ margin-top: 0;
62+ }
63+}
64+
5065 //
5166 // Notes
5267 //
assets/stylesheets/pages/_bootstrap.scss+2−10
@@ -15,22 +15,14 @@ · 14 unmodified lines
1515
1616 .text-danger { @extend %label, %label-red; }
1717
18− .bs-example {
18+ p.bs-example {
1919 padding: .375rem .625rem;
2020 line-height: 1.5;
2121 @extend %heading-box;
2222 }
2323
2424 div.bs-example {
25− border-bottom-left-radius: 0;
26− border-bottom-right-radius: 0;
27−
28− + pre {
29− border-top-left-radius: 0;
30− border-top-right-radius: 0;
31− border-top: 0;
32− margin-top: 0;
33− }
25+ @extend %pre-heading;
3426 }
3527
3628 a.thumbnail {
assets/stylesheets/pages/_ember.scss+10−41
@@ -1,53 +1,22 @@
11 ._ember {
2− > .class-info { @extend %note, %note-blue; }
3− > .class-info > p { margin: 0; }
2+ @extend %simple;
43
5− > .description > h2, > .description > h3 { font-size: 1rem; }
4+ blockquote.class-info { @extend %note-blue; }
5+ blockquote.class-info > p { margin: 0; }
66
7− .item-entry { padding-left: 1rem; }
7+ .pre-title { @extend %pre-heading; }
88
9− .title {
10− margin-left: -1rem;
11− @extend %block-heading;
12−
13− > h2, > .args, > .flag {
14− display: inline-block;
15− vertical-align: top;
16− margin: 0;
17− line-height: inherit;
18− font-size: inherit;
19− }
20−
21− > .flag { // "static"
22− margin-left: .5em;
23− color: $textColorLight;
24− }
25−
26− > .type {
27− float: right;
28− font-weight: normal;
29− }
30− }
31−
32− .meta { // "defined in"
9+ h2 > .flag, h2 > .type {
10+ margin-left: .5em;
3311 color: $textColorLight;
34− margin-bottom: 1em;
12+ font-weight: normal;
3513 }
3614
37− .return, .params {
38− margin-top: 1.5em;
15+ h2 > .type { float: right; }
3916
40− > h3 {
41− display: inline-block;
42− vertical-align: top;
43− margin: 0 0 1em;
44− font-size: inherit;
45− @extend %label, %label-blue;
46− }
47− }
17+ .meta { color: $textColorLight; }
4818
4919 dl { margin: 0 1em; }
5020 dt + dt, dd + dt { margin-top: .5em; }
51−
52− p > code { @extend %label; }
21+ dt > code { @extend %label; }
5322 }
lib/docs/filters/ember/clean_html.rb+56−19
@@ -2,7 +2,23 @@ module Docs · 1 unmodified line
22 class Ember
33 class CleanHtmlFilter < Filter
44 def call
5− root_page? ? root : other
5+ css('hr', '.edit-page').remove
6+
7+ # Remove code highlighting
8+ css('.highlight').each do |node|
9+ node.before(%(<div class="pre-title"><code>#{node.at_css('thead').content.strip}</code></div>)) if node.at_css('thead')
10+ node.content = node.at_css('.code pre').content
11+ node.name = 'pre'
12+ node['data-language'] = node['class'][/(javascript|js|html|hbs|handlebars)/, 1]
13+ node['data-language'] = node['data-language'].sub(/(hbs|handlebars)/, 'html')
14+ end
15+
16+ if base_url.path.start_with?('/api')
17+ root_page? ? root : api
18+ else
19+ guide
20+ end
21+
622 doc
723 end
824
@@ -28,8 +44,8 @@ module Docs · 19 unmodified lines
2844 end
2945 end
3046
31− def other
32− css(*%w(hr .edit-page #api-options .toc-anchor .inherited .protected .private .deprecated)).remove
47+ def api
48+ css('#api-options', '.toc-anchor', '.inherited').remove
3349
3450 # Remove tabs and "Index"
3551 css('.tabs').each do |node|
@@ -41,24 +57,45 @@ module Docs · 5 unmodified lines
4157 css('.method', '.property', '.event').remove_attr('id')
4258
4359 css('h3[data-id]').each do |node|
44− # Put id attributes on headings
45− node.name = 'h2'
46− node['id'] = node['data-id']
47− node.remove_attribute 'data-id'
48− node.content = node.content
49−
50− # Move headings, span.args, etc. into a div.title
51− div = Nokogiri::XML::Node.new 'div', doc
52− div['class'] = 'title'
53− node.before(div).parent = div
54− div.add_child(div.next_element) while div.next_element.name == 'span'
60+ heading = Nokogiri::XML::Node.new 'h2', doc
61+ heading['id'] = node['data-id']
62+ node.before(heading).remove
63+ heading.content = node.content
64+ heading.add_child(heading.next_element) while heading.next_element.name == 'span'
5565 end
5666
57− # Remove code highlighting
58− css('.highlight').each do |node|
59− node.content = node.at_css('.code pre').content
60− node.name = 'pre'
61− node['data-language'] = node['class'][/(javascript|js|html)/, 1]
67+ css('> .class-info').each do |node|
68+ node.name = 'blockquote'
69+ end
70+
71+ css('div.meta').each do |node|
72+ node.name = 'p'
73+ end
74+
75+ css('span.type').each do |node|
76+ node.name = 'code'
77+ end
78+
79+ css('.pane', '.item-entry').each do |node|
80+ node.before(node.children).remove
81+ end
82+ end
83+
84+ def guide
85+ @doc = at_css('article')
86+
87+ css('.previous-guide', '.next-guide').remove
88+
89+ css('img').each do |node|
90+ node['src'] = node['src'].sub('https://guides.emberjs.com/', base_url.to_s)
91+ end
92+
93+ css('h3, h4, h5').each do |node|
94+ node.name = node.name.sub(/\d/) { |i| i.to_i - 1 }
95+ end unless at_css('h2')
96+
97+ css('blockquote > p > em').each do |node|
98+ node.before(node.children).remove
6299 end
63100 end
64101 end
lib/docs/filters/ember/entries.rb+35−23
@@ -2,40 +2,52 @@ module Docs · 1 unmodified line
22 class Ember
33 class EntriesFilter < Docs::EntriesFilter
44 def get_name
5− name = at_css('.api-header').content.split.first
6− # Remove "Ember." prefix if the next character is uppercase
7− name.sub! %r{\AEmber\.([A-Z])(?!EATURES)}, '\1'
8− name == 'Handlebars.helpers' ? 'Handlebars Helpers' : name
5+ if base_url.path.start_with?('/api')
6+ name = at_css('.api-header').content.split.first
7+ # Remove "Ember." prefix if the next character is uppercase
8+ name.sub! %r{\AEmber\.([A-Z])(?!EATURES)}, '\1'
9+ name == 'Handlebars.helpers' ? 'Handlebars Helpers' : name
10+ else
11+ name = at_css('article h1').content.remove('Edit Page').strip
12+ name = at_css('li.toc-level-0.selected > a').content if name == 'Introduction'
13+ name
14+ end
915 end
1016
1117 def get_type
12− group = if css('p').any? { |node| node.content.include?('PRIVATE') }
13− 'Private'
14− elsif css('p').any? { |node| node.content.include?('DEPRECATED') }
15− 'Deprecated'
16− end
17−
18− if at_css('.api-header').content.include?('Module')
19− 'Modules'
20− elsif name.start_with? 'DS'
21− group ? "Data (#{group})" : 'Data'
22− elsif name.start_with? 'RSVP'
23− 'RSVP'
24− elsif name.start_with? 'Test'
25− 'Test'
18+ if base_url.path.start_with?('/api')
19+ if at_css('.api-header').content.include?('Module')
20+ 'Modules'
21+ elsif name.start_with? 'DS'
22+ 'Data'
23+ elsif name.start_with? 'RSVP'
24+ 'RSVP'
25+ elsif name.start_with? 'Test'
26+ 'Test'
27+ elsif name.start_with?('Ember')
28+ name.split('.')[0..1].join('.')
29+ else
30+ name.split('.').first
31+ end
2632 else
27− group || name
33+ if node = at_css('li.toc-level-0.selected > a')
34+ "Guide: #{node.content.strip}"
35+ else
36+ 'Guide'
37+ end
2838 end
2939 end
3040
3141 def additional_entries
32− css('.item-entry').map do |node|
33− heading = node.at_css('h2')
42+ return [] unless base_url.path.start_with?('/api')
43+
44+ css('.item-entry:not(.inherited)').map do |node|
45+ heading = node.at_css('h3[data-id]')
3446 name = heading.content.strip
3547
3648 if self.name == 'Handlebars Helpers'
3749 name << ' (handlebars helper)'
38− next [name, heading['id']]
50+ next [name, heading['data-id']]
3951 end
4052
4153 # Give their own type to "Ember.platform", "Ember.run", etc.
@@ -51,7 +63,7 @@ module Docs · 9 unmodified lines
5163 name << '()' if node['class'].include? 'method'
5264 name << ' event' if node['class'].include? 'event'
5365
54− [name, heading['id'], type]
66+ [name, heading['data-id'], type]
5567 end
5668 end
5769 end
lib/docs/scrapers/ember.rb+39−4
@@ -1,32 +1,67 @@
11 module Docs
22 class Ember < UrlScraper
3+ class << self
4+ attr_accessor :guide_url
5+ end
6+
37 self.name = 'Ember.js'
48 self.slug = 'ember'
59 self.type = 'ember'
610 self.release = '2.7.0'
711 self.base_url = 'http://emberjs.com/api/'
12+ self.guide_url = "https://guides.emberjs.com/v#{self.release}/"
13+ self.initial_urls = [guide_url]
814 self.links = {
915 home: 'http://emberjs.com/',
1016 code: 'https://github.com/emberjs/ember.js'
1117 }
1218
13− html_filters.push 'ember/clean_html', 'ember/entries', 'title'
19+ html_filters.push 'ember/entries', 'ember/clean_html', 'title'
20+
21+ options[:trailing_slash] = false
1422
1523 options[:title] = false
1624 options[:root_title] = 'Ember.js'
1725
1826 options[:container] = ->(filter) do
19− filter.root_page? ? '#toc-list' : '#content'
27+ if filter.base_url.path.start_with?('/api')
28+ filter.root_page? ? '#toc-list' : '#content'
29+ else
30+ 'main'
31+ end
2032 end
2133
2234 # Duplicates
2335 options[:skip] = %w(classes/String.html data/classes/DS.html)
24−
25− options[:skip_patterns] = [/\._/]
36+ options[:skip_patterns] = [/\._/, /contributing/]
2637
2738 options[:attribution] = <<-HTML
2839 &copy; 2016 Yehuda Katz, Tom Dale and Ember.js contributors<br>
2940 Licensed under the MIT License.
3041 HTML
42+
43+ def guide_url
44+ @guide_url ||= URL.parse(self.class.guide_url)
45+ end
46+
47+ private
48+
49+ def process_url?(url)
50+ base_url.contains?(url) || guide_url.contains?(url)
51+ end
52+
53+ def process_response(response)
54+ original_scheme = @base_url.scheme
55+ original_host = @base_url.host
56+ original_path = @base_url.path
57+ @base_url.scheme = response.effective_url.scheme
58+ @base_url.host = response.effective_url.host
59+ @base_url.path = response.effective_url.path[/\A\/v[\d\.]+\//, 0] || '/api/'
60+ super
61+ ensure
62+ @base_url.scheme = original_scheme
63+ @base_url.host = original_host
64+ @base_url.path = original_path
65+ end
3166 end
3267 end