History

Improve Ember.js scraper

Thibaut Courouble committed 10 years ago

Commit 9386a2d1 parent 3a7cc9c6 files · +157 −97Browse files

Unified · Split

6 files
  1. assets/stylesheets/global/_classes.scss +15 −0
  2. assets/stylesheets/pages/_bootstrap.scss +2 −10
  3. assets/stylesheets/pages/_ember.scss +10 −41
  4. lib/docs/filters/ember/clean_html.rb +56 −19
  5. lib/docs/filters/ember/entries.rb +35 −23
  6. lib/docs/scrapers/ember.rb +39 −4
assets/stylesheets/global/_classes.scss+15−0
@@ -47,6 +47,21 @@ · 46 unmodified lines
47 @extend %heading-box;47 @extend %heading-box;
48}48}
49 49
50%pre-heading {
51 padding: .375rem .625rem;
52 line-height: 1.5;
53 border-bottom-left-radius: 0;
54 border-bottom-right-radius: 0;
55 @extend %heading-box;
56
57 + pre {
58 border-top-left-radius: 0;
59 border-top-right-radius: 0;
60 border-top: 0;
61 margin-top: 0;
62 }
63}
64
50//65//
51// Notes66// Notes
52//67//
assets/stylesheets/pages/_bootstrap.scss+2−10
@@ -15,22 +15,14 @@ · 14 unmodified lines
15 15
16 .text-danger { @extend %label, %label-red; }16 .text-danger { @extend %label, %label-red; }
17 17
18 .bs-example {18 p.bs-example {
19 padding: .375rem .625rem;19 padding: .375rem .625rem;
20 line-height: 1.5;20 line-height: 1.5;
21 @extend %heading-box;21 @extend %heading-box;
22 }22 }
23 23
24 div.bs-example {24 div.bs-example {
25 border-bottom-left-radius: 0;25 @extend %pre-heading;
26 border-bottom-right-radius: 0;
27
28 + pre {
29 border-top-left-radius: 0;
30 border-top-right-radius: 0;
31 border-top: 0;
32 margin-top: 0;
33 }
34 }26 }
35 27
36 a.thumbnail {28 a.thumbnail {
assets/stylesheets/pages/_ember.scss+10−41
@@ -1,53 +1,22 @@
1._ember {1._ember {
2 > .class-info { @extend %note, %note-blue; }2 @extend %simple;
3 > .class-info > p { margin: 0; }
4 3
5 > .description > h2, > .description > h3 { font-size: 1rem; }4 blockquote.class-info { @extend %note-blue; }
5 blockquote.class-info > p { margin: 0; }
6 6
7 .item-entry { padding-left: 1rem; }7 .pre-title { @extend %pre-heading; }
8 8
9 .title {9 h2 > .flag, h2 > .type {
10 margin-left: -1rem;10 margin-left: .5em;
11 @extend %block-heading;
12
13 > h2, > .args, > .flag {
14 display: inline-block;
15 vertical-align: top;
16 margin: 0;
17 line-height: inherit;
18 font-size: inherit;
19 }
20
21 > .flag { // "static"
22 margin-left: .5em;
23 color: $textColorLight;
24 }
25
26 > .type {
27 float: right;
28 font-weight: normal;
29 }
30 }
31
32 .meta { // "defined in"
33 color: $textColorLight;11 color: $textColorLight;
34 margin-bottom: 1em;12 font-weight: normal;
35 }13 }
36 14
37 .return, .params {15 h2 > .type { float: right; }
38 margin-top: 1.5em;
39 16
40 > h3 {17 .meta { color: $textColorLight; }
41 display: inline-block;
42 vertical-align: top;
43 margin: 0 0 1em;
44 font-size: inherit;
45 @extend %label, %label-blue;
46 }
47 }
48 18
49 dl { margin: 0 1em; }19 dl { margin: 0 1em; }
50 dt + dt, dd + dt { margin-top: .5em; }20 dt + dt, dd + dt { margin-top: .5em; }
51 21 dt > code { @extend %label; }
52 p > code { @extend %label; }
53}22}
lib/docs/filters/ember/clean_html.rb+56−19
@@ -2,7 +2,23 @@ module Docs · 1 unmodified line
2 class Ember2 class Ember
3 class CleanHtmlFilter < Filter3 class CleanHtmlFilter < Filter
4 def call4 def call
5 root_page? ? root : other5 css('hr', '.edit-page').remove
6
7 # Remove code highlighting
8 css('.highlight').each do |node|
9 node.before(%(<div class="pre-title"><code>#{node.at_css('thead').content.strip}</code></div>)) if node.at_css('thead')
10 node.content = node.at_css('.code pre').content
11 node.name = 'pre'
12 node['data-language'] = node['class'][/(javascript|js|html|hbs|handlebars)/, 1]
13 node['data-language'] = node['data-language'].sub(/(hbs|handlebars)/, 'html')
14 end
15
16 if base_url.path.start_with?('/api')
17 root_page? ? root : api
18 else
19 guide
20 end
21
6 doc22 doc
7 end23 end
8 24
@@ -28,8 +44,8 @@ module Docs · 19 unmodified lines
28 end44 end
29 end45 end
30 46
31 def other47 def api
32 css(*%w(hr .edit-page #api-options .toc-anchor .inherited .protected .private .deprecated)).remove48 css('#api-options', '.toc-anchor', '.inherited').remove
33 49
34 # Remove tabs and "Index"50 # Remove tabs and "Index"
35 css('.tabs').each do |node|51 css('.tabs').each do |node|
@@ -41,24 +57,45 @@ module Docs · 5 unmodified lines
41 css('.method', '.property', '.event').remove_attr('id')57 css('.method', '.property', '.event').remove_attr('id')
42 58
43 css('h3[data-id]').each do |node|59 css('h3[data-id]').each do |node|
44 # Put id attributes on headings60 heading = Nokogiri::XML::Node.new 'h2', doc
45 node.name = 'h2'61 heading['id'] = node['data-id']
46 node['id'] = node['data-id']62 node.before(heading).remove
47 node.remove_attribute 'data-id'63 heading.content = node.content
48 node.content = node.content64 heading.add_child(heading.next_element) while heading.next_element.name == 'span'
49
50 # Move headings, span.args, etc. into a div.title
51 div = Nokogiri::XML::Node.new 'div', doc
52 div['class'] = 'title'
53 node.before(div).parent = div
54 div.add_child(div.next_element) while div.next_element.name == 'span'
55 end65 end
56 66
57 # Remove code highlighting67 css('> .class-info').each do |node|
58 css('.highlight').each do |node|68 node.name = 'blockquote'
59 node.content = node.at_css('.code pre').content69 end
60 node.name = 'pre'70
61 node['data-language'] = node['class'][/(javascript|js|html)/, 1]71 css('div.meta').each do |node|
72 node.name = 'p'
73 end
74
75 css('span.type').each do |node|
76 node.name = 'code'
77 end
78
79 css('.pane', '.item-entry').each do |node|
80 node.before(node.children).remove
81 end
82 end
83
84 def guide
85 @doc = at_css('article')
86
87 css('.previous-guide', '.next-guide').remove
88
89 css('img').each do |node|
90 node['src'] = node['src'].sub('https://guides.emberjs.com/', base_url.to_s)
91 end
92
93 css('h3, h4, h5').each do |node|
94 node.name = node.name.sub(/\d/) { |i| i.to_i - 1 }
95 end unless at_css('h2')
96
97 css('blockquote > p > em').each do |node|
98 node.before(node.children).remove
62 end99 end
63 end100 end
64 end101 end
lib/docs/filters/ember/entries.rb+35−23
@@ -2,40 +2,52 @@ module Docs · 1 unmodified line
2 class Ember2 class Ember
3 class EntriesFilter < Docs::EntriesFilter3 class EntriesFilter < Docs::EntriesFilter
4 def get_name4 def get_name
5 name = at_css('.api-header').content.split.first5 if base_url.path.start_with?('/api')
6 # Remove "Ember." prefix if the next character is uppercase6 name = at_css('.api-header').content.split.first
7 name.sub! %r{\AEmber\.([A-Z])(?!EATURES)}, '\1'7 # Remove "Ember." prefix if the next character is uppercase
8 name == 'Handlebars.helpers' ? 'Handlebars Helpers' : name8 name.sub! %r{\AEmber\.([A-Z])(?!EATURES)}, '\1'
9 name == 'Handlebars.helpers' ? 'Handlebars Helpers' : name
10 else
11 name = at_css('article h1').content.remove('Edit Page').strip
12 name = at_css('li.toc-level-0.selected > a').content if name == 'Introduction'
13 name
14 end
9 end15 end
10 16
11 def get_type17 def get_type
12 group = if css('p').any? { |node| node.content.include?('PRIVATE') }18 if base_url.path.start_with?('/api')
13 'Private'19 if at_css('.api-header').content.include?('Module')
14 elsif css('p').any? { |node| node.content.include?('DEPRECATED') }20 'Modules'
15 'Deprecated'21 elsif name.start_with? 'DS'
16 end22 'Data'
17 23 elsif name.start_with? 'RSVP'
18 if at_css('.api-header').content.include?('Module')24 'RSVP'
19 'Modules'25 elsif name.start_with? 'Test'
20 elsif name.start_with? 'DS'26 'Test'
21 group ? "Data (#{group})" : 'Data'27 elsif name.start_with?('Ember')
22 elsif name.start_with? 'RSVP'28 name.split('.')[0..1].join('.')
23 'RSVP'29 else
24 elsif name.start_with? 'Test'30 name.split('.').first
25 'Test'31 end
26 else32 else
27 group || name33 if node = at_css('li.toc-level-0.selected > a')
34 "Guide: #{node.content.strip}"
35 else
36 'Guide'
37 end
28 end38 end
29 end39 end
30 40
31 def additional_entries41 def additional_entries
32 css('.item-entry').map do |node|42 return [] unless base_url.path.start_with?('/api')
33 heading = node.at_css('h2')43
44 css('.item-entry:not(.inherited)').map do |node|
45 heading = node.at_css('h3[data-id]')
34 name = heading.content.strip46 name = heading.content.strip
35 47
36 if self.name == 'Handlebars Helpers'48 if self.name == 'Handlebars Helpers'
37 name << ' (handlebars helper)'49 name << ' (handlebars helper)'
38 next [name, heading['id']]50 next [name, heading['data-id']]
39 end51 end
40 52
41 # Give their own type to "Ember.platform", "Ember.run", etc.53 # Give their own type to "Ember.platform", "Ember.run", etc.
@@ -51,7 +63,7 @@ module Docs · 9 unmodified lines
51 name << '()' if node['class'].include? 'method'63 name << '()' if node['class'].include? 'method'
52 name << ' event' if node['class'].include? 'event'64 name << ' event' if node['class'].include? 'event'
53 65
54 [name, heading['id'], type]66 [name, heading['data-id'], type]
55 end67 end
56 end68 end
57 end69 end
lib/docs/scrapers/ember.rb+39−4
@@ -1,32 +1,67 @@
1module Docs1module Docs
2 class Ember < UrlScraper2 class Ember < UrlScraper
3 class << self
4 attr_accessor :guide_url
5 end
6
3 self.name = 'Ember.js'7 self.name = 'Ember.js'
4 self.slug = 'ember'8 self.slug = 'ember'
5 self.type = 'ember'9 self.type = 'ember'
6 self.release = '2.7.0'10 self.release = '2.7.0'
7 self.base_url = 'http://emberjs.com/api/'11 self.base_url = 'http://emberjs.com/api/'
12 self.guide_url = "https://guides.emberjs.com/v#{self.release}/"
13 self.initial_urls = [guide_url]
8 self.links = {14 self.links = {
9 home: 'http://emberjs.com/',15 home: 'http://emberjs.com/',
10 code: 'https://github.com/emberjs/ember.js'16 code: 'https://github.com/emberjs/ember.js'
11 }17 }
12 18
13 html_filters.push 'ember/clean_html', 'ember/entries', 'title'19 html_filters.push 'ember/entries', 'ember/clean_html', 'title'
20
21 options[:trailing_slash] = false
14 22
15 options[:title] = false23 options[:title] = false
16 options[:root_title] = 'Ember.js'24 options[:root_title] = 'Ember.js'
17 25
18 options[:container] = ->(filter) do26 options[:container] = ->(filter) do
19 filter.root_page? ? '#toc-list' : '#content'27 if filter.base_url.path.start_with?('/api')
28 filter.root_page? ? '#toc-list' : '#content'
29 else
30 'main'
31 end
20 end32 end
21 33
22 # Duplicates34 # Duplicates
23 options[:skip] = %w(classes/String.html data/classes/DS.html)35 options[:skip] = %w(classes/String.html data/classes/DS.html)
24 36 options[:skip_patterns] = [/\._/, /contributing/]
25 options[:skip_patterns] = [/\._/]
26 37
27 options[:attribution] = <<-HTML38 options[:attribution] = <<-HTML
28 &copy; 2016 Yehuda Katz, Tom Dale and Ember.js contributors<br>39 &copy; 2016 Yehuda Katz, Tom Dale and Ember.js contributors<br>
29 Licensed under the MIT License.40 Licensed under the MIT License.
30 HTML41 HTML
42
43 def guide_url
44 @guide_url ||= URL.parse(self.class.guide_url)
45 end
46
47 private
48
49 def process_url?(url)
50 base_url.contains?(url) || guide_url.contains?(url)
51 end
52
53 def process_response(response)
54 original_scheme = @base_url.scheme
55 original_host = @base_url.host
56 original_path = @base_url.path
57 @base_url.scheme = response.effective_url.scheme
58 @base_url.host = response.effective_url.host
59 @base_url.path = response.effective_url.path[/\A\/v[\d\.]+\//, 0] || '/api/'
60 super
61 ensure
62 @base_url.scheme = original_scheme
63 @base_url.host = original_host
64 @base_url.path = original_path
65 end
31 end66 end
32end67end