History

rxjs: finish scraper and filters

Jasper van Merle committed 7 years ago

Commit 17528d91 parent 20438856 files · +116 −72Browse files

Unified · Split

6 files
  1. assets/javascripts/templates/pages/about_tmpl.coffee +10 −10
  2. lib/docs/filters/rxjs/clean_html.rb +46 −8
  3. lib/docs/filters/rxjs/entries.rb +9 −3
  4. lib/docs/scrapers/rxjs.rb +51 −51
  5. public/icons/docs/rxjs/16.png +0 −0
  6. public/icons/docs/rxjs/SOURCE +1 −1
assets/javascripts/templates/pages/about_tmpl.coffee+10−10
@@ -211,8 +211,7 @@ credits = [ · 210 unmodified lines
211 '2017 Cypress.io',211 '2017 Cypress.io',
212 'MIT',212 'MIT',
213 'https://raw.githubusercontent.com/cypress-io/cypress-documentation/develop/LICENSE.md'213 'https://raw.githubusercontent.com/cypress-io/cypress-documentation/develop/LICENSE.md'
214 ],214 ], [
215 [
216 'D',215 'D',
217 '1999-2018 The D Language Foundation',216 '1999-2018 The D Language Foundation',
218 'Boost',217 'Boost',
@@ -572,8 +571,7 @@ credits = [ · 353 unmodified lines
572 '2016-2018, The Pony Developers & 2014-2015, Causality Ltd.',571 '2016-2018, The Pony Developers & 2014-2015, Causality Ltd.',
573 'BSD',572 'BSD',
574 'https://raw.githubusercontent.com/ponylang/ponyc/master/LICENSE'573 'https://raw.githubusercontent.com/ponylang/ponyc/master/LICENSE'
575 ],574 ], [
576 [
577 'PostgreSQL',575 'PostgreSQL',
578 '1996-2018 The PostgreSQL Global Development Group<br>&copy; 1994 The Regents of the University of California',576 '1996-2018 The PostgreSQL Global Development Group<br>&copy; 1994 The Regents of the University of California',
579 'PostgreSQL',577 'PostgreSQL',
@@ -648,13 +646,17 @@ credits = [ · 68 unmodified lines
648 '2010 The Rust Project Developers',646 '2010 The Rust Project Developers',
649 'MIT',647 'MIT',
650 'https://raw.githubusercontent.com/rust-lang/rust/master/LICENSE-MIT'648 'https://raw.githubusercontent.com/rust-lang/rust/master/LICENSE-MIT'
649 ], [
650 'RxJS',
651 '2015-2018 Google, Inc., Netflix, Inc., Microsoft Corp. and contributors',
652 'Apache',
653 'https://raw.githubusercontent.com/ReactiveX/rxjs/master/LICENSE.txt'
651 ], [654 ], [
652 'Salt Stack',655 'Salt Stack',
653 '2019 SaltStack',656 '2019 SaltStack',
654 'Apache',657 'Apache',
655 'https://raw.githubusercontent.com/saltstack/salt/develop/LICENSE'658 'https://raw.githubusercontent.com/saltstack/salt/develop/LICENSE'
656 ],659 ], [
657 [
658 'Sass',660 'Sass',
659 '2006-2016 Hampton Catlin, Nathan Weizenbaum, and Chris Eppstein',661 '2006-2016 Hampton Catlin, Nathan Weizenbaum, and Chris Eppstein',
660 'MIT',662 'MIT',
@@ -664,8 +666,7 @@ credits = [ · 3 unmodified lines
664 '2002-2019 EPFL, with contributions from Lightbend',666 '2002-2019 EPFL, with contributions from Lightbend',
665 'Apache',667 'Apache',
666 'https://raw.githubusercontent.com/scala/scala-lang/master/license.md'668 'https://raw.githubusercontent.com/scala/scala-lang/master/license.md'
667 ],669 ], [
668 [
669 'scikit-image',670 'scikit-image',
670 '2011 the scikit-image team',671 '2011 the scikit-image team',
671 'BSD',672 'BSD',
@@ -765,8 +766,7 @@ credits = [ · 93 unmodified lines
765 '2003-2019 WordPress Foundation',766 '2003-2019 WordPress Foundation',
766 'GPLv2+',767 'GPLv2+',
767 'https://wordpress.org/about/license/'768 'https://wordpress.org/about/license/'
768 ],769 ], [
769 [
770 'Yarn',770 'Yarn',
771 '2016-present Yarn Contributors',771 '2016-present Yarn Contributors',
772 'BSD',772 'BSD',
lib/docs/filters/rxjs/clean_html.rb+46−8
@@ -7,6 +7,11 @@ module Docs · 6 unmodified lines
7 at_css('h1').content = 'RxJS Documentation'7 at_css('h1').content = 'RxJS Documentation'
8 end8 end
9 9
10 if at_css('h1').nil?
11 title = subpath.rpartition('/').last.titleize
12 doc.prepend_child("<h1>#{title}</h1>")
13 end
14
10 css('br', 'hr', '.material-icons', '.header-link', '.breadcrumb').remove15 css('br', 'hr', '.material-icons', '.header-link', '.breadcrumb').remove
11 16
12 css('.content', 'article', '.api-header', 'section', '.instance-member').each do |node|17 css('.content', 'article', '.api-header', 'section', '.instance-member').each do |node|
@@ -65,6 +70,16 @@ module Docs · 52 unmodified lines
65 70
66 if node['class'] && node['class'].include?('api-heading')71 if node['class'] && node['class'].include?('api-heading')
67 node.name = 'h3'72 node.name = 'h3'
73
74 unless node.ancestors('.instance-method').empty?
75 matches = node.inner_html.scan(/([^(& ]+)[(&]/)
76
77 unless matches.empty? || matches[0][0] == 'constructor'
78 node['name'] = matches[0][0]
79 node['id'] = node['name'].downcase + '-'
80 end
81 end
82
68 node.inner_html = "<code>#{node.inner_html}</code>"83 node.inner_html = "<code>#{node.inner_html}</code>"
69 end84 end
70 85
@@ -77,25 +92,48 @@ module Docs · 6 unmodified lines
77 node.remove_attribute('class')92 node.remove_attribute('class')
78 end93 end
79 94
80 css('h1[class]').remove_attr('class')95 css('td > .overloads').each do |node|
81 css('table[class]').remove_attr('class')96 node.replace node.at_css('.detail-contents')
82 css('table[width]').remove_attr('width')97 end
83 css('tr[style]').remove_attr('style')98
99 css('td.short-description p').each do |node|
100 signature = node.parent.parent.next_element.at_css('h3[id]')
101 signature.after(node) unless signature.nil?
102 end
84 103
85 if at_css('.api-type-label.module')104 css('.method-table').each do |node|
86 at_css('h1').content = subpath.remove('api/')105 node.replace node.at_css('tbody')
87 end106 end
88 107
89 css('th h3').each do |node|108 css('.api-body > table > caption').each do |node|
90 node.name = 'span'109 node.name = 'center'
110 lift_out_of_table node
91 end111 end
92 112
113 css('.api-body > table > tbody > tr:not([class]) > td > *').each do |node|
114 lift_out_of_table node
115 end
116
117 css('.api-body > table').each do |node|
118 node.remove if node.content.strip.blank?
119 end
120
121 css('h1[class]').remove_attr('class')
122 css('table[class]').remove_attr('class')
123 css('table[width]').remove_attr('width')
124 css('tr[style]').remove_attr('style')
125
93 css('code code').each do |node|126 css('code code').each do |node|
94 node.before(node.children).remove127 node.before(node.children).remove
95 end128 end
96 129
97 doc130 doc
98 end131 end
132
133 def lift_out_of_table(node)
134 table = node.ancestors('table').first
135 table.previous_element.after(node)
136 end
99 end137 end
100 end138 end
101end139end
lib/docs/filters/rxjs/entries.rb+9−3
@@ -2,22 +2,28 @@ module Docs · 1 unmodified line
2 class Rxjs2 class Rxjs
3 class EntriesFilter < Docs::EntriesFilter3 class EntriesFilter < Docs::EntriesFilter
4 def get_name4 def get_name
5 name = at_css('h1').content5 title = at_css('h1')
6 name = title.nil? ? subpath.rpartition('/').last.titleize : title.content
6 name.prepend "#{$1}. " if subpath =~ /\-pt(\d+)/7 name.prepend "#{$1}. " if subpath =~ /\-pt(\d+)/
8 name += '()' unless at_css('.api-type-label.function').nil?
7 name9 name
8 end10 end
9 11
10 def get_type12 def get_type
11 if slug.start_with?('guide')13 if slug.start_with?('guide')
12 'Guide'14 'Guide'
13 elsif at_css('.api-type-label.module')
14 name.split('/').first
15 elsif slug.start_with?('api/')15 elsif slug.start_with?('api/')
16 slug.split('/').second16 slug.split('/').second
17 else17 else
18 'Miscellaneous'18 'Miscellaneous'
19 end19 end
20 end20 end
21
22 def additional_entries
23 css('h3[id]').map do |node|
24 ["#{name}.#{node['name']}()", node['id']]
25 end
26 end
21 end27 end
22 end28 end
23end29end
lib/docs/scrapers/rxjs.rb+51−51
@@ -4,11 +4,26 @@ module Docs · 3 unmodified lines
4 class Rxjs < UrlScraper4 class Rxjs < UrlScraper
5 self.name = 'RxJS'5 self.name = 'RxJS'
6 self.type = 'rxjs'6 self.type = 'rxjs'
7 self.release = '6.5.2'
8 self.base_url = 'https://rxjs.dev/'
9 self.root_path = 'guide/overview'
7 self.links = {10 self.links = {
8 home: 'https://rxjs.dev/',11 home: 'https://rxjs.dev/',
9 code: 'https://github.com/ReactiveX/rxjs'12 code: 'https://github.com/ReactiveX/rxjs'
10 }13 }
11 14
15 html_filters.push 'rxjs/clean_html', 'rxjs/entries'
16
17 options[:follow_links] = false
18 options[:only_patterns] = [/guide\//, /api\//]
19 options[:skip_patterns] = [/api\/([^\/]+)\.json/]
20 options[:fix_urls_before_parse] = ->(url) do
21 url.sub! %r{\Aguide/}, '/guide/'
22 url.sub! %r{\Aapi/}, '/api/'
23 url.sub! %r{\Agenerated/}, '/generated/'
24 url
25 end
26
12 options[:max_image_size] = 256_00027 options[:max_image_size] = 256_000
13 28
14 options[:attribution] = <<-HTML29 options[:attribution] = <<-HTML
@@ -16,69 +31,54 @@ module Docs · 1 unmodified line
16 Code licensed under an Apache-2.0 License. Documentation licensed under CC BY 4.0.31 Code licensed under an Apache-2.0 License. Documentation licensed under CC BY 4.0.
17 HTML32 HTML
18 33
19 module Common34 def get_latest_version(opts)
20 private35 json = fetch_json('https://rxjs.dev/generated/navigation.json', opts)
36 json['__versionInfo']['raw']
37 end
21 38
22 def initial_urls39 private
23 initial_urls = []
24 40
25 Request.run "#{self.class.base_url}generated/navigation.json" do |response|41 def initial_urls
26 data = JSON.parse(response.body)42 initial_urls = []
27 dig = ->(entry) do
28 initial_urls << url_for("generated/docs/#{entry['url']}.json") if entry['url'] && entry['url'] != 'api'
29 entry['children'].each(&dig) if entry['children']
30 end
31 data['SideNav'].each(&dig)
32 end
33 43
34 Request.run "#{self.class.base_url}generated/docs/api/api-list.json" do |response|44 Request.run "#{self.class.base_url}generated/navigation.json" do |response|
35 data = JSON.parse(response.body)45 data = JSON.parse(response.body)
36 dig = ->(entry) do46 dig = ->(entry) do
37 initial_urls << url_for("generated/docs/#{entry['path']}.json") if entry['path']47 initial_urls << url_for("generated/docs/#{entry['url']}.json") if entry['url'] && entry['url'] != 'api'
38 initial_urls << url_for("generated/docs/api/#{entry['name']}.json") if entry['name'] && !entry['path']48 entry['children'].each(&dig) if entry['children']
39 entry['items'].each(&dig) if entry['items']
40 end
41 data.each(&dig)
42 end49 end
43 50 data['SideNav'].each(&dig)
44 initial_urls
45 end51 end
46 52
47 def handle_response(response)53 Request.run "#{self.class.base_url}generated/docs/api/api-list.json" do |response|
48 if response.mime_type.include?('json')54 data = JSON.parse(response.body)
49 begin55 dig = ->(entry) do
50 response.options[:response_body] = JSON.parse(response.body)['contents']56 initial_urls << url_for("generated/docs/#{entry['path']}.json") if entry['path']
51 rescue JSON::ParserError57 initial_urls << url_for("generated/docs/api/#{entry['name']}.json") if entry['name'] && !entry['path']
52 response.options[:response_body] = ''58 entry['items'].each(&dig) if entry['items']
53 end
54 response.headers['Content-Type'] = 'text/html'
55 response.url.path = response.url.path.sub('/generated/docs/', '/').remove('.json')
56 response.effective_url.path = response.effective_url.path.sub('/generated/docs/', '/').remove('.json')
57 end59 end
58 super60 data.each(&dig)
59 end61 end
60 end
61 62
62 version do63 initial_urls.select do |url|
63 self.release = '6.3.3'64 options[:only_patterns].any? { |pattern| url =~ pattern } &&
64 self.base_url = 'https://rxjs.dev/'65 options[:skip_patterns].none? { |pattern| url =~ pattern }
65 self.root_path = 'guide/overview'
66
67 html_filters.push 'rxjs/clean_html', 'rxjs/entries'
68
69 options[:follow_links] = false
70 options[:only_patterns] = [/\Aguide/, /\Aapi/]
71 options[:fix_urls_before_parse] = ->(url) do
72 url.sub! %r{\Aguide/}, '/guide/'
73 url.sub! %r{\Aapi/}, '/api/'
74 url.sub! %r{\Agenerated/}, '/generated/'
75 url
76 end66 end
77
78 include Docs::Rxjs::Common
79 end67 end
80 68
81 private69 def handle_response(response)
70 if response.mime_type.include?('json')
71 begin
72 response.options[:response_body] = JSON.parse(response.body)['contents']
73 rescue JSON::ParserError
74 response.options[:response_body] = ''
75 end
76 response.headers['Content-Type'] = 'text/html'
77 response.url.path = response.url.path.sub('/generated/docs/', '/').remove('.json')
78 response.effective_url.path = response.effective_url.path.sub('/generated/docs/', '/').remove('.json')
79 end
80 super
81 end
82 82
83 def parse(response)83 def parse(response)
84 response.body.gsub! '<code-example', '<pre'84 response.body.gsub! '<code-example', '<pre'
public/icons/docs/rxjs/16.png+0−0

Binary file not shown.

public/icons/docs/rxjs/SOURCE+1−1
@@ -1 +1 @@
1http://reactivex.io/1https://github.com/ReactiveX/reactivex.github.io/blob/develop/favicon.ico