History

rxjs: finish scraper and filters

Jasper van Merle committed 7 years ago

Commit 17528d91 parent 20438856 files · +116 −72Browse files

Unified · Split

6 files
  1. assets/javascripts/templates/pages/about_tmpl.coffee +10 −10
  2. lib/docs/filters/rxjs/clean_html.rb +46 −8
  3. lib/docs/filters/rxjs/entries.rb +9 −3
  4. lib/docs/scrapers/rxjs.rb +51 −51
  5. public/icons/docs/rxjs/16.png +0 −0
  6. public/icons/docs/rxjs/SOURCE +1 −1
assets/javascripts/templates/pages/about_tmpl.coffee+10−10
@@ -211,8 +211,7 @@ credits = [ · 210 unmodified lines
211211 '2017 Cypress.io',
212212 'MIT',
213213 'https://raw.githubusercontent.com/cypress-io/cypress-documentation/develop/LICENSE.md'
214− ],
215− [
214+ ], [
216215 'D',
217216 '1999-2018 The D Language Foundation',
218217 'Boost',
@@ -572,8 +571,7 @@ credits = [ · 353 unmodified lines
572571 '2016-2018, The Pony Developers & 2014-2015, Causality Ltd.',
573572 'BSD',
574573 'https://raw.githubusercontent.com/ponylang/ponyc/master/LICENSE'
575− ],
576− [
574+ ], [
577575 'PostgreSQL',
578576 '1996-2018 The PostgreSQL Global Development Group<br>&copy; 1994 The Regents of the University of California',
579577 'PostgreSQL',
@@ -648,13 +646,17 @@ credits = [ · 68 unmodified lines
648646 '2010 The Rust Project Developers',
649647 'MIT',
650648 'https://raw.githubusercontent.com/rust-lang/rust/master/LICENSE-MIT'
649+ ], [
650+ 'RxJS',
651+ '2015-2018 Google, Inc., Netflix, Inc., Microsoft Corp. and contributors',
652+ 'Apache',
653+ 'https://raw.githubusercontent.com/ReactiveX/rxjs/master/LICENSE.txt'
651654 ], [
652655 'Salt Stack',
653656 '2019 SaltStack',
654657 'Apache',
655658 'https://raw.githubusercontent.com/saltstack/salt/develop/LICENSE'
656− ],
657− [
659+ ], [
658660 'Sass',
659661 '2006-2016 Hampton Catlin, Nathan Weizenbaum, and Chris Eppstein',
660662 'MIT',
@@ -664,8 +666,7 @@ credits = [ · 3 unmodified lines
664666 '2002-2019 EPFL, with contributions from Lightbend',
665667 'Apache',
666668 'https://raw.githubusercontent.com/scala/scala-lang/master/license.md'
667− ],
668− [
669+ ], [
669670 'scikit-image',
670671 '2011 the scikit-image team',
671672 'BSD',
@@ -765,8 +766,7 @@ credits = [ · 93 unmodified lines
765766 '2003-2019 WordPress Foundation',
766767 'GPLv2+',
767768 'https://wordpress.org/about/license/'
768− ],
769− [
769+ ], [
770770 'Yarn',
771771 '2016-present Yarn Contributors',
772772 'BSD',
lib/docs/filters/rxjs/clean_html.rb+46−8
@@ -7,6 +7,11 @@ module Docs · 6 unmodified lines
77 at_css('h1').content = 'RxJS Documentation'
88 end
99
10+ if at_css('h1').nil?
11+ title = subpath.rpartition('/').last.titleize
12+ doc.prepend_child("<h1>#{title}</h1>")
13+ end
14+
1015 css('br', 'hr', '.material-icons', '.header-link', '.breadcrumb').remove
1116
1217 css('.content', 'article', '.api-header', 'section', '.instance-member').each do |node|
@@ -65,6 +70,16 @@ module Docs · 52 unmodified lines
6570
6671 if node['class'] && node['class'].include?('api-heading')
6772 node.name = 'h3'
73+
74+ unless node.ancestors('.instance-method').empty?
75+ matches = node.inner_html.scan(/([^(& ]+)[(&]/)
76+
77+ unless matches.empty? || matches[0][0] == 'constructor'
78+ node['name'] = matches[0][0]
79+ node['id'] = node['name'].downcase + '-'
80+ end
81+ end
82+
6883 node.inner_html = "<code>#{node.inner_html}</code>"
6984 end
7085
@@ -77,25 +92,48 @@ module Docs · 6 unmodified lines
7792 node.remove_attribute('class')
7893 end
7994
80− css('h1[class]').remove_attr('class')
81− css('table[class]').remove_attr('class')
82− css('table[width]').remove_attr('width')
83− css('tr[style]').remove_attr('style')
95+ css('td > .overloads').each do |node|
96+ node.replace node.at_css('.detail-contents')
97+ end
98+
99+ css('td.short-description p').each do |node|
100+ signature = node.parent.parent.next_element.at_css('h3[id]')
101+ signature.after(node) unless signature.nil?
102+ end
84103
85− if at_css('.api-type-label.module')
86− at_css('h1').content = subpath.remove('api/')
104+ css('.method-table').each do |node|
105+ node.replace node.at_css('tbody')
87106 end
88107
89− css('th h3').each do |node|
90− node.name = 'span'
108+ css('.api-body > table > caption').each do |node|
109+ node.name = 'center'
110+ lift_out_of_table node
91111 end
92112
113+ css('.api-body > table > tbody > tr:not([class]) > td > *').each do |node|
114+ lift_out_of_table node
115+ end
116+
117+ css('.api-body > table').each do |node|
118+ node.remove if node.content.strip.blank?
119+ end
120+
121+ css('h1[class]').remove_attr('class')
122+ css('table[class]').remove_attr('class')
123+ css('table[width]').remove_attr('width')
124+ css('tr[style]').remove_attr('style')
125+
93126 css('code code').each do |node|
94127 node.before(node.children).remove
95128 end
96129
97130 doc
98131 end
132+
133+ def lift_out_of_table(node)
134+ table = node.ancestors('table').first
135+ table.previous_element.after(node)
136+ end
99137 end
100138 end
101139 end
lib/docs/filters/rxjs/entries.rb+9−3
@@ -2,22 +2,28 @@ module Docs · 1 unmodified line
22 class Rxjs
33 class EntriesFilter < Docs::EntriesFilter
44 def get_name
5− name = at_css('h1').content
5+ title = at_css('h1')
6+ name = title.nil? ? subpath.rpartition('/').last.titleize : title.content
67 name.prepend "#{$1}. " if subpath =~ /\-pt(\d+)/
8+ name += '()' unless at_css('.api-type-label.function').nil?
79 name
810 end
911
1012 def get_type
1113 if slug.start_with?('guide')
1214 'Guide'
13− elsif at_css('.api-type-label.module')
14− name.split('/').first
1515 elsif slug.start_with?('api/')
1616 slug.split('/').second
1717 else
1818 'Miscellaneous'
1919 end
2020 end
21+
22+ def additional_entries
23+ css('h3[id]').map do |node|
24+ ["#{name}.#{node['name']}()", node['id']]
25+ end
26+ end
2127 end
2228 end
2329 end
lib/docs/scrapers/rxjs.rb+51−51
@@ -4,11 +4,26 @@ module Docs · 3 unmodified lines
44 class Rxjs < UrlScraper
55 self.name = 'RxJS'
66 self.type = 'rxjs'
7+ self.release = '6.5.2'
8+ self.base_url = 'https://rxjs.dev/'
9+ self.root_path = 'guide/overview'
710 self.links = {
811 home: 'https://rxjs.dev/',
912 code: 'https://github.com/ReactiveX/rxjs'
1013 }
1114
15+ html_filters.push 'rxjs/clean_html', 'rxjs/entries'
16+
17+ options[:follow_links] = false
18+ options[:only_patterns] = [/guide\//, /api\//]
19+ options[:skip_patterns] = [/api\/([^\/]+)\.json/]
20+ options[:fix_urls_before_parse] = ->(url) do
21+ url.sub! %r{\Aguide/}, '/guide/'
22+ url.sub! %r{\Aapi/}, '/api/'
23+ url.sub! %r{\Agenerated/}, '/generated/'
24+ url
25+ end
26+
1227 options[:max_image_size] = 256_000
1328
1429 options[:attribution] = <<-HTML
@@ -16,69 +31,54 @@ module Docs · 1 unmodified line
1631 Code licensed under an Apache-2.0 License. Documentation licensed under CC BY 4.0.
1732 HTML
1833
19− module Common
20− private
34+ def get_latest_version(opts)
35+ json = fetch_json('https://rxjs.dev/generated/navigation.json', opts)
36+ json['__versionInfo']['raw']
37+ end
2138
22− def initial_urls
23− initial_urls = []
39+ private
2440
25− Request.run "#{self.class.base_url}generated/navigation.json" do |response|
26− data = JSON.parse(response.body)
27− dig = ->(entry) do
28− initial_urls << url_for("generated/docs/#{entry['url']}.json") if entry['url'] && entry['url'] != 'api'
29− entry['children'].each(&dig) if entry['children']
30− end
31− data['SideNav'].each(&dig)
32− end
41+ def initial_urls
42+ initial_urls = []
3343
34− Request.run "#{self.class.base_url}generated/docs/api/api-list.json" do |response|
35− data = JSON.parse(response.body)
36− dig = ->(entry) do
37− initial_urls << url_for("generated/docs/#{entry['path']}.json") if entry['path']
38− initial_urls << url_for("generated/docs/api/#{entry['name']}.json") if entry['name'] && !entry['path']
39− entry['items'].each(&dig) if entry['items']
40− end
41− data.each(&dig)
44+ Request.run "#{self.class.base_url}generated/navigation.json" do |response|
45+ data = JSON.parse(response.body)
46+ dig = ->(entry) do
47+ initial_urls << url_for("generated/docs/#{entry['url']}.json") if entry['url'] && entry['url'] != 'api'
48+ entry['children'].each(&dig) if entry['children']
4249 end
43−
44− initial_urls
50+ data['SideNav'].each(&dig)
4551 end
4652
47− def handle_response(response)
48− if response.mime_type.include?('json')
49− begin
50− response.options[:response_body] = JSON.parse(response.body)['contents']
51− rescue JSON::ParserError
52− response.options[:response_body] = ''
53− end
54− response.headers['Content-Type'] = 'text/html'
55− response.url.path = response.url.path.sub('/generated/docs/', '/').remove('.json')
56− response.effective_url.path = response.effective_url.path.sub('/generated/docs/', '/').remove('.json')
53+ Request.run "#{self.class.base_url}generated/docs/api/api-list.json" do |response|
54+ data = JSON.parse(response.body)
55+ dig = ->(entry) do
56+ initial_urls << url_for("generated/docs/#{entry['path']}.json") if entry['path']
57+ initial_urls << url_for("generated/docs/api/#{entry['name']}.json") if entry['name'] && !entry['path']
58+ entry['items'].each(&dig) if entry['items']
5759 end
58− super
60+ data.each(&dig)
5961 end
60− end
6162
62− version do
63− self.release = '6.3.3'
64− self.base_url = 'https://rxjs.dev/'
65− self.root_path = 'guide/overview'
66−
67− html_filters.push 'rxjs/clean_html', 'rxjs/entries'
68−
69− options[:follow_links] = false
70− options[:only_patterns] = [/\Aguide/, /\Aapi/]
71− options[:fix_urls_before_parse] = ->(url) do
72− url.sub! %r{\Aguide/}, '/guide/'
73− url.sub! %r{\Aapi/}, '/api/'
74− url.sub! %r{\Agenerated/}, '/generated/'
75− url
63+ initial_urls.select do |url|
64+ options[:only_patterns].any? { |pattern| url =~ pattern } &&
65+ options[:skip_patterns].none? { |pattern| url =~ pattern }
7666 end
77−
78− include Docs::Rxjs::Common
7967 end
8068
81− private
69+ def handle_response(response)
70+ if response.mime_type.include?('json')
71+ begin
72+ response.options[:response_body] = JSON.parse(response.body)['contents']
73+ rescue JSON::ParserError
74+ response.options[:response_body] = ''
75+ end
76+ response.headers['Content-Type'] = 'text/html'
77+ response.url.path = response.url.path.sub('/generated/docs/', '/').remove('.json')
78+ response.effective_url.path = response.effective_url.path.sub('/generated/docs/', '/').remove('.json')
79+ end
80+ super
81+ end
8282
8383 def parse(response)
8484 response.body.gsub! '<code-example', '<pre'
public/icons/docs/rxjs/16.png+0−0

Binary file not shown.

public/icons/docs/rxjs/SOURCE+1−1
@@ -1 +1 @@
1−http://reactivex.io/
1+https://github.com/ReactiveX/reactivex.github.io/blob/develop/favicon.ico