History

Decouple Relay scraper from React scraper

Thibaut Courouble committed 9 years ago

Commit 6d580a91 parent 94044b73 files · +111 −4Browse files

Unified · Split

3 files
  1. lib/docs/filters/relay/clean_html.rb +63 −0
  2. lib/docs/filters/relay/entries.rb +43 −0
  3. lib/docs/scrapers/relay.rb +5 −4
lib/docs/filters/relay/clean_html.rb+63−0
@@ -0,0 +1,63 @@
1module Docs
2 class Relay
3 class CleanHtmlFilter < Filter
4 def call
5 @doc = at_css('.inner-content, article.withtoc')
6
7 if root_page?
8 at_css('h1').content = 'Relay Documentation'
9 end
10
11 css('.docs-prevnext', '.hash-link', '.edit-page-link', '.edit-github', 'a.hash', '.edit-page-block', 'a.show', 'a.hide', 'hr').remove
12
13 css('table h1', 'table h2', 'table h3').each do |node|
14 table = node
15 table = table.parent until table.name == 'table'
16 table.replace(node)
17 end
18
19 css('a.anchor', 'a.hashref').each do |node|
20 node.parent['id'] ||= node['name'] || node['id']
21 end
22
23 css('.highlight').each do |node|
24 node.name = 'pre'
25 node.css('.gutter').remove
26 node['data-language'] = node.at_css('[data-lang]').try(:[], 'data-lang') || 'js'
27 node.content = node.content.strip
28 end
29
30 css('table.highlighttable').each do |node|
31 node.replace(node.at_css('pre.highlight'))
32 end
33
34 css('.prism').each do |node|
35 node.name = 'pre'
36 node['data-language'] = node['class'][/(?<=language\-)(\w+)/]
37 node.content = node.content
38 end
39
40 css('blockquote > p:first-child').each do |node|
41 node.remove if node.content.strip == 'Note:'
42 end
43
44 css('h3#props', 'h3#methods').each { |node| node.name = 'h2' }
45 css('h4.propTitle').each { |node| node.name = 'h3' }
46
47 css('> div > div', '> div', 'div > span', '.props', '.prop').each do |node|
48 node.before(node.children).remove
49 end
50
51 css('a pre', 'h3 .propType').each do |node|
52 node.name = 'code'
53 end
54
55 css('a[target]').each do |node|
56 node.remove_attribute('target')
57 end
58
59 doc
60 end
61 end
62 end
63end
lib/docs/filters/relay/entries.rb+43−0
@@ -0,0 +1,43 @@
1module Docs
2 class Relay
3 class EntriesFilter < Docs::EntriesFilter
4 def get_name
5 at_css('h1').children.select(&:text?).map(&:content).join.strip
6 end
7
8 def get_type
9 link = at_css('.nav-docs-section .active, .toc .active')
10 section = link.ancestors('.nav-docs-section, section').first
11 type = section.at_css('h3').content.strip
12 type
13 end
14
15 def additional_entries
16 entries = []
17
18 css('.inner-content h3 code, .inner-content h4 code').each do |node|
19 name = node.content
20 name.remove! %r{[#\(\)]}
21 name.remove! %r{\w+\:}
22 name.strip!
23 id = name.parameterize
24 node.parent['id'] = id
25 entries << [name, id, 'Reference']
26 end
27
28 css('.apiIndex a pre').each do |node|
29 next unless node.parent['href'].start_with?('#')
30 id = node.parent['href'].remove('#')
31 name = node.content.strip
32 sep = name.start_with?('static') ? '.' : '#'
33 name.remove! %r{(abstract|static) }
34 name.sub! %r{\(.*\)}, '()'
35 name.prepend(self.name + sep)
36 entries << [name, id]
37 end
38
39 entries
40 end
41 end
42 end
43end
lib/docs/scrapers/relay.rb+5−4
@@ -1,6 +1,6 @@
1module Docs1module Docs
2 class Relay < React2 class Relay < UrlScraper
3 self.type = 'react'3 self.type = 'simple'
4 self.release = '1.4.1'4 self.release = '1.4.1'
5 self.base_url = 'https://facebook.github.io/relay/docs/'5 self.base_url = 'https://facebook.github.io/relay/docs/'
6 self.root_path = 'getting-started.html'6 self.root_path = 'getting-started.html'
@@ -9,8 +9,9 @@ module Docs · 2 unmodified lines
9 code: 'https://github.com/facebook/relay'9 code: 'https://github.com/facebook/relay'
10 }10 }
11 11
12 options[:root_title] = 'Relay Documentation'12 html_filters.push 'relay/entries', 'relay/clean_html'
13 options[:only_patterns] = nil13
14 options[:container] = '.documentationContent'
14 options[:skip] = %w(videos.html graphql-further-reading.html)15 options[:skip] = %w(videos.html graphql-further-reading.html)
15 16
16 options[:attribution] = <<-HTML17 options[:attribution] = <<-HTML