History

Decouple Relay scraper from React scraper

Thibaut Courouble committed 9 years ago

Commit 6d580a91 parent 94044b73 files · +111 −4Browse files

Unified · Split

3 files
  1. lib/docs/filters/relay/clean_html.rb +63 −0
  2. lib/docs/filters/relay/entries.rb +43 −0
  3. lib/docs/scrapers/relay.rb +5 −4
lib/docs/filters/relay/clean_html.rb+63−0
@@ -0,0 +1,63 @@
1+module Docs
2+ class Relay
3+ class CleanHtmlFilter < Filter
4+ def call
5+ @doc = at_css('.inner-content, article.withtoc')
6+
7+ if root_page?
8+ at_css('h1').content = 'Relay Documentation'
9+ end
10+
11+ css('.docs-prevnext', '.hash-link', '.edit-page-link', '.edit-github', 'a.hash', '.edit-page-block', 'a.show', 'a.hide', 'hr').remove
12+
13+ css('table h1', 'table h2', 'table h3').each do |node|
14+ table = node
15+ table = table.parent until table.name == 'table'
16+ table.replace(node)
17+ end
18+
19+ css('a.anchor', 'a.hashref').each do |node|
20+ node.parent['id'] ||= node['name'] || node['id']
21+ end
22+
23+ css('.highlight').each do |node|
24+ node.name = 'pre'
25+ node.css('.gutter').remove
26+ node['data-language'] = node.at_css('[data-lang]').try(:[], 'data-lang') || 'js'
27+ node.content = node.content.strip
28+ end
29+
30+ css('table.highlighttable').each do |node|
31+ node.replace(node.at_css('pre.highlight'))
32+ end
33+
34+ css('.prism').each do |node|
35+ node.name = 'pre'
36+ node['data-language'] = node['class'][/(?<=language\-)(\w+)/]
37+ node.content = node.content
38+ end
39+
40+ css('blockquote > p:first-child').each do |node|
41+ node.remove if node.content.strip == 'Note:'
42+ end
43+
44+ css('h3#props', 'h3#methods').each { |node| node.name = 'h2' }
45+ css('h4.propTitle').each { |node| node.name = 'h3' }
46+
47+ css('> div > div', '> div', 'div > span', '.props', '.prop').each do |node|
48+ node.before(node.children).remove
49+ end
50+
51+ css('a pre', 'h3 .propType').each do |node|
52+ node.name = 'code'
53+ end
54+
55+ css('a[target]').each do |node|
56+ node.remove_attribute('target')
57+ end
58+
59+ doc
60+ end
61+ end
62+ end
63+end
lib/docs/filters/relay/entries.rb+43−0
@@ -0,0 +1,43 @@
1+module Docs
2+ class Relay
3+ class EntriesFilter < Docs::EntriesFilter
4+ def get_name
5+ at_css('h1').children.select(&:text?).map(&:content).join.strip
6+ end
7+
8+ def get_type
9+ link = at_css('.nav-docs-section .active, .toc .active')
10+ section = link.ancestors('.nav-docs-section, section').first
11+ type = section.at_css('h3').content.strip
12+ type
13+ end
14+
15+ def additional_entries
16+ entries = []
17+
18+ css('.inner-content h3 code, .inner-content h4 code').each do |node|
19+ name = node.content
20+ name.remove! %r{[#\(\)]}
21+ name.remove! %r{\w+\:}
22+ name.strip!
23+ id = name.parameterize
24+ node.parent['id'] = id
25+ entries << [name, id, 'Reference']
26+ end
27+
28+ css('.apiIndex a pre').each do |node|
29+ next unless node.parent['href'].start_with?('#')
30+ id = node.parent['href'].remove('#')
31+ name = node.content.strip
32+ sep = name.start_with?('static') ? '.' : '#'
33+ name.remove! %r{(abstract|static) }
34+ name.sub! %r{\(.*\)}, '()'
35+ name.prepend(self.name + sep)
36+ entries << [name, id]
37+ end
38+
39+ entries
40+ end
41+ end
42+ end
43+end
lib/docs/scrapers/relay.rb+5−4
@@ -1,6 +1,6 @@
11 module Docs
2− class Relay < React
3− self.type = 'react'
2+ class Relay < UrlScraper
3+ self.type = 'simple'
44 self.release = '1.4.1'
55 self.base_url = 'https://facebook.github.io/relay/docs/'
66 self.root_path = 'getting-started.html'
@@ -9,8 +9,9 @@ module Docs · 2 unmodified lines
99 code: 'https://github.com/facebook/relay'
1010 }
1111
12− options[:root_title] = 'Relay Documentation'
13− options[:only_patterns] = nil
12+ html_filters.push 'relay/entries', 'relay/clean_html'
13+
14+ options[:container] = '.documentationContent'
1415 options[:skip] = %w(videos.html graphql-further-reading.html)
1516
1617 options[:attribution] = <<-HTML