History

Cleanup, version, and improve Relay scraper

Phil Scherer committed 5 years ago

Commit 7c01b591 parent 37d5fd33 files · +50 −73Browse files

Unified · Split

3 files
  1. lib/docs/filters/relay/clean_html.rb +5 −26
  2. lib/docs/filters/relay/entries.rb +27 −31
  3. lib/docs/scrapers/relay.rb +18 −16
lib/docs/filters/relay/clean_html.rb+5−26
@@ -2,38 +2,17 @@ module Docs · 1 unmodified line
22 class Relay
33 class CleanHtmlFilter < Filter
44 def call
5+ @doc = at_css('.post')
56
6− if slug == 'index'
7− css('img').remove
7+ header = at_css('h1')
8+ header.parent.before(header).remove
89
9− css('.projectTitle').each do |node|
10− node.name = 'h1'
11− node.content = 'Relay'
12− end
13−
14− css('pre').remove
15−
16− end
17−
18− css('.docLastUpdate').remove
19−
20− css('.docs-prevnext').remove
21−
22− css('.edit-page-link').remove
10+ css('footer').remove
2311
2412 css('h2, h3').each do |node|
25− node.css('a').remove
26− node['id'] = node.content.gsub(/\s/, '-').downcase
13+ node['id'] = node.at_css('a.anchor')['id']
2714 end
2815
29− css('.onPageNav').remove
30−
31− css('#docsNav').remove
32−
33− css('.fixedHeaderContainer').remove
34−
35− css('footer').remove
36−
3716 # syntax highlight
3817 css('pre').each do |node|
3918 node['data-language'] = 'javascript'
lib/docs/filters/relay/entries.rb+27−31
@@ -1,51 +1,47 @@
11 module Docs
22 class Relay
33 class EntriesFilter < Docs::EntriesFilter
4−
5− def get_name
6− if slug == 'index'
7− return 'Relay'
4+ ONLY_SECTIONS = ['API Reference', 'Principles & Architecture']
5+ ONLY_SLUGS = []
6+
7+ def call
8+ if root_page?
9+ css('.navGroup > h3').each do |node|
10+ next if not ONLY_SECTIONS.include? node.content
11+ node.next_element.css('a').each do |anchor|
12+ ONLY_SLUGS << anchor['href'].split('/').last.strip
13+ end
14+ end
815 end
16+ super
17+ end
918
19+ def get_name
1020 at_css('h1').content
1121 end
1222
1323 def get_type
14− if slug == 'index'
15− return 'Relay'
16− end
17−
1824 at_css('h1').content
1925 end
2026
21− def additional_entries
22− entries = []
23−
24− if slug == 'index'
25− return entries
26− end
27−
28− ## avoid adding non-desired entries removing tags
29− # remove header which contains a <h2> tag
30− css('.fixedHeaderContainer').remove
27+ def include_default_entry?
28+ ONLY_SLUGS.include? slug
29+ end
3130
32− # remove table of content whose title is an <h2> tag
33− css('.toc').remove
34− ##
31+ def additional_entries
32+ return [] if not include_default_entry?
3533
36− css('h2, h3').each do |node|
37− next if node.content.include?('Argument')
38− entry_name = node.content
34+ css('article h2, article h3').each_with_object [] do |node, entries|
35+ next if node.content.include?('Argument') ||
36+ node.content.starts_with?('Example')
3937
40− if entry_name.include?('(')
41− entry_name = entry_name.match(/.*\(/)[0] + ')'
38+ name = node.content
39+ if name.include?('(')
40+ name = name.match(/.*\(/)[0] + ')'
4241 end
43−
44− entry_id = node.content.gsub(/\s/, '-').downcase
45− entries << [entry_name, entry_id]
42+ id = node.at_css('a.anchor')['id']
43+ entries << [name, id]
4644 end
47−
48− entries
4945 end
5046
5147 end
lib/docs/scrapers/relay.rb+18−16
@@ -1,9 +1,7 @@
11 module Docs
22 class Relay < UrlScraper
33 self.type = 'simple'
4− self.release = '10.1.0'
5− self.base_url = 'https://relay.dev'
6− self.root_path = 'index.html'
4+ self.root_path = 'introduction-to-relay'
75 self.links = {
86 home: 'https://relay.dev/',
97 code: 'https://github.com/facebook/relay'
@@ -11,19 +9,7 @@ module Docs · 1 unmodified line
119
1210 html_filters.push 'relay/entries', 'relay/clean_html'
1311
14− options[:only] = [
15− '/docs/en/graphql-in-relay',
16− '/docs/en//relay-environment',
17− '/docs/en/network-layer',
18− '/docs/en/query-renderer',
19− '/docs/en/fragment-container',
20− '/docs/en/refetch-container',
21− '/docs/en/pagination-container',
22− '/docs/en/mutations',
23− '/docs/en/subscriptions',
24− '/docs/en/relay-store',
25− '/docs/en/fetch-query'
26− ]
12+ options[:skip] = %w(videos)
2713
2814 options[:attribution] = <<-HTML
2915 &copy; 2020&ndash;present Facebook Inc.<br>
@@ -34,5 +20,21 @@ module Docs · 4 unmodified lines
3420 get_latest_github_release('facebook', 'relay', opts)
3521 end
3622
23+ version '10' do
24+ self.release = '10.1.0'
25+ self.base_url = "https://relay.dev/docs/en/"
26+ # For some reason, the most-recent version isn't available at a versioned URL
27+ end
28+
29+ version '9' do
30+ self.release = '9.1.0'
31+ self.base_url = "https://relay.dev/docs/en/v#{self.release}/"
32+ end
33+
34+ version '8' do
35+ self.release = '8.0.0'
36+ self.base_url = "https://relay.dev/docs/en/v#{self.release}/"
37+ end
38+
3739 end
3840 end