History

Cleanup, version, and improve Relay scraper

Phil Scherer committed 5 years ago

Commit 7c01b591 parent 37d5fd33 files · +50 −73Browse files

Unified · Split

3 files
  1. lib/docs/filters/relay/clean_html.rb +5 −26
  2. lib/docs/filters/relay/entries.rb +27 −31
  3. lib/docs/scrapers/relay.rb +18 −16
lib/docs/filters/relay/clean_html.rb+5−26
@@ -2,38 +2,17 @@ module Docs · 1 unmodified line
2 class Relay2 class Relay
3 class CleanHtmlFilter < Filter3 class CleanHtmlFilter < Filter
4 def call4 def call
5 @doc = at_css('.post')
5 6
6 if slug == 'index'7 header = at_css('h1')
7 css('img').remove8 header.parent.before(header).remove
8 9
9 css('.projectTitle').each do |node|10 css('footer').remove
10 node.name = 'h1'
11 node.content = 'Relay'
12 end
13
14 css('pre').remove
15
16 end
17
18 css('.docLastUpdate').remove
19
20 css('.docs-prevnext').remove
21
22 css('.edit-page-link').remove
23 11
24 css('h2, h3').each do |node|12 css('h2, h3').each do |node|
25 node.css('a').remove13 node['id'] = node.at_css('a.anchor')['id']
26 node['id'] = node.content.gsub(/\s/, '-').downcase
27 end14 end
28 15
29 css('.onPageNav').remove
30
31 css('#docsNav').remove
32
33 css('.fixedHeaderContainer').remove
34
35 css('footer').remove
36
37 # syntax highlight16 # syntax highlight
38 css('pre').each do |node|17 css('pre').each do |node|
39 node['data-language'] = 'javascript'18 node['data-language'] = 'javascript'
lib/docs/filters/relay/entries.rb+27−31
@@ -1,51 +1,47 @@
1module Docs1module Docs
2 class Relay2 class Relay
3 class EntriesFilter < Docs::EntriesFilter3 class EntriesFilter < Docs::EntriesFilter
4 4 ONLY_SECTIONS = ['API Reference', 'Principles & Architecture']
5 def get_name5 ONLY_SLUGS = []
6 if slug == 'index'6
7 return 'Relay'7 def call
8 if root_page?
9 css('.navGroup > h3').each do |node|
10 next if not ONLY_SECTIONS.include? node.content
11 node.next_element.css('a').each do |anchor|
12 ONLY_SLUGS << anchor['href'].split('/').last.strip
13 end
14 end
8 end15 end
16 super
17 end
9 18
19 def get_name
10 at_css('h1').content20 at_css('h1').content
11 end21 end
12 22
13 def get_type23 def get_type
14 if slug == 'index'
15 return 'Relay'
16 end
17
18 at_css('h1').content24 at_css('h1').content
19 end25 end
20 26
21 def additional_entries27 def include_default_entry?
22 entries = []28 ONLY_SLUGS.include? slug
23 29 end
24 if slug == 'index'
25 return entries
26 end
27
28 ## avoid adding non-desired entries removing tags
29 # remove header which contains a <h2> tag
30 css('.fixedHeaderContainer').remove
31 30
32 # remove table of content whose title is an <h2> tag31 def additional_entries
33 css('.toc').remove32 return [] if not include_default_entry?
34 ##
35 33
36 css('h2, h3').each do |node|34 css('article h2, article h3').each_with_object [] do |node, entries|
37 next if node.content.include?('Argument')35 next if node.content.include?('Argument') ||
38 entry_name = node.content36 node.content.starts_with?('Example')
39 37
40 if entry_name.include?('(')38 name = node.content
41 entry_name = entry_name.match(/.*\(/)[0] + ')'39 if name.include?('(')
40 name = name.match(/.*\(/)[0] + ')'
42 end41 end
43 42 id = node.at_css('a.anchor')['id']
44 entry_id = node.content.gsub(/\s/, '-').downcase43 entries << [name, id]
45 entries << [entry_name, entry_id]
46 end44 end
47
48 entries
49 end45 end
50 46
51 end47 end
lib/docs/scrapers/relay.rb+18−16
@@ -1,9 +1,7 @@
1module Docs1module Docs
2 class Relay < UrlScraper2 class Relay < UrlScraper
3 self.type = 'simple'3 self.type = 'simple'
4 self.release = '10.1.0'4 self.root_path = 'introduction-to-relay'
5 self.base_url = 'https://relay.dev'
6 self.root_path = 'index.html'
7 self.links = {5 self.links = {
8 home: 'https://relay.dev/',6 home: 'https://relay.dev/',
9 code: 'https://github.com/facebook/relay'7 code: 'https://github.com/facebook/relay'
@@ -11,19 +9,7 @@ module Docs · 1 unmodified line
11 9
12 html_filters.push 'relay/entries', 'relay/clean_html'10 html_filters.push 'relay/entries', 'relay/clean_html'
13 11
14 options[:only] = [12 options[:skip] = %w(videos)
15 '/docs/en/graphql-in-relay',
16 '/docs/en//relay-environment',
17 '/docs/en/network-layer',
18 '/docs/en/query-renderer',
19 '/docs/en/fragment-container',
20 '/docs/en/refetch-container',
21 '/docs/en/pagination-container',
22 '/docs/en/mutations',
23 '/docs/en/subscriptions',
24 '/docs/en/relay-store',
25 '/docs/en/fetch-query'
26 ]
27 13
28 options[:attribution] = <<-HTML14 options[:attribution] = <<-HTML
29 &copy; 2020&ndash;present Facebook Inc.<br>15 &copy; 2020&ndash;present Facebook Inc.<br>
@@ -34,5 +20,21 @@ module Docs · 4 unmodified lines
34 get_latest_github_release('facebook', 'relay', opts)20 get_latest_github_release('facebook', 'relay', opts)
35 end21 end
36 22
23 version '10' do
24 self.release = '10.1.0'
25 self.base_url = "https://relay.dev/docs/en/"
26 # For some reason, the most-recent version isn't available at a versioned URL
27 end
28
29 version '9' do
30 self.release = '9.1.0'
31 self.base_url = "https://relay.dev/docs/en/v#{self.release}/"
32 end
33
34 version '8' do
35 self.release = '8.0.0'
36 self.base_url = "https://relay.dev/docs/en/v#{self.release}/"
37 end
38
37 end39 end
38end40end