History

trio: finish scraper and filters

Jasper van Merle committed 7 years ago

Commit 3a6a1581 parent 5d224ce6 files · +23 −9Browse files

Unified · Split

6 files
  1. .gitignore +0 −1
  2. assets/javascripts/templates/pages/about_tmpl.coffee +5 −0
  3. lib/docs/filters/trio/clean_html.rb +8 −4
  4. lib/docs/filters/trio/entries.rb +3 −1
  5. lib/docs/scrapers/trio.rb +8 −4
  6. public/icons/docs/trio/16@2x.png +0 −0
.gitignore+0−1
@@ -7,4 +7,3 @@ public/fonts · 6 unmodified lines
7public/docs/**/*7public/docs/**/*
8docs/**/*8docs/**/*
9!docs/*.md9!docs/*.md
10vendor
assets/javascripts/templates/pages/about_tmpl.coffee+5−0
@@ -721,6 +721,11 @@ credits = [ · 720 unmodified lines
721 '2018 HashiCorp',721 '2018 HashiCorp',
722 'MPL',722 'MPL',
723 'https://raw.githubusercontent.com/hashicorp/terraform-website/master/LICENSE.md'723 'https://raw.githubusercontent.com/hashicorp/terraform-website/master/LICENSE.md'
724 ], [
725 'Trio',
726 '2017 Nathaniel J. Smith',
727 'MIT',
728 'https://raw.githubusercontent.com/python-trio/trio/master/LICENSE.MIT'
724 ], [729 ], [
725 'Twig',730 'Twig',
726 '2009-2018 The Twig Team',731 '2009-2018 The Twig Team',
lib/docs/filters/trio/clean_html.rb+8−4
@@ -3,6 +3,7 @@ module Docs · 2 unmodified lines
3 class CleanHtmlFilter < Filter3 class CleanHtmlFilter < Filter
4 def call4 def call
5 @doc = at_css('div[role="main"]')5 @doc = at_css('div[role="main"]')
6
6 css('.section, [itemprop=articleBody]').each do |node|7 css('.section, [itemprop=articleBody]').each do |node|
7 node.replace node.children8 node.replace node.children
8 end9 end
@@ -11,9 +12,11 @@ module Docs · 2 unmodified lines
11 12
12 css('dt').each do |node|13 css('dt').each do |node|
13 node.name = 'h3'14 node.name = 'h3'
15
14 if node.parent.classes.include? 'field-list'16 if node.parent.classes.include? 'field-list'
15 node.name = 'h4'17 node.name = 'h4'
16 node['style'] = 'margin: 0'18 node['style'] = 'margin: 0'
19
17 if node.text == 'Parameters' or node.text == 'Raises'20 if node.text == 'Parameters' or node.text == 'Raises'
18 node.next_element.css('strong').each do |n|21 node.next_element.css('strong').each do |n|
19 n.name = 'code'22 n.name = 'code'
@@ -21,30 +24,31 @@ module Docs · 1 unmodified line
21 end24 end
22 else25 else
23 code = doc.document.create_element 'code'26 code = doc.document.create_element 'code'
27
24 if em = node.at_css('.property')28 if em = node.at_css('.property')
25 code.inner_html = "<em>#{em.text.strip}</em> "29 code.inner_html = "<em>#{em.text.strip}</em> "
26 em.remove30 em.remove
27 end31 end
32
28 code.inner_html += node.inner_text.strip33 code.inner_html += node.inner_text.strip
29 node.inner_html = code34 node.inner_html = code
30 end35 end
31 end36 end
32 37
33 css('pre').each do |node|38 css('pre').each do |node|
39 node.content = node.content.strip
40
34 classes = node.parent.parent.classes41 classes = node.parent.parent.classes
35 if classes.include? 'highlight-python3'42 if classes.include? 'highlight-python3'
36 node['class'] = 'language-python'
37 node['data-language'] = 'python'43 node['data-language'] = 'python'
38 end44 end
45
39 node.parent.parent.replace(node)46 node.parent.parent.replace(node)
40 end47 end
41 48
42 css('.admonition').each do |node|49 css('.admonition').each do |node|
43 node.name = 'blockquote'50 node.name = 'blockquote'
44 node.at_css('.admonition-title').name = 'h4'51 node.at_css('.admonition-title').name = 'h4'
45 # new_node = node.document.create_element 'blockquote'
46 # new_node.inner_html = node.inner_html
47 # node.replace new_node
48 end52 end
49 53
50 doc54 doc
lib/docs/filters/trio/entries.rb+3−1
@@ -11,7 +11,6 @@ module Docs · 10 unmodified lines
11 11
12 def additional_entries12 def additional_entries
13 css('.descname').each_with_object [] do |node, entries|13 css('.descname').each_with_object [] do |node, entries|
14
15 name = node.text14 name = node.text
16 if node.previous.classes.include? 'descclassname'15 if node.previous.classes.include? 'descclassname'
17 name = node.previous.text + name16 name = node.previous.text + name
@@ -25,9 +24,11 @@ module Docs · 7 unmodified lines
25 or dl.classes.include?('data')24 or dl.classes.include?('data')
26 parent = dl.parent.previous_element25 parent = dl.parent.previous_element
27 cls = ''26 cls = ''
27
28 if n = parent.at_css('.descclassname')28 if n = parent.at_css('.descclassname')
29 cls += n.text29 cls += n.text
30 end30 end
31
31 if n = parent.at_css('.descname')32 if n = parent.at_css('.descname')
32 if n.text == "The nursery interface"33 if n.text == "The nursery interface"
33 cls += "Nursery."34 cls += "Nursery."
@@ -35,6 +36,7 @@ module Docs · 1 unmodified line
35 cls += n.text + '.'36 cls += n.text + '.'
36 end37 end
37 end38 end
39
38 name = cls + name40 name = cls + name
39 end41 end
40 42
lib/docs/scrapers/trio.rb+8−4
@@ -1,8 +1,8 @@
1module Docs1module Docs
2 class Trio < UrlScraper2 class Trio < UrlScraper
3 self.type = 'simple'3 self.type = 'simple'
4 self.release = '0.11'4 self.release = '0.12.1'
5 self.base_url = 'https://trio.readthedocs.io/en/latest/'5 self.base_url = 'https://trio.readthedocs.io/en/v0.12.1/'
6 self.root_path = 'index.html'6 self.root_path = 'index.html'
7 self.links = {7 self.links = {
8 home: 'https://trio.readthedocs.io/',8 home: 'https://trio.readthedocs.io/',
@@ -19,9 +19,13 @@ module Docs · 10 unmodified lines
19 ]19 ]
20 20
21 options[:attribution] = <<-HTML21 options[:attribution] = <<-HTML
22 &copy; 2017-2019 Nathaniel J. Smith<br>22 &copy; 2017 Nathaniel J. Smith<br>
23 Licensed under MIT and Apache2.23 Licensed under the MIT License.
24 HTML24 HTML
25 25
26 def get_latest_version(opts)
27 doc = fetch_doc('https://trio.readthedocs.io/en/stable/', opts)
28 doc.at_css('.rst-other-versions a[href^="/en/v"]').content[1..-1]
29 end
26 end30 end
27end31end
public/icons/docs/trio/16@2x.pngrenamed from public/icons/docs/trio/16@2.png+0−0

File renamed without changes.