History

trio: finish scraper and filters

Jasper van Merle committed 7 years ago

Commit 3a6a1581 parent 5d224ce6 files · +23 −9Browse files

Showing the changes to lib/docs (3 of 6 files). Show all

Unified · Split

3 files
  1. lib/docs/filters/trio/clean_html.rb +8 −4
  2. lib/docs/filters/trio/entries.rb +3 −1
  3. lib/docs/scrapers/trio.rb +8 −4
lib/docs/filters/trio/clean_html.rb+8−4
@@ -3,6 +3,7 @@ module Docs · 2 unmodified lines
33 class CleanHtmlFilter < Filter
44 def call
55 @doc = at_css('div[role="main"]')
6+
67 css('.section, [itemprop=articleBody]').each do |node|
78 node.replace node.children
89 end
@@ -11,9 +12,11 @@ module Docs · 2 unmodified lines
1112
1213 css('dt').each do |node|
1314 node.name = 'h3'
15+
1416 if node.parent.classes.include? 'field-list'
1517 node.name = 'h4'
1618 node['style'] = 'margin: 0'
19+
1720 if node.text == 'Parameters' or node.text == 'Raises'
1821 node.next_element.css('strong').each do |n|
1922 n.name = 'code'
@@ -21,30 +24,31 @@ module Docs · 1 unmodified line
2124 end
2225 else
2326 code = doc.document.create_element 'code'
27+
2428 if em = node.at_css('.property')
2529 code.inner_html = "<em>#{em.text.strip}</em> "
2630 em.remove
2731 end
32+
2833 code.inner_html += node.inner_text.strip
2934 node.inner_html = code
3035 end
3136 end
3237
3338 css('pre').each do |node|
39+ node.content = node.content.strip
40+
3441 classes = node.parent.parent.classes
3542 if classes.include? 'highlight-python3'
36− node['class'] = 'language-python'
3743 node['data-language'] = 'python'
3844 end
45+
3946 node.parent.parent.replace(node)
4047 end
4148
4249 css('.admonition').each do |node|
4350 node.name = 'blockquote'
4451 node.at_css('.admonition-title').name = 'h4'
45− # new_node = node.document.create_element 'blockquote'
46− # new_node.inner_html = node.inner_html
47− # node.replace new_node
4852 end
4953
5054 doc
lib/docs/filters/trio/entries.rb+3−1
@@ -11,7 +11,6 @@ module Docs · 10 unmodified lines
1111
1212 def additional_entries
1313 css('.descname').each_with_object [] do |node, entries|
14−
1514 name = node.text
1615 if node.previous.classes.include? 'descclassname'
1716 name = node.previous.text + name
@@ -25,9 +24,11 @@ module Docs · 7 unmodified lines
2524 or dl.classes.include?('data')
2625 parent = dl.parent.previous_element
2726 cls = ''
27+
2828 if n = parent.at_css('.descclassname')
2929 cls += n.text
3030 end
31+
3132 if n = parent.at_css('.descname')
3233 if n.text == "The nursery interface"
3334 cls += "Nursery."
@@ -35,6 +36,7 @@ module Docs · 1 unmodified line
3536 cls += n.text + '.'
3637 end
3738 end
39+
3840 name = cls + name
3941 end
4042
lib/docs/scrapers/trio.rb+8−4
@@ -1,8 +1,8 @@
11 module Docs
22 class Trio < UrlScraper
33 self.type = 'simple'
4− self.release = '0.11'
5− self.base_url = 'https://trio.readthedocs.io/en/latest/'
4+ self.release = '0.12.1'
5+ self.base_url = 'https://trio.readthedocs.io/en/v0.12.1/'
66 self.root_path = 'index.html'
77 self.links = {
88 home: 'https://trio.readthedocs.io/',
@@ -19,9 +19,13 @@ module Docs · 10 unmodified lines
1919 ]
2020
2121 options[:attribution] = <<-HTML
22− &copy; 2017-2019 Nathaniel J. Smith<br>
23− Licensed under MIT and Apache2.
22+ &copy; 2017 Nathaniel J. Smith<br>
23+ Licensed under the MIT License.
2424 HTML
2525
26+ def get_latest_version(opts)
27+ doc = fetch_doc('https://trio.readthedocs.io/en/stable/', opts)
28+ doc.at_css('.rst-other-versions a[href^="/en/v"]').content[1..-1]
29+ end
2630 end
2731 end

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t