History

Finish Codeception scraper

Thibaut Courouble committed 9 years ago

Commit 802a8d31 parent 72ab34313 files · +64 −66Browse files

Showing the changes to lib/docs (3 of 13 files). Show all

Unified · Split

3 files
  1. lib/docs/filters/codeception/clean_html.rb +31 −8
  2. lib/docs/filters/codeception/entries.rb +23 −21
  3. lib/docs/scrapers/codeception.rb +4 −6
lib/docs/filters/codeception/clean_html.rb+31−8
@@ -2,17 +2,40 @@ module Docs · 1 unmodified line
22 class Codeception
33 class CleanHtmlFilter < Filter
44 def call
5− root_page? ? root : other
6− doc
7− end
5+ @doc = doc.at_css('div.page, div.content')
86
9− def root
10− doc.inner_html = ' '
11− end
7+ while doc.element_children.length == 1
8+ doc.first_element_child.before(doc.first_element_child.children).remove
9+ end
10+
11+ if root_page?
12+ at_css('h1').content = 'Codeception Documentation'
13+ end
14+
15+ unless at_css('h1')
16+ at_css('h2').name = 'h1'
17+ end
18+
19+ unless at_css('h2')
20+ css('h3').each { |node| node.name = 'h2' }
21+ css('h4').each { |node| node.name = 'h3' }
22+ end
1223
13− def other
14− @doc = doc.at_css('div.page, div.content')
1524 css('.btn-group').remove
25+
26+ css('.alert:last-child').each do |node|
27+ node.remove if node.content.include?('taken from the source code')
28+ end
29+
30+ css('.highlight').each do |node|
31+ node.before(node.children).remove
32+ end
33+
34+ css('pre > code').each do |node|
35+ node.parent['data-language'] = node['data-lang']
36+ node.parent.content = node.parent.content
37+ end
38+
1639 doc
1740 end
1841 end
lib/docs/filters/codeception/entries.rb+23−21
@@ -2,46 +2,48 @@ module Docs · 1 unmodified line
22 class Codeception
33 class EntriesFilter < Docs::EntriesFilter
44 def get_name
5− (subpath.scan(/\d\d/).first || '') + ' ' + (at_css('h1') || at_css('h2')).content
6− end
5+ name = (at_css('h1') || at_css('h2')).content
76
8− def get_type
9− return 'Module::' + (at_css('h1') || at_css('h2')).content if subpath.start_with?('modules')
10− return 'Guides' if subpath =~ /\d\d/
11− (at_css('h1') || at_css('h2')).content
7+ if number = subpath[/\A\d+/]
8+ name.prepend "#{number.to_i}. "
9+ end
10+
11+ name
1212 end
1313
14− def include_default_entry?
15− return true if %w(Guides).include? type
16− return true if type =~ /(Module)|(Util)/
17− false
14+ def get_type
15+ if subpath =~ /\d\d/
16+ 'Guides'
17+ elsif subpath.start_with?('modules')
18+ "Module: #{name}"
19+ elsif name.include?('Util')
20+ "Util Class: #{name.split('\\').last}"
21+ else
22+ "Reference: #{name}"
23+ end
1824 end
1925
2026 def additional_entries
21−
2227 if type =~ /Module/
23− prefix = type.sub(/Module::/, '')+ '::'
28+ prefix = "#{name}::"
2429 pattern = '#actions ~ h4'
2530 elsif type =~ /Functions/
2631 prefix = ''
27− pattern = 'h4'
28− elsif type =~ /Util/
29− prefix = type.sub(/Codeception\/Util/, '')+ '::'
32+ pattern = '#page h4'
33+ elsif name =~ /Util/
34+ prefix = "#{name.remove('Codeception\\Util\\')}::"
3035 pattern = 'h3'
3136 elsif type =~ /(Commands)|(Configuration)/
32− prefix = ''
33− pattern = 'h2'
34− else
3537 prefix = ''
36− pattern = 'none'
38+ pattern = 'h2'
3739 end
3840
41+ return [] unless pattern
42+
3943 css(pattern).map do |node|
4044 [prefix + node.content, node['id']]
4145 end.compact
42−
4346 end
44−
4547 end
4648 end
4749 end
lib/docs/scrapers/codeception.rb+4−6
@@ -2,22 +2,20 @@ module Docs · 1 unmodified line
22 class Codeception < UrlScraper
33 self.name = 'Codeception'
44 self.type = 'codeception'
5− self.root_path = 'index.html'
6− self.release = '2.2'
5+ self.release = '2.2.6'
76 self.base_url = 'http://codeception.com/docs/'
7+ self.root_path = 'index.html'
88 self.links = {
99 home: 'http://codeception.com/',
1010 code: 'https://github.com/codeception/codeception'
1111 }
1212
13− html_filters.push 'codeception/clean_html', 'codeception/entries', 'title'
13+ html_filters.push 'codeception/entries', 'codeception/clean_html'
1414
15− options[:root_title] = 'Codeception'
16− options[:title] = false
1715 options[:skip_patterns] = [/install/]
1816
1917 options[:attribution] = <<-HTML
20− &copy; 2011–2016 Michael Bodnarchuk and Contributors<br>
18+ &copy; 2011&ndash;2016 Michael Bodnarchuk and contributors<br>
2119 Licensed under the MIT License.
2220 HTML
2321 end

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t