History

Finish Codeception scraper

Thibaut Courouble committed 9 years ago

Commit 802a8d31 parent 72ab34313 files · +64 −66Browse files

Showing the changes to lib/docs (3 of 13 files). Show all

Unified · Split

3 files
  1. lib/docs/filters/codeception/clean_html.rb +31 −8
  2. lib/docs/filters/codeception/entries.rb +23 −21
  3. lib/docs/scrapers/codeception.rb +4 −6
lib/docs/filters/codeception/clean_html.rb+31−8
@@ -2,17 +2,40 @@ module Docs · 1 unmodified line
2 class Codeception2 class Codeception
3 class CleanHtmlFilter < Filter3 class CleanHtmlFilter < Filter
4 def call4 def call
5 root_page? ? root : other5 @doc = doc.at_css('div.page, div.content')
6 doc
7 end
8 6
9 def root7 while doc.element_children.length == 1
10 doc.inner_html = ' '8 doc.first_element_child.before(doc.first_element_child.children).remove
11 end9 end
10
11 if root_page?
12 at_css('h1').content = 'Codeception Documentation'
13 end
14
15 unless at_css('h1')
16 at_css('h2').name = 'h1'
17 end
18
19 unless at_css('h2')
20 css('h3').each { |node| node.name = 'h2' }
21 css('h4').each { |node| node.name = 'h3' }
22 end
12 23
13 def other
14 @doc = doc.at_css('div.page, div.content')
15 css('.btn-group').remove24 css('.btn-group').remove
25
26 css('.alert:last-child').each do |node|
27 node.remove if node.content.include?('taken from the source code')
28 end
29
30 css('.highlight').each do |node|
31 node.before(node.children).remove
32 end
33
34 css('pre > code').each do |node|
35 node.parent['data-language'] = node['data-lang']
36 node.parent.content = node.parent.content
37 end
38
16 doc39 doc
17 end40 end
18 end41 end
lib/docs/filters/codeception/entries.rb+23−21
@@ -2,46 +2,48 @@ module Docs · 1 unmodified line
2 class Codeception2 class Codeception
3 class EntriesFilter < Docs::EntriesFilter3 class EntriesFilter < Docs::EntriesFilter
4 def get_name4 def get_name
5 (subpath.scan(/\d\d/).first || '') + ' ' + (at_css('h1') || at_css('h2')).content5 name = (at_css('h1') || at_css('h2')).content
6 end
7 6
8 def get_type7 if number = subpath[/\A\d+/]
9 return 'Module::' + (at_css('h1') || at_css('h2')).content if subpath.start_with?('modules')8 name.prepend "#{number.to_i}. "
10 return 'Guides' if subpath =~ /\d\d/9 end
11 (at_css('h1') || at_css('h2')).content10
11 name
12 end12 end
13 13
14 def include_default_entry?14 def get_type
15 return true if %w(Guides).include? type15 if subpath =~ /\d\d/
16 return true if type =~ /(Module)|(Util)/16 'Guides'
17 false17 elsif subpath.start_with?('modules')
18 "Module: #{name}"
19 elsif name.include?('Util')
20 "Util Class: #{name.split('\\').last}"
21 else
22 "Reference: #{name}"
23 end
18 end24 end
19 25
20 def additional_entries26 def additional_entries
21
22 if type =~ /Module/27 if type =~ /Module/
23 prefix = type.sub(/Module::/, '')+ '::'28 prefix = "#{name}::"
24 pattern = '#actions ~ h4'29 pattern = '#actions ~ h4'
25 elsif type =~ /Functions/30 elsif type =~ /Functions/
26 prefix = ''31 prefix = ''
27 pattern = 'h4'32 pattern = '#page h4'
28 elsif type =~ /Util/33 elsif name =~ /Util/
29 prefix = type.sub(/Codeception\/Util/, '')+ '::'34 prefix = "#{name.remove('Codeception\\Util\\')}::"
30 pattern = 'h3'35 pattern = 'h3'
31 elsif type =~ /(Commands)|(Configuration)/36 elsif type =~ /(Commands)|(Configuration)/
32 prefix = ''
33 pattern = 'h2'
34 else
35 prefix = ''37 prefix = ''
36 pattern = 'none'38 pattern = 'h2'
37 end39 end
38 40
41 return [] unless pattern
42
39 css(pattern).map do |node|43 css(pattern).map do |node|
40 [prefix + node.content, node['id']]44 [prefix + node.content, node['id']]
41 end.compact45 end.compact
42
43 end46 end
44
45 end47 end
46 end48 end
47end49end
lib/docs/scrapers/codeception.rb+4−6
@@ -2,22 +2,20 @@ module Docs · 1 unmodified line
2 class Codeception < UrlScraper2 class Codeception < UrlScraper
3 self.name = 'Codeception'3 self.name = 'Codeception'
4 self.type = 'codeception'4 self.type = 'codeception'
5 self.root_path = 'index.html'5 self.release = '2.2.6'
6 self.release = '2.2'
7 self.base_url = 'http://codeception.com/docs/'6 self.base_url = 'http://codeception.com/docs/'
7 self.root_path = 'index.html'
8 self.links = {8 self.links = {
9 home: 'http://codeception.com/',9 home: 'http://codeception.com/',
10 code: 'https://github.com/codeception/codeception'10 code: 'https://github.com/codeception/codeception'
11 }11 }
12 12
13 html_filters.push 'codeception/clean_html', 'codeception/entries', 'title'13 html_filters.push 'codeception/entries', 'codeception/clean_html'
14 14
15 options[:root_title] = 'Codeception'
16 options[:title] = false
17 options[:skip_patterns] = [/install/]15 options[:skip_patterns] = [/install/]
18 16
19 options[:attribution] = <<-HTML17 options[:attribution] = <<-HTML
20 &copy; 2011–2016 Michael Bodnarchuk and Contributors<br>18 &copy; 2011&ndash;2016 Michael Bodnarchuk and contributors<br>
21 Licensed under the MIT License.19 Licensed under the MIT License.
22 HTML20 HTML
23 end21 end

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t