History

Finish Codeception scraper

Thibaut Courouble committed 9 years ago

Commit 802a8d31 parent 72ab34313 files · +64 −66Browse files

Unified · Split

13 files
  1. assets/images/docs.png +0 −0
  2. assets/images/docs@2x.png +0 −0
  3. assets/javascripts/news.json +1 −1
  4. assets/javascripts/templates/pages/about_tmpl.coffee +5 −0
  5. assets/javascripts/views/pages/codeception.coffee +0 −10
  6. assets/javascripts/views/pages/simple.coffee +1 −0
  7. assets/stylesheets/global/_icons.scss +1 −0
  8. assets/stylesheets/pages/_codeception.scss +2 −24
  9. lib/docs/filters/codeception/clean_html.rb +31 −8
  10. lib/docs/filters/codeception/entries.rb +23 −21
  11. lib/docs/scrapers/codeception.rb +4 −6
  12. public/icons/docs/codeception/16.png +0 −0
  13. public/icons/docs/codeception/16@2x.png +0 −0
assets/images/docs.png+0−0

Binary file not shown.

assets/images/docs@2x.png+0−0

Binary file not shown.

assets/javascripts/news.json+1−1
@@ -1,7 +1,7 @@
1[1[
2 [2 [
3 "2016-12-04",3 "2016-12-04",
4 "New documentation: <a href=\"/sqlite/\">SQLite</a>"4 "New documentations: <a href=\"/sqlite/\">SQLite</a> and <a href=\"/codeception/\">Codeception</a>"
5 ], [5 ], [
6 "2016-11-20",6 "2016-11-20",
7 "New documentations: <a href=\"/yarn/\">Yarn</a>, <a href=\"/immutable/\">Immutable.js</a> and <a href=\"/async/\">Async</a>"7 "New documentations: <a href=\"/yarn/\">Yarn</a>, <a href=\"/immutable/\">Immutable.js</a> and <a href=\"/async/\">Async</a>"
assets/javascripts/templates/pages/about_tmpl.coffee+5−0
@@ -154,6 +154,11 @@ credits = [ · 153 unmodified lines
154 '2000-2016 Kitware, Inc.<br>&copy; 2000-2011 Insight Software Consortium',154 '2000-2016 Kitware, Inc.<br>&copy; 2000-2011 Insight Software Consortium',
155 'BSD',155 'BSD',
156 'https://cmake.org/licensing/'156 'https://cmake.org/licensing/'
157 ], [
158 'Codeception',
159 '2011-2016 Michael Bodnarchuk and contributors',
160 'MIT',
161 'https://raw.githubusercontent.com/Codeception/Codeception/master/LICENSE'
157 ], [162 ], [
158 'CodeIgniter',163 'CodeIgniter',
159 '2014-2016 British Columbia Institute of Technology',164 '2014-2016 British Columbia Institute of Technology',
assets/javascripts/views/pages/codeception.coffee+0−10
@@ -1,10 +0,0 @@
1#= require views/pages/base
2
3class app.views.CodeceptionPage extends app.views.BasePage
4 LANGUAGE_RGX = /language-(\w+)/
5
6 prepare: ->
7 for el in @findAll('pre > code')
8 if el.className.match(LANGUAGE_RGX)
9 @highlightCode(el, el.className.match(LANGUAGE_RGX)[1])
10 return
assets/javascripts/views/pages/simple.coffee+1−0
@@ -14,6 +14,7 @@ app.views.BowerPage = · 13 unmodified lines
14app.views.CPage =14app.views.CPage =
15app.views.CakephpPage =15app.views.CakephpPage =
16app.views.ChaiPage =16app.views.ChaiPage =
17app.views.CodeceptionPage =
17app.views.CoffeescriptPage =18app.views.CoffeescriptPage =
18app.views.CordovaPage =19app.views.CordovaPage =
19app.views.CrystalPage =20app.views.CrystalPage =
assets/stylesheets/global/_icons.scss+1−0
@@ -56,6 +56,7 @@ · 55 unmodified lines
56%icon-close-white { background-position: -2rem -5rem; }56%icon-close-white { background-position: -2rem -5rem; }
57%icon-back { background-position: -3rem -5rem; @extend %darkIconFix !optional; }57%icon-back { background-position: -3rem -5rem; @extend %darkIconFix !optional; }
58 58
59._icon-codeception:before { background-position: -4rem 0; }
59._icon-sqlite:before { background-position: -5rem 0; @extend %darkIconFix !optional; }60._icon-sqlite:before { background-position: -5rem 0; @extend %darkIconFix !optional; }
60._icon-async:before { background-position: -6rem 0; @extend %darkIconFix !optional; }61._icon-async:before { background-position: -6rem 0; @extend %darkIconFix !optional; }
61._icon-http:before { background-position: -7rem 0; @extend %darkIconFix !optional; }62._icon-http:before { background-position: -7rem 0; @extend %darkIconFix !optional; }
assets/stylesheets/pages/_codeception.scss+2−24
@@ -1,29 +1,7 @@
1._codeception {1._codeception {
2 @extend %simple;2 @extend %simple;
3 3
4 .warning, .alert {4 h4 { @extend %block-label; }
5 @extend %note;5 .warning, .alert { @extend %note; }
6 }
7
8 h4 {
9 @extend %block-label;
10 }
11
12 .alert-danger { @extend %note-red; }6 .alert-danger { @extend %note-red; }
13
14 .language-yaml {
15 @extend .token;
16
17 .c1 {
18 @extend .token.comment;
19 }
20
21 .pi {
22 @extend .token.operator;
23 }
24
25 .s {
26 @extend .token.string;
27 }
28 }
29}7}
lib/docs/filters/codeception/clean_html.rb+31−8
@@ -2,17 +2,40 @@ module Docs · 1 unmodified line
2 class Codeception2 class Codeception
3 class CleanHtmlFilter < Filter3 class CleanHtmlFilter < Filter
4 def call4 def call
5 root_page? ? root : other5 @doc = doc.at_css('div.page, div.content')
6 doc
7 end
8 6
9 def root7 while doc.element_children.length == 1
10 doc.inner_html = ' '8 doc.first_element_child.before(doc.first_element_child.children).remove
11 end9 end
10
11 if root_page?
12 at_css('h1').content = 'Codeception Documentation'
13 end
14
15 unless at_css('h1')
16 at_css('h2').name = 'h1'
17 end
18
19 unless at_css('h2')
20 css('h3').each { |node| node.name = 'h2' }
21 css('h4').each { |node| node.name = 'h3' }
22 end
12 23
13 def other
14 @doc = doc.at_css('div.page, div.content')
15 css('.btn-group').remove24 css('.btn-group').remove
25
26 css('.alert:last-child').each do |node|
27 node.remove if node.content.include?('taken from the source code')
28 end
29
30 css('.highlight').each do |node|
31 node.before(node.children).remove
32 end
33
34 css('pre > code').each do |node|
35 node.parent['data-language'] = node['data-lang']
36 node.parent.content = node.parent.content
37 end
38
16 doc39 doc
17 end40 end
18 end41 end
lib/docs/filters/codeception/entries.rb+23−21
@@ -2,46 +2,48 @@ module Docs · 1 unmodified line
2 class Codeception2 class Codeception
3 class EntriesFilter < Docs::EntriesFilter3 class EntriesFilter < Docs::EntriesFilter
4 def get_name4 def get_name
5 (subpath.scan(/\d\d/).first || '') + ' ' + (at_css('h1') || at_css('h2')).content5 name = (at_css('h1') || at_css('h2')).content
6 end
7 6
8 def get_type7 if number = subpath[/\A\d+/]
9 return 'Module::' + (at_css('h1') || at_css('h2')).content if subpath.start_with?('modules')8 name.prepend "#{number.to_i}. "
10 return 'Guides' if subpath =~ /\d\d/9 end
11 (at_css('h1') || at_css('h2')).content10
11 name
12 end12 end
13 13
14 def include_default_entry?14 def get_type
15 return true if %w(Guides).include? type15 if subpath =~ /\d\d/
16 return true if type =~ /(Module)|(Util)/16 'Guides'
17 false17 elsif subpath.start_with?('modules')
18 "Module: #{name}"
19 elsif name.include?('Util')
20 "Util Class: #{name.split('\\').last}"
21 else
22 "Reference: #{name}"
23 end
18 end24 end
19 25
20 def additional_entries26 def additional_entries
21
22 if type =~ /Module/27 if type =~ /Module/
23 prefix = type.sub(/Module::/, '')+ '::'28 prefix = "#{name}::"
24 pattern = '#actions ~ h4'29 pattern = '#actions ~ h4'
25 elsif type =~ /Functions/30 elsif type =~ /Functions/
26 prefix = ''31 prefix = ''
27 pattern = 'h4'32 pattern = '#page h4'
28 elsif type =~ /Util/33 elsif name =~ /Util/
29 prefix = type.sub(/Codeception\/Util/, '')+ '::'34 prefix = "#{name.remove('Codeception\\Util\\')}::"
30 pattern = 'h3'35 pattern = 'h3'
31 elsif type =~ /(Commands)|(Configuration)/36 elsif type =~ /(Commands)|(Configuration)/
32 prefix = ''
33 pattern = 'h2'
34 else
35 prefix = ''37 prefix = ''
36 pattern = 'none'38 pattern = 'h2'
37 end39 end
38 40
41 return [] unless pattern
42
39 css(pattern).map do |node|43 css(pattern).map do |node|
40 [prefix + node.content, node['id']]44 [prefix + node.content, node['id']]
41 end.compact45 end.compact
42
43 end46 end
44
45 end47 end
46 end48 end
47end49end
lib/docs/scrapers/codeception.rb+4−6
@@ -2,22 +2,20 @@ module Docs · 1 unmodified line
2 class Codeception < UrlScraper2 class Codeception < UrlScraper
3 self.name = 'Codeception'3 self.name = 'Codeception'
4 self.type = 'codeception'4 self.type = 'codeception'
5 self.root_path = 'index.html'5 self.release = '2.2.6'
6 self.release = '2.2'
7 self.base_url = 'http://codeception.com/docs/'6 self.base_url = 'http://codeception.com/docs/'
7 self.root_path = 'index.html'
8 self.links = {8 self.links = {
9 home: 'http://codeception.com/',9 home: 'http://codeception.com/',
10 code: 'https://github.com/codeception/codeception'10 code: 'https://github.com/codeception/codeception'
11 }11 }
12 12
13 html_filters.push 'codeception/clean_html', 'codeception/entries', 'title'13 html_filters.push 'codeception/entries', 'codeception/clean_html'
14 14
15 options[:root_title] = 'Codeception'
16 options[:title] = false
17 options[:skip_patterns] = [/install/]15 options[:skip_patterns] = [/install/]
18 16
19 options[:attribution] = <<-HTML17 options[:attribution] = <<-HTML
20 &copy; 2011–2016 Michael Bodnarchuk and Contributors<br>18 &copy; 2011&ndash;2016 Michael Bodnarchuk and contributors<br>
21 Licensed under the MIT License.19 Licensed under the MIT License.
22 HTML20 HTML
23 end21 end
public/icons/docs/codeception/16.png+0−0

Binary file not shown.

public/icons/docs/codeception/16@2x.png+0−0

Binary file not shown.