History

Finish Codeception scraper

Thibaut Courouble committed 9 years ago

Commit 802a8d31 parent 72ab34313 files · +64 −66Browse files

Unified · Split

13 files
  1. assets/images/docs.png +0 −0
  2. assets/images/docs@2x.png +0 −0
  3. assets/javascripts/news.json +1 −1
  4. assets/javascripts/templates/pages/about_tmpl.coffee +5 −0
  5. assets/javascripts/views/pages/codeception.coffee +0 −10
  6. assets/javascripts/views/pages/simple.coffee +1 −0
  7. assets/stylesheets/global/_icons.scss +1 −0
  8. assets/stylesheets/pages/_codeception.scss +2 −24
  9. lib/docs/filters/codeception/clean_html.rb +31 −8
  10. lib/docs/filters/codeception/entries.rb +23 −21
  11. lib/docs/scrapers/codeception.rb +4 −6
  12. public/icons/docs/codeception/16.png +0 −0
  13. public/icons/docs/codeception/16@2x.png +0 −0
assets/images/docs.png+0−0

Binary file not shown.

assets/images/docs@2x.png+0−0

Binary file not shown.

assets/javascripts/news.json+1−1
@@ -1,7 +1,7 @@
11 [
22 [
33 "2016-12-04",
4− "New documentation: <a href=\"/sqlite/\">SQLite</a>"
4+ "New documentations: <a href=\"/sqlite/\">SQLite</a> and <a href=\"/codeception/\">Codeception</a>"
55 ], [
66 "2016-11-20",
77 "New documentations: <a href=\"/yarn/\">Yarn</a>, <a href=\"/immutable/\">Immutable.js</a> and <a href=\"/async/\">Async</a>"
assets/javascripts/templates/pages/about_tmpl.coffee+5−0
@@ -154,6 +154,11 @@ credits = [ · 153 unmodified lines
154154 '2000-2016 Kitware, Inc.<br>&copy; 2000-2011 Insight Software Consortium',
155155 'BSD',
156156 'https://cmake.org/licensing/'
157+ ], [
158+ 'Codeception',
159+ '2011-2016 Michael Bodnarchuk and contributors',
160+ 'MIT',
161+ 'https://raw.githubusercontent.com/Codeception/Codeception/master/LICENSE'
157162 ], [
158163 'CodeIgniter',
159164 '2014-2016 British Columbia Institute of Technology',
assets/javascripts/views/pages/codeception.coffee+0−10
@@ -1,10 +0,0 @@
1−#= require views/pages/base
2−
3−class app.views.CodeceptionPage extends app.views.BasePage
4− LANGUAGE_RGX = /language-(\w+)/
5−
6− prepare: ->
7− for el in @findAll('pre > code')
8− if el.className.match(LANGUAGE_RGX)
9− @highlightCode(el, el.className.match(LANGUAGE_RGX)[1])
10− return
assets/javascripts/views/pages/simple.coffee+1−0
@@ -14,6 +14,7 @@ app.views.BowerPage = · 13 unmodified lines
1414 app.views.CPage =
1515 app.views.CakephpPage =
1616 app.views.ChaiPage =
17+app.views.CodeceptionPage =
1718 app.views.CoffeescriptPage =
1819 app.views.CordovaPage =
1920 app.views.CrystalPage =
assets/stylesheets/global/_icons.scss+1−0
@@ -56,6 +56,7 @@ · 55 unmodified lines
5656 %icon-close-white { background-position: -2rem -5rem; }
5757 %icon-back { background-position: -3rem -5rem; @extend %darkIconFix !optional; }
5858
59+._icon-codeception:before { background-position: -4rem 0; }
5960 ._icon-sqlite:before { background-position: -5rem 0; @extend %darkIconFix !optional; }
6061 ._icon-async:before { background-position: -6rem 0; @extend %darkIconFix !optional; }
6162 ._icon-http:before { background-position: -7rem 0; @extend %darkIconFix !optional; }
assets/stylesheets/pages/_codeception.scss+2−24
@@ -1,29 +1,7 @@
11 ._codeception {
22 @extend %simple;
33
4− .warning, .alert {
5− @extend %note;
6− }
7−
8− h4 {
9− @extend %block-label;
10− }
11−
4+ h4 { @extend %block-label; }
5+ .warning, .alert { @extend %note; }
126 .alert-danger { @extend %note-red; }
13−
14− .language-yaml {
15− @extend .token;
16−
17− .c1 {
18− @extend .token.comment;
19− }
20−
21− .pi {
22− @extend .token.operator;
23− }
24−
25− .s {
26− @extend .token.string;
27− }
28− }
297 }
lib/docs/filters/codeception/clean_html.rb+31−8
@@ -2,17 +2,40 @@ module Docs · 1 unmodified line
22 class Codeception
33 class CleanHtmlFilter < Filter
44 def call
5− root_page? ? root : other
6− doc
7− end
5+ @doc = doc.at_css('div.page, div.content')
86
9− def root
10− doc.inner_html = ' '
11− end
7+ while doc.element_children.length == 1
8+ doc.first_element_child.before(doc.first_element_child.children).remove
9+ end
10+
11+ if root_page?
12+ at_css('h1').content = 'Codeception Documentation'
13+ end
14+
15+ unless at_css('h1')
16+ at_css('h2').name = 'h1'
17+ end
18+
19+ unless at_css('h2')
20+ css('h3').each { |node| node.name = 'h2' }
21+ css('h4').each { |node| node.name = 'h3' }
22+ end
1223
13− def other
14− @doc = doc.at_css('div.page, div.content')
1524 css('.btn-group').remove
25+
26+ css('.alert:last-child').each do |node|
27+ node.remove if node.content.include?('taken from the source code')
28+ end
29+
30+ css('.highlight').each do |node|
31+ node.before(node.children).remove
32+ end
33+
34+ css('pre > code').each do |node|
35+ node.parent['data-language'] = node['data-lang']
36+ node.parent.content = node.parent.content
37+ end
38+
1639 doc
1740 end
1841 end
lib/docs/filters/codeception/entries.rb+23−21
@@ -2,46 +2,48 @@ module Docs · 1 unmodified line
22 class Codeception
33 class EntriesFilter < Docs::EntriesFilter
44 def get_name
5− (subpath.scan(/\d\d/).first || '') + ' ' + (at_css('h1') || at_css('h2')).content
6− end
5+ name = (at_css('h1') || at_css('h2')).content
76
8− def get_type
9− return 'Module::' + (at_css('h1') || at_css('h2')).content if subpath.start_with?('modules')
10− return 'Guides' if subpath =~ /\d\d/
11− (at_css('h1') || at_css('h2')).content
7+ if number = subpath[/\A\d+/]
8+ name.prepend "#{number.to_i}. "
9+ end
10+
11+ name
1212 end
1313
14− def include_default_entry?
15− return true if %w(Guides).include? type
16− return true if type =~ /(Module)|(Util)/
17− false
14+ def get_type
15+ if subpath =~ /\d\d/
16+ 'Guides'
17+ elsif subpath.start_with?('modules')
18+ "Module: #{name}"
19+ elsif name.include?('Util')
20+ "Util Class: #{name.split('\\').last}"
21+ else
22+ "Reference: #{name}"
23+ end
1824 end
1925
2026 def additional_entries
21−
2227 if type =~ /Module/
23− prefix = type.sub(/Module::/, '')+ '::'
28+ prefix = "#{name}::"
2429 pattern = '#actions ~ h4'
2530 elsif type =~ /Functions/
2631 prefix = ''
27− pattern = 'h4'
28− elsif type =~ /Util/
29− prefix = type.sub(/Codeception\/Util/, '')+ '::'
32+ pattern = '#page h4'
33+ elsif name =~ /Util/
34+ prefix = "#{name.remove('Codeception\\Util\\')}::"
3035 pattern = 'h3'
3136 elsif type =~ /(Commands)|(Configuration)/
32− prefix = ''
33− pattern = 'h2'
34− else
3537 prefix = ''
36− pattern = 'none'
38+ pattern = 'h2'
3739 end
3840
41+ return [] unless pattern
42+
3943 css(pattern).map do |node|
4044 [prefix + node.content, node['id']]
4145 end.compact
42−
4346 end
44−
4547 end
4648 end
4749 end
lib/docs/scrapers/codeception.rb+4−6
@@ -2,22 +2,20 @@ module Docs · 1 unmodified line
22 class Codeception < UrlScraper
33 self.name = 'Codeception'
44 self.type = 'codeception'
5− self.root_path = 'index.html'
6− self.release = '2.2'
5+ self.release = '2.2.6'
76 self.base_url = 'http://codeception.com/docs/'
7+ self.root_path = 'index.html'
88 self.links = {
99 home: 'http://codeception.com/',
1010 code: 'https://github.com/codeception/codeception'
1111 }
1212
13− html_filters.push 'codeception/clean_html', 'codeception/entries', 'title'
13+ html_filters.push 'codeception/entries', 'codeception/clean_html'
1414
15− options[:root_title] = 'Codeception'
16− options[:title] = false
1715 options[:skip_patterns] = [/install/]
1816
1917 options[:attribution] = <<-HTML
20− &copy; 2011–2016 Michael Bodnarchuk and Contributors<br>
18+ &copy; 2011&ndash;2016 Michael Bodnarchuk and contributors<br>
2119 Licensed under the MIT License.
2220 HTML
2321 end
public/icons/docs/codeception/16.png+0−0

Binary file not shown.

public/icons/docs/codeception/16@2x.png+0−0

Binary file not shown.