History

Finish Twig scraper

Thibaut Courouble committed 10 years ago

Commit 9ad721e1 parent 6b0e60f11 files · +57 −28Browse files

Unified · Split

11 files
  1. assets/images/docs.png +0 −0
  2. assets/images/docs@2x.png +0 −0
  3. assets/javascripts/news.json +3 −0
  4. assets/javascripts/templates/pages/about_tmpl.coffee +5 −0
  5. assets/stylesheets/application-dark.css.scss +1 −1
  6. assets/stylesheets/global/_icons.scss +1 −0
  7. lib/docs/filters/twig/clean_html.rb +29 −1
  8. lib/docs/filters/twig/entries.rb +13 −21
  9. lib/docs/scrapers/twig.rb +9 −9
  10. public/icons/docs/twig/16.png +0 −0
  11. public/icons/docs/twig/16@2x.png +0 −0
assets/images/docs.png+0−0

Binary file not shown.

assets/images/docs@2x.png+0−0

Binary file not shown.

assets/javascripts/news.json+3−0
@@ -1,5 +1,8 @@
1[1[
2 [2 [
3 "2016-09-18",
4 "New documentation: <a href=\"/twig/\">Twig</a>"
5 ], [
3 "2016-09-05",6 "2016-09-05",
4 "New documentations: <a href=\"/fish/\">Fish</a>, <a href=\"/bottle/\">Bottle</a> and <a href=\"/scikit_image/\">scikit-image</a>"7 "New documentations: <a href=\"/fish/\">Fish</a>, <a href=\"/bottle/\">Bottle</a> and <a href=\"/scikit_image/\">scikit-image</a>"
5 ], [8 ], [
assets/javascripts/templates/pages/about_tmpl.coffee+5−0
@@ -524,6 +524,11 @@ credits = [ · 523 unmodified lines
524 '2015 The TensorFlow Authors',524 '2015 The TensorFlow Authors',
525 'Apache',525 'Apache',
526 'https://raw.githubusercontent.com/tensorflow/tensorflow/master/LICENSE'526 'https://raw.githubusercontent.com/tensorflow/tensorflow/master/LICENSE'
527 ], [
528 'Twig',
529 '2009-2016 The Twig Team',
530 'BSD',
531 'http://twig.sensiolabs.org/license'
527 ], [532 ], [
528 'TypeScript',533 'TypeScript',
529 'Microsoft and other contributors',534 'Microsoft and other contributors',
assets/stylesheets/application-dark.css.scss+1−1
@@ -84,8 +84,8 @@ · 83 unmodified lines
84 'pages/socketio',84 'pages/socketio',
85 'pages/sphinx',85 'pages/sphinx',
86 'pages/sphinx_simple',86 'pages/sphinx_simple',
87 'pages/tcl_tk',
88 'pages/support_tables',87 'pages/support_tables',
88 'pages/tcl_tk',
89 'pages/tensorflow',89 'pages/tensorflow',
90 'pages/underscore',90 'pages/underscore',
91 'pages/vagrant',91 'pages/vagrant',
assets/stylesheets/global/_icons.scss+1−0
@@ -118,6 +118,7 @@ · 117 unmodified lines
118._icon-bower:before { background-position: -4rem -6rem; }118._icon-bower:before { background-position: -4rem -6rem; }
119._icon-fish:before { background-position: -5rem -6rem; @extend %darkIconFix !optional; }119._icon-fish:before { background-position: -5rem -6rem; @extend %darkIconFix !optional; }
120._icon-scikit_image:before { background-position: -6rem -6rem; }120._icon-scikit_image:before { background-position: -6rem -6rem; }
121._icon-twig:before { background-position: -7rem -6rem; }
121._icon-bottle:before { background-position: 0 -7rem; }122._icon-bottle:before { background-position: 0 -7rem; }
122._icon-docker:before { background-position: -1rem -7rem; }123._icon-docker:before { background-position: -1rem -7rem; }
123._icon-cakephp:before { background-position: -2rem -7rem; }124._icon-cakephp:before { background-position: -2rem -7rem; }
lib/docs/filters/twig/clean_html.rb+29−1
@@ -2,8 +2,36 @@ module Docs · 1 unmodified line
2 class Twig2 class Twig
3 class CleanHtmlFilter < Filter3 class CleanHtmlFilter < Filter
4 def call4 def call
5 css('.infobar', '.offline-docs', '.headerlink').remove
5 6
6 css('.infobar', '.admonition-note', 'ul.pages', '.offline-docs').remove7 css('.builtin-reference', '.admonition-wrapper', 'h1 > code', 'h2 > code', '.body-web', '.reference em').each do |node|
8 node.before(node.children).remove
9 end
10
11 css('.section').each do |node|
12 node.first_element_child['id'] = node['id'] if node['id']
13 node.before(node.children).remove
14 end
15
16 doc.child.remove until doc.child.name == 'h1'
17
18 css('.literal-block').each do |node|
19 pre = node.at_css('.highlight pre') || node.at_css('pre')
20 pre.content = pre.content
21 node.replace(pre)
22 end
23
24 css('p.versionadded').each do |node|
25 node.name = 'div'
26 end
27
28 css('h3:contains("Arguments")').each do |node|
29 node.name = 'h4'
30 end
31
32 css('.navigation').each do |node|
33 node['style'] = 'text-align: center'
34 end
7 35
8 doc36 doc
9 end37 end
lib/docs/filters/twig/entries.rb+13−21
@@ -2,32 +2,24 @@ module Docs · 1 unmodified line
2 class Twig2 class Twig
3 class EntriesFilter < Docs::EntriesFilter3 class EntriesFilter < Docs::EntriesFilter
4 def get_name4 def get_name
5 name = at_css('h1').content.strip.tr('¶', '')5 at_css('h1').content
6 name
7 end6 end
8 7
9 def get_type8 def get_type
10 if slug.include?('deprecate')9 if slug.start_with?('filters')
11 'Deprecated'10 'Filters'
12 elsif slug.include?('extensions')11 elsif slug.start_with?('functions')
13 'Doc\\Extensions'12 'Functions'
14 elsif slug.include?('tags')13 elsif slug.start_with?('tags')
15 'tags'14 'Tags'
16 elsif slug.include?('filters')15 elsif slug.start_with?('tests')
17 'filters'16 'Tests'
18 elsif slug.include?('functions')17 elsif slug.start_with?('extensions')
19 'functions'18 'Extensions'
20 elsif slug.include?('tests')19 else
21 'tests'20 'Miscellaneous'
22 elsif slug.include?('-operator') || slug.include?('#math') || slug.include?('comparisons')
23 'operators'
24 elsif slug.in?('doc/index') || slug.include?('intro') || slug.include?('recipes') || slug.include?('internals') || slug.include?('coding_standards') || slug.include?('installation') || slug.include?('api') || slug.include?('advanced')
25 'Doc'
26 elsif slug.include?('templates')
27 'Doc\\Templates'
28 end21 end
29 end22 end
30
31 end23 end
32 end24 end
33end25end
lib/docs/scrapers/twig.rb+9−9
@@ -1,20 +1,20 @@
1module Docs1module Docs
2 class Twig < UrlScraper2 class Twig < UrlScraper
3 self.type = 'twig'3 self.type = 'sphinx'
4 self.release = '1.24'
5 self.base_url = 'http://twig.sensiolabs.org/doc/'
6 self.root_path = 'index.html'
7 self.initial_paths = %w(extensions/index.html)
4 8
5 options[:attribution] = <<-HTML9 options[:attribution] = <<-HTML
6 &copy; 2009&ndash;2016 by SensioLabs<br>10 &copy; 2009&ndash;2016 by the Twig Team<br>
7 Licensed under the three clause BSD license.11 Licensed under the three clause BSD license.<br>
12 The Twig logo is &copy; 2010&ndash;2016 SensioLabs
8 HTML13 HTML
9 14
10 self.release = '1.24.1'
11 self.base_url = 'http://twig.sensiolabs.org/'
12 self.root_path = 'documentation'
13
14 html_filters.push 'twig/clean_html', 'twig/entries'15 html_filters.push 'twig/clean_html', 'twig/entries'
15 16
16 options[:container] = 'div.bd > div.content'17 options[:container] = 'div.bd > div.content'
17 options[:skip_patterns] = [/\Aapi/, /\Alicense/]18 options[:skip] = %w(deprecated.html advanced_legacy.html)
18 options[:skip] = %w(doc/deprecated.html doc/advanced_legacy.html)
19 end19 end
20end20end
public/icons/docs/twig/16.png+0−0

Binary file not shown.

public/icons/docs/twig/16@2x.png+0−0

Binary file not shown.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t