Finish Twig scraper

Thibaut Courouble committed 10 years ago

Commit 9ad721e1 parent 6b0e60f11 files · +57 −28Browse files

Unified · Split

11 files

Binary file not shown.

Binary file not shown.

@@ -1,5 +1,8 @@
1	1	[
2	2	[
	3	+ "2016-09-18",
	4	+ "New documentation: <a href=\"/twig/\">Twig</a>"
	5	+ ], [
3	6	"2016-09-05",
4	7	"New documentations: <a href=\"/fish/\">Fish</a>, <a href=\"/bottle/\">Bottle</a> and <a href=\"/scikit_image/\">scikit-image</a>"
5	8	], [

@@ -524,6 +524,11 @@ credits = [ · 523 unmodified lines
524	524	'2015 The TensorFlow Authors',
525	525	'Apache',
526	526	'https://raw.githubusercontent.com/tensorflow/tensorflow/master/LICENSE'
	527	+ ], [
	528	+ 'Twig',
	529	+ '2009-2016 The Twig Team',
	530	+ 'BSD',
	531	+ 'http://twig.sensiolabs.org/license'
527	532	], [
528	533	'TypeScript',
529	534	'Microsoft and other contributors',

@@ -118,6 +118,7 @@ · 117 unmodified lines
118	118	._icon-bower:before { background-position: -4rem -6rem; }
119	119	._icon-fish:before { background-position: -5rem -6rem; @extend %darkIconFix !optional; }
120	120	._icon-scikit_image:before { background-position: -6rem -6rem; }
	121	+._icon-twig:before { background-position: -7rem -6rem; }
121	122	._icon-bottle:before { background-position: 0 -7rem; }
122	123	._icon-docker:before { background-position: -1rem -7rem; }
123	124	._icon-cakephp:before { background-position: -2rem -7rem; }

@@ -2,8 +2,36 @@ module Docs · 1 unmodified line
2	2	class Twig
3	3	class CleanHtmlFilter < Filter
4	4	def call
	5	+ css('.infobar', '.offline-docs', '.headerlink').remove
5	6
6		− css('.infobar', '.admonition-note', 'ul.pages', '.offline-docs').remove
	7	+ css('.builtin-reference', '.admonition-wrapper', 'h1 > code', 'h2 > code', '.body-web', '.reference em').each do \|node\|
	8	+ node.before(node.children).remove
	9	+ end
	10	+
	11	+ css('.section').each do \|node\|
	12	+ node.first_element_child['id'] = node['id'] if node['id']
	13	+ node.before(node.children).remove
	14	+ end
	15	+
	16	+ doc.child.remove until doc.child.name == 'h1'
	17	+
	18	+ css('.literal-block').each do \|node\|
	19	+ pre = node.at_css('.highlight pre') \|\| node.at_css('pre')
	20	+ pre.content = pre.content
	21	+ node.replace(pre)
	22	+ end
	23	+
	24	+ css('p.versionadded').each do \|node\|
	25	+ node.name = 'div'
	26	+ end
	27	+
	28	+ css('h3:contains("Arguments")').each do \|node\|
	29	+ node.name = 'h4'
	30	+ end
	31	+
	32	+ css('.navigation').each do \|node\|
	33	+ node['style'] = 'text-align: center'
	34	+ end
7	35
8	36	doc
9	37	end

@@ -2,32 +2,24 @@ module Docs · 1 unmodified line
2	2	class Twig
3	3	class EntriesFilter < Docs::EntriesFilter
4	4	def get_name
5		− name = at_css('h1').content.strip.tr('¶', '')
6		− name
	5	+ at_css('h1').content
7	6	end
8	7
9	8	def get_type
10		− if slug.include?('deprecate')
11		− 'Deprecated'
12		− elsif slug.include?('extensions')
13		− 'Doc\\Extensions'
14		− elsif slug.include?('tags')
15		− 'tags'
16		− elsif slug.include?('filters')
17		− 'filters'
18		− elsif slug.include?('functions')
19		− 'functions'
20		− elsif slug.include?('tests')
21		− 'tests'
22		− elsif slug.include?('-operator') \|\| slug.include?('#math') \|\| slug.include?('comparisons')
23		− 'operators'
24		− elsif slug.in?('doc/index') \|\| slug.include?('intro') \|\| slug.include?('recipes') \|\| slug.include?('internals') \|\| slug.include?('coding_standards') \|\| slug.include?('installation') \|\| slug.include?('api') \|\| slug.include?('advanced')
25		− 'Doc'
26		− elsif slug.include?('templates')
27		− 'Doc\\Templates'
	9	+ if slug.start_with?('filters')
	10	+ 'Filters'
	11	+ elsif slug.start_with?('functions')
	12	+ 'Functions'
	13	+ elsif slug.start_with?('tags')
	14	+ 'Tags'
	15	+ elsif slug.start_with?('tests')
	16	+ 'Tests'
	17	+ elsif slug.start_with?('extensions')
	18	+ 'Extensions'
	19	+ else
	20	+ 'Miscellaneous'
28	21	end
29	22	end
30		−
31	23	end
32	24	end
33	25	end

@@ -1,20 +1,20 @@
1	1	module Docs
2	2	class Twig < UrlScraper
3		− self.type = 'twig'
	3	+ self.type = 'sphinx'
	4	+ self.release = '1.24'
	5	+ self.base_url = 'http://twig.sensiolabs.org/doc/'
	6	+ self.root_path = 'index.html'
	7	+ self.initial_paths = %w(extensions/index.html)
4	8
5	9	options[:attribution] = <<-HTML
6		− © 2009–2016 by SensioLabs<br>
7		− Licensed under the three clause BSD license.
	10	+ © 2009–2016 by the Twig Team<br>
	11	+ Licensed under the three clause BSD license.<br>
	12	+ The Twig logo is © 2010–2016 SensioLabs
8	13	HTML
9	14
10		− self.release = '1.24.1'
11		− self.base_url = 'http://twig.sensiolabs.org/'
12		− self.root_path = 'documentation'
13		−
14	15	html_filters.push 'twig/clean_html', 'twig/entries'
15	16
16	17	options[:container] = 'div.bd > div.content'
17		− options[:skip_patterns] = [/\Aapi/, /\Alicense/]
18		− options[:skip] = %w(doc/deprecated.html doc/advanced_legacy.html)
	18	+ options[:skip] = %w(deprecated.html advanced_legacy.html)
19	19	end
20	20	end

Binary file not shown.

Binary file not shown.

Keyboard shortcuts