Finish Vulkan scraper

Thibaut Courouble committed 9 years ago

Commit 0df3a771 parent 2f62bca11 files · +62 −52Browse files

Unified · Split

11 files

assets/images/docs-2.png +0 −0
assets/images/docs-2@2x.png +0 −0
assets/javascripts/news.json +1 −1
assets/javascripts/templates/pages/about_tmpl.coffee +5 −0
assets/stylesheets/global/_icons.scss +1 −0
assets/stylesheets/pages/_simple.scss +1 −0
lib/docs/filters/vulkan/clean_html.rb +43 −3
lib/docs/filters/vulkan/entries.rb +5 −32
lib/docs/scrapers/vulkan.rb +10 −20
public/icons/docs/vulkan/16.png +0 −0
public/icons/docs/vulkan/16@2x.png +0 −0

assets/images/docs-2.png+0−0

Binary file not shown.

assets/images/docs-2@2x.png+0−0

Binary file not shown.

assets/javascripts/news.json+1−1

@@ -1,7 +1,7 @@
1	1	[
2	2	[
3	3	"2017-09-03",
4		− "New documentation: <a href=\"/nim/\">Nim</a>"
	4	+ "New documentations: <a href=\"/nim/\">Nim</a> and <a href=\"/vulkan/\">Vulkan</a>"
5	5	], [
6	6	"2017-07-23",
7	7	"New documentation: <a href=\"/godot/\">Godot</a>"

assets/javascripts/templates/pages/about_tmpl.coffee+5−0

@@ -633,6 +633,11 @@ credits = [ · 632 unmodified lines
633	633	'2013-2017 Evan You, Vue.js contributors',
634	634	'MIT',
635	635	'https://raw.githubusercontent.com/vuejs/vue/master/LICENSE'
	636	+ ], [
	637	+ 'Vulkan',
	638	+ '2014-2017 Khronos Group Inc.<br>Vulkan and the Vulkan logo are registered trademarks of the Khronos Group Inc.',
	639	+ 'CC BY',
	640	+ 'https://creativecommons.org/licenses/by/4.0/'
636	641	], [
637	642	'webpack',
638	643	'JS Foundation and other contributors',

assets/stylesheets/global/_icons.scss+1−0

@@ -174,3 +174,4 @@ · 173 unmodified lines
174	174	._icon-falcon:before { background-position: -3rem -2rem; @extend %doc-icon-2; }
175	175	._icon-godot:before { background-position: -4rem -2rem; @extend %doc-icon-2; }
176	176	._icon-nim:before { background-position: -5rem -2rem; @extend %doc-icon-2; @extend %darkIconFix !optional; }
	177	+._icon-vulkan:before { background-position: -6rem -2rem; @extend %doc-icon-2; @extend %darkIconFix !optional; }

assets/stylesheets/pages/_simple.scss+1−0

@@ -43,6 +43,7 @@ · 42 unmodified lines
43	43	._requirejs,
44	44	._typescript,
45	45	._vagrant,
	46	+._vulkan,
46	47	._yarn {
47	48	@extend %simple;
48	49	}

lib/docs/filters/vulkan/clean_html.rb+43−3

@@ -2,11 +2,51 @@ module Docs · 1 unmodified line
2	2	class Vulkan
3	3	class CleanHtmlFilter < Filter
4	4	def call
5		− # Copyright is already added via attribution option
6		− css('#_copyright').map do \|node\|
7		− node.parent.remove
	5	+ at_css('#_copyright').parent.remove
	6	+
	7	+ css('.sect1', '.sectionbody', '.sect2', '.sect3', 'div.paragraph', 'li > p:only-child', 'dd > p:only-child', 'span', '.ulist').each do \|node\|
	8	+ node.before(node.children).remove
	9	+ end
	10	+
	11	+ css('a[id]:empty').each do \|node\|
	12	+ node.parent['id'] \|\|= node['id']
	13	+ node.remove
	14	+ end
	15	+
	16	+ css('.listingblock').each do \|node\|
	17	+ node['data-language'] = node.at_css('[data-lang]')['data-lang']
	18	+ node.content = node.content.strip
	19	+ node.name = 'pre'
	20	+ node.remove_attribute('class')
	21	+ end
	22	+
	23	+ css('.sidebarblock').each do \|node\|
	24	+ node.name = 'blockquote'
	25	+ node.at_css('.title').name = 'h5'
	26	+ node.css('div').each { \|n\| n.before(n.children).remove }
	27	+ node.remove_attribute('class')
8	28	end
9	29
	30	+ css('.admonitionblock').each do \|node\|
	31	+ node.name = 'blockquote'
	32	+ node.children = node.at_css('.content').children
	33	+ node.at_css('.title').name = 'h5'
	34	+ node.remove_attribute('class')
	35	+ end
	36	+
	37	+ css('table').each do \|node\|
	38	+ node.before %(<div class="_table"></div>)
	39	+ node.previous_element << node
	40	+ end
	41	+
	42	+ css('strong', 'dt', 'a').remove_attr('class')
	43	+
	44	+ css('h4 + h4').each do \|node\|
	45	+ node.previous_element.remove
	46	+ end
	47	+
	48	+ css('p:contains("This page is extracted from the Vulkan Specification. Fixes and changes should be made to the Specification, not directly.")').remove
	49	+
10	50	doc
11	51	end
12	52	end

lib/docs/filters/vulkan/entries.rb+5−32

@@ -1,39 +1,12 @@
1	1	module Docs
2	2	class Vulkan
3	3	class EntriesFilter < Docs::EntriesFilter
4		− def get_name
5		− name = at_css('h1').content.strip
6		− name
7		− end
8		−
9		− def get_type
10		− # As only documentation is single-paged, hardcode type
11		− initial_page? ? 'Vulkan' : 'Specifications'
12		− end
13		−
14		− def include_default_entry?
15		− # additional_entries is responsible to extract relevant entries
16		− false
17		− end
18		−
19	4	def additional_entries
20		− if initial_page?
21		− # We pack each subsections into their corresponding category for apispec.html
22		− subsections = css('.sect2').map do \|node\|
23		− # Parse '.sect1' parent, to know what is the entry's type
24		− parent_node = node.parent.parent
25		− # Type is the parent's h2 header
26		− type = parent_node.at_css('h2').content.strip
27		− # Entry node is the one under h3
28		− header_node = node.at_css('h3')
29		− [header_node.content, header_node['id'], type]
30		− end
31		− else
32		− # We create a new category for vkspec.html page
33		− main_sections = css('.sect1').map do \|node\|
34		− # Entry node is the one under h2
35		− header_node = node.at_css('h2')
36		− [header_node.content, header_node['id'], 'Specifications']
	5	+ css('.sect1').each_with_object [] do \|node, entries\|
	6	+ type = node.at_css('h2').content
	7	+
	8	+ node.css('h3').each do \|n\|
	9	+ entries << [n.content, n['id'], type]
37	10	end
38	11	end
39	12	end

lib/docs/scrapers/vulkan.rb+10−20

@@ -1,34 +1,24 @@
1	1	module Docs
2		− # class Vulkan < FileScraper
3	2	class Vulkan < UrlScraper
4	3	self.name = 'Vulkan'
5		−
6		− self.slug = 'vk'
7	4	self.type = 'vulkan'
	5	+ self.release = '1.0.59'
	6	+ self.base_url = 'https://www.khronos.org/registry/vulkan/specs/1.0/'
	7	+ self.root_path = 'apispec.html'
8	8	self.links = {
9		− home: 'https://www.khronos.org/registry/vulkan/specs/',
10		− code: 'https://github.com/KhronosGroup/Vulkan-Docs'
	9	+ home: 'https://www.khronos.org/vulkan/'
11	10	}
12	11
13		− self.root_path = 'apispec.html'
14		−
15		− self.release = '1.0.56'
16		− # self.dir = '/mnt/d/theblackunknown/Documents/GitHub/Vulkan-Docs/out/1.0/'
17		− self.base_url = 'https://www.khronos.org/registry/vulkan/specs/1.0/'
18		−
19		− html_filters.push 'vulkan/entries', 'vulkan/clean_html'
	12	+ html_filters.push 'vulkan/entries', 'vulkan/clean_html', 'title'
20	13
21		− # in apispec.html, skip #header and #footer
	14	+ options[:skip_links] = true
22	15	options[:container] = '#content'
23		−
24		− # If we only want API, we should skip this one
25		− options[:skip] = %w(
26		− html/vkspec.html
27		− )
	16	+ options[:root_title] = 'Vulkan API Reference'
28	17
29	18	options[:attribution] = <<-HTML
30		− Copyright © 2014-2017 Khronos Group. <br>
31		− This work is licensed under a Creative Commons Attribution 4.0 International License
	19	+ © 2014–2017 Khronos Group Inc.<br>
	20	+ Licensed under the Creative Commons Attribution 4.0 International License.<br>
	21	+ Vulkan and the Vulkan logo are registered trademarks of the Khronos Group Inc.
32	22	HTML
33	23	end
34	24	end

public/icons/docs/vulkan/16.png+0−0

Binary file not shown.

public/icons/docs/vulkan/16@2x.png+0−0

Binary file not shown.