History

Finish Vulkan scraper

Thibaut Courouble committed 9 years ago

Commit 0df3a771 parent 2f62bca11 files · +62 −52Browse files

Unified · Split

11 files
  1. assets/images/docs-2.png +0 −0
  2. assets/images/docs-2@2x.png +0 −0
  3. assets/javascripts/news.json +1 −1
  4. assets/javascripts/templates/pages/about_tmpl.coffee +5 −0
  5. assets/stylesheets/global/_icons.scss +1 −0
  6. assets/stylesheets/pages/_simple.scss +1 −0
  7. lib/docs/filters/vulkan/clean_html.rb +43 −3
  8. lib/docs/filters/vulkan/entries.rb +5 −32
  9. lib/docs/scrapers/vulkan.rb +10 −20
  10. public/icons/docs/vulkan/16.png +0 −0
  11. public/icons/docs/vulkan/16@2x.png +0 −0
assets/images/docs-2.png+0−0

Binary file not shown.

assets/images/docs-2@2x.png+0−0

Binary file not shown.

assets/javascripts/news.json+1−1
@@ -1,7 +1,7 @@
1[1[
2 [2 [
3 "2017-09-03",3 "2017-09-03",
4 "New documentation: <a href=\"/nim/\">Nim</a>"4 "New documentations: <a href=\"/nim/\">Nim</a> and <a href=\"/vulkan/\">Vulkan</a>"
5 ], [5 ], [
6 "2017-07-23",6 "2017-07-23",
7 "New documentation: <a href=\"/godot/\">Godot</a>"7 "New documentation: <a href=\"/godot/\">Godot</a>"
assets/javascripts/templates/pages/about_tmpl.coffee+5−0
@@ -633,6 +633,11 @@ credits = [ · 632 unmodified lines
633 '2013-2017 Evan You, Vue.js contributors',633 '2013-2017 Evan You, Vue.js contributors',
634 'MIT',634 'MIT',
635 'https://raw.githubusercontent.com/vuejs/vue/master/LICENSE'635 'https://raw.githubusercontent.com/vuejs/vue/master/LICENSE'
636 ], [
637 'Vulkan',
638 '2014-2017 Khronos Group Inc.<br>Vulkan and the Vulkan logo are registered trademarks of the Khronos Group Inc.',
639 'CC BY',
640 'https://creativecommons.org/licenses/by/4.0/'
636 ], [641 ], [
637 'webpack',642 'webpack',
638 'JS Foundation and other contributors',643 'JS Foundation and other contributors',
assets/stylesheets/global/_icons.scss+1−0
@@ -174,3 +174,4 @@ · 173 unmodified lines
174._icon-falcon:before { background-position: -3rem -2rem; @extend %doc-icon-2; }174._icon-falcon:before { background-position: -3rem -2rem; @extend %doc-icon-2; }
175._icon-godot:before { background-position: -4rem -2rem; @extend %doc-icon-2; }175._icon-godot:before { background-position: -4rem -2rem; @extend %doc-icon-2; }
176._icon-nim:before { background-position: -5rem -2rem; @extend %doc-icon-2; @extend %darkIconFix !optional; }176._icon-nim:before { background-position: -5rem -2rem; @extend %doc-icon-2; @extend %darkIconFix !optional; }
177._icon-vulkan:before { background-position: -6rem -2rem; @extend %doc-icon-2; @extend %darkIconFix !optional; }
assets/stylesheets/pages/_simple.scss+1−0
@@ -43,6 +43,7 @@ · 42 unmodified lines
43._requirejs,43._requirejs,
44._typescript,44._typescript,
45._vagrant,45._vagrant,
46._vulkan,
46._yarn {47._yarn {
47 @extend %simple;48 @extend %simple;
48}49}
lib/docs/filters/vulkan/clean_html.rb+43−3
@@ -2,11 +2,51 @@ module Docs · 1 unmodified line
2 class Vulkan2 class Vulkan
3 class CleanHtmlFilter < Filter3 class CleanHtmlFilter < Filter
4 def call4 def call
5 # Copyright is already added via attribution option5 at_css('#_copyright').parent.remove
6 css('#_copyright').map do |node|6
7 node.parent.remove7 css('.sect1', '.sectionbody', '.sect2', '.sect3', 'div.paragraph', 'li > p:only-child', 'dd > p:only-child', 'span', '.ulist').each do |node|
8 node.before(node.children).remove
9 end
10
11 css('a[id]:empty').each do |node|
12 node.parent['id'] ||= node['id']
13 node.remove
14 end
15
16 css('.listingblock').each do |node|
17 node['data-language'] = node.at_css('[data-lang]')['data-lang']
18 node.content = node.content.strip
19 node.name = 'pre'
20 node.remove_attribute('class')
21 end
22
23 css('.sidebarblock').each do |node|
24 node.name = 'blockquote'
25 node.at_css('.title').name = 'h5'
26 node.css('div').each { |n| n.before(n.children).remove }
27 node.remove_attribute('class')
8 end28 end
9 29
30 css('.admonitionblock').each do |node|
31 node.name = 'blockquote'
32 node.children = node.at_css('.content').children
33 node.at_css('.title').name = 'h5'
34 node.remove_attribute('class')
35 end
36
37 css('table').each do |node|
38 node.before %(<div class="_table"></div>)
39 node.previous_element << node
40 end
41
42 css('strong', 'dt', 'a').remove_attr('class')
43
44 css('h4 + h4').each do |node|
45 node.previous_element.remove
46 end
47
48 css('p:contains("This page is extracted from the Vulkan Specification. Fixes and changes should be made to the Specification, not directly.")').remove
49
10 doc50 doc
11 end51 end
12 end52 end
lib/docs/filters/vulkan/entries.rb+5−32
@@ -1,39 +1,12 @@
1module Docs1module Docs
2 class Vulkan2 class Vulkan
3 class EntriesFilter < Docs::EntriesFilter3 class EntriesFilter < Docs::EntriesFilter
4 def get_name
5 name = at_css('h1').content.strip
6 name
7 end
8
9 def get_type
10 # As only documentation is single-paged, hardcode type
11 initial_page? ? 'Vulkan' : 'Specifications'
12 end
13
14 def include_default_entry?
15 # additional_entries is responsible to extract relevant entries
16 false
17 end
18
19 def additional_entries4 def additional_entries
20 if initial_page?5 css('.sect1').each_with_object [] do |node, entries|
21 # We pack each subsections into their corresponding category for apispec.html6 type = node.at_css('h2').content
22 subsections = css('.sect2').map do |node|7
23 # Parse '.sect1' parent, to know what is the entry's type8 node.css('h3').each do |n|
24 parent_node = node.parent.parent9 entries << [n.content, n['id'], type]
25 # Type is the parent's h2 header
26 type = parent_node.at_css('h2').content.strip
27 # Entry node is the one under h3
28 header_node = node.at_css('h3')
29 [header_node.content, header_node['id'], type]
30 end
31 else
32 # We create a new category for vkspec.html page
33 main_sections = css('.sect1').map do |node|
34 # Entry node is the one under h2
35 header_node = node.at_css('h2')
36 [header_node.content, header_node['id'], 'Specifications']
37 end10 end
38 end11 end
39 end12 end
lib/docs/scrapers/vulkan.rb+10−20
@@ -1,34 +1,24 @@
1module Docs1module Docs
2 # class Vulkan < FileScraper
3 class Vulkan < UrlScraper2 class Vulkan < UrlScraper
4 self.name = 'Vulkan'3 self.name = 'Vulkan'
5
6 self.slug = 'vk'
7 self.type = 'vulkan'4 self.type = 'vulkan'
5 self.release = '1.0.59'
6 self.base_url = 'https://www.khronos.org/registry/vulkan/specs/1.0/'
7 self.root_path = 'apispec.html'
8 self.links = {8 self.links = {
9 home: 'https://www.khronos.org/registry/vulkan/specs/',9 home: 'https://www.khronos.org/vulkan/'
10 code: 'https://github.com/KhronosGroup/Vulkan-Docs'
11 }10 }
12 11
13 self.root_path = 'apispec.html'12 html_filters.push 'vulkan/entries', 'vulkan/clean_html', 'title'
14
15 self.release = '1.0.56'
16 # self.dir = '/mnt/d/theblackunknown/Documents/GitHub/Vulkan-Docs/out/1.0/'
17 self.base_url = 'https://www.khronos.org/registry/vulkan/specs/1.0/'
18
19 html_filters.push 'vulkan/entries', 'vulkan/clean_html'
20 13
21 # in apispec.html, skip #header and #footer14 options[:skip_links] = true
22 options[:container] = '#content'15 options[:container] = '#content'
23 16 options[:root_title] = 'Vulkan API Reference'
24 # If we only want API, we should skip this one
25 options[:skip] = %w(
26 html/vkspec.html
27 )
28 17
29 options[:attribution] = <<-HTML18 options[:attribution] = <<-HTML
30 Copyright &copy; 2014-2017 Khronos Group. <br>19 &copy; 2014&ndash;2017 Khronos Group Inc.<br>
31 This work is licensed under a Creative Commons Attribution 4.0 International License20 Licensed under the Creative Commons Attribution 4.0 International License.<br>
21 Vulkan and the Vulkan logo are registered trademarks of the Khronos Group Inc.
32 HTML22 HTML
33 end23 end
34end24end
public/icons/docs/vulkan/16.png+0−0

Binary file not shown.

public/icons/docs/vulkan/16@2x.png+0−0

Binary file not shown.