History

Finish Crystal scraper

Thibaut Courouble committed 10 years ago

Commit 3036c711 parent 0324bdc16 files · +156 −34Browse files

Unified · Split

16 files
  1. assets/images/docs.png +0 −0
  2. assets/images/docs@2x.png +0 −0
  3. assets/javascripts/collections/types.coffee +1 −1
  4. assets/javascripts/news.json +3 −0
  5. assets/javascripts/templates/pages/about_tmpl.coffee +5 −0
  6. assets/javascripts/views/pages/crystal.coffee +0 −6
  7. assets/javascripts/views/pages/simple.coffee +1 −0
  8. assets/stylesheets/application-dark.css.scss +1 −0
  9. assets/stylesheets/application.css.scss +1 −0
  10. assets/stylesheets/global/_icons.scss +1 −0
  11. assets/stylesheets/pages/_crystal.scss +21 −2
  12. lib/docs/filters/crystal/clean_html.rb +36 −7
  13. lib/docs/filters/crystal/entries.rb +62 −9
  14. lib/docs/scrapers/crystal.rb +28 −13
  15. public/icons/docs/crystal/16.png +0 −0
  16. public/icons/docs/crystal/16@2x.png +0 −0
assets/images/docs.png+0−0

Binary file not shown.

assets/images/docs@2x.png+0−0

Binary file not shown.

assets/javascripts/collections/types.coffee+1−1
@@ -7,7 +7,7 @@ class app.collections.Types extends app.Collection · 6 unmodified lines
7 (result[@_groupFor(type)] ||= []).push(type)7 (result[@_groupFor(type)] ||= []).push(type)
8 result.filter (e) -> e.length > 08 result.filter (e) -> e.length > 0
9 9
10 GUIDES_RGX = /(^|[\s\(])(guides?|tutorials?|reference|playbooks|getting\ started|manual)($|[\s\):])/i10 GUIDES_RGX = /(^|[\s\(])(guides?|tutorials?|reference|book|getting\ started|manual)($|[\s\):])/i
11 11
12 _groupFor: (type) ->12 _groupFor: (type) ->
13 if GUIDES_RGX.test(type.name)13 if GUIDES_RGX.test(type.name)
assets/javascripts/news.json+3−0
@@ -1,5 +1,8 @@
1[1[
2 [2 [
3 "2016-07-24",
4 "New documentation: <a href=\"/crystal/\">Crystal</a>"
5 ], [
3 "2016-07-03",6 "2016-07-03",
4 "New documentations: <a href=\"/cmake/\">CMake</a> and <a href=\"/matplotlib/\">Matplotlib</a>"7 "New documentations: <a href=\"/cmake/\">CMake</a> and <a href=\"/matplotlib/\">Matplotlib</a>"
5 ], [8 ], [
assets/javascripts/templates/pages/about_tmpl.coffee+5−0
@@ -160,6 +160,11 @@ credits = [ · 159 unmodified lines
160 '2005-2016 Mozilla Developer Network and individual contributors',160 '2005-2016 Mozilla Developer Network and individual contributors',
161 'CC BY-SA',161 'CC BY-SA',
162 'https://creativecommons.org/licenses/by-sa/2.5/'162 'https://creativecommons.org/licenses/by-sa/2.5/'
163 ], [
164 'Crystal',
165 '2012-2016 Manas Technology Solutions',
166 'Apache',
167 'https://raw.githubusercontent.com/crystal-lang/crystal/master/LICENSE'
163 ], [168 ], [
164 'D3.js',169 'D3.js',
165 '2010-2016 Michael Bostock',170 '2010-2016 Michael Bostock',
assets/javascripts/views/pages/crystal.coffee+0−6
@@ -1,6 +0,0 @@
1#= require views/pages/base
2
3class app.views.CrystalPage extends app.views.BasePage
4 prepare: ->
5 @highlightCode @findAllByTag('pre'), 'ruby'
6 return
assets/javascripts/views/pages/simple.coffee+1−0
@@ -10,6 +10,7 @@ app.views.AngularPage = · 9 unmodified lines
10app.views.AngularjsPage =10app.views.AngularjsPage =
11app.views.CakephpPage =11app.views.CakephpPage =
12app.views.ChaiPage =12app.views.ChaiPage =
13app.views.CrystalPage =
13app.views.DrupalPage =14app.views.DrupalPage =
14app.views.ElixirPage =15app.views.ElixirPage =
15app.views.EmberPage =16app.views.EmberPage =
assets/stylesheets/application-dark.css.scss+1−0
@@ -39,6 +39,7 @@ · 38 unmodified lines
39 'pages/cakephp',39 'pages/cakephp',
40 'pages/clojure',40 'pages/clojure',
41 'pages/coffeescript',41 'pages/coffeescript',
42 'pages/crystal',
42 'pages/d3',43 'pages/d3',
43 'pages/dojo',44 'pages/dojo',
44 'pages/drupal',45 'pages/drupal',
assets/stylesheets/application.css.scss+1−0
@@ -39,6 +39,7 @@ · 38 unmodified lines
39 'pages/cakephp',39 'pages/cakephp',
40 'pages/clojure',40 'pages/clojure',
41 'pages/coffeescript',41 'pages/coffeescript',
42 'pages/crystal',
42 'pages/d3',43 'pages/d3',
43 'pages/dojo',44 'pages/dojo',
44 'pages/drupal',45 'pages/drupal',
assets/stylesheets/global/_icons.scss+1−0
@@ -130,6 +130,7 @@ · 129 unmodified lines
130._icon-vue:before { background-position: -3rem -8rem; }130._icon-vue:before { background-position: -3rem -8rem; }
131._icon-opentsdb:before { background-position: -4rem -8rem; }131._icon-opentsdb:before { background-position: -4rem -8rem; }
132._icon-q:before { background-position: -5rem -8rem; }132._icon-q:before { background-position: -5rem -8rem; }
133._icon-crystal:before { background-position: -6rem -8rem; @extend %darkIconFix !optional; }
133._icon-react_native:before { background-position: 0 -9rem; }134._icon-react_native:before { background-position: 0 -9rem; }
134._icon-phalcon:before { background-position: -1rem -9rem; }135._icon-phalcon:before { background-position: -1rem -9rem; }
135._icon-matplotlib:before { background-position: -2rem -9rem; }136._icon-matplotlib:before { background-position: -2rem -9rem; }
assets/stylesheets/pages/_crystal.scss+21−2
@@ -1,7 +1,26 @@
1._crystal {1._crystal {
2 @extend %simple;2 @extend %simple;
3 3
4 blockquote {4 .signature { @extend %code; }
5 @extend %note;5 a.signature, .superclass > a { @extend %label; }
6
7 .entry-detail { margin-top: 1em; }
8 .view-source { float: right; }
9
10 .superclass-hierarchy {
11 list-style: none;
12 padding: 0;
13 overflow: hidden;
14 }
15
16 li.superclass {
17 float: left;
18 margin: 0 .5em 0 0;
19 padding: 0;
20 }
21
22 li.superclass + li.superclass:before {
23 content: '<';
24 margin-right: .5em;
6 }25 }
7}26}
lib/docs/filters/crystal/clean_html.rb+36−7
@@ -2,18 +2,47 @@ module Docs · 1 unmodified line
2 class Crystal2 class Crystal
3 class CleanHtmlFilter < Filter3 class CleanHtmlFilter < Filter
4 def call4 def call
5 slug.start_with?('docs') ? book : api
6 doc
7 end
5 8
6 # Remove class attr from div and child nodes9 def book
7 css("div").each do |node|10 @doc = at_css('.page-inner > section')
8 node.xpath("//@class").remove11
12 css('pre > code').each do |node|
13 node.parent['data-language'] = node['class'][/lang-(\w+)/, 1] if node['class']
14 node.parent.content = node.parent.content
9 end15 end
16 end
17
18 def api
19 @doc = at_css('#main-content')
20
21 at_css('h1 + p').remove if root_page?
22
23 css('.method-permalink', '.doc + br', 'hr', 'a > br', 'div + br').remove
10 24
11 # Set id attributes on <h1> instead of an empty <a>25 css('pre > code').each do |node|
12 css("h1").each do |node|26 node.parent['data-language'] = 'crystal'
13 node["id"] = node.at_css("a")["id"]27 node.parent.content = node.parent.content
14 end28 end
15 29
16 doc30 css('span').each do |node|
31 node.before(node.children).remove
32 end
33
34 css('div.signature').each do |node|
35 node.name = 'h3'
36 node.inner_html = node.inner_html.strip
37 end
38
39 css('.entry-detail a:contains("View source")').each do |node|
40 node['class'] = 'view-source'
41 node.content = 'Source'
42 parent = node.parent
43 node.ancestors('.entry-detail').first.at_css('h3') << node
44 parent.remove
45 end
17 end46 end
18 end47 end
19 end48 end
lib/docs/filters/crystal/entries.rb+62−9
@@ -1,21 +1,74 @@
1module Docs1module Docs
2 class Crystal2 class Crystal
3 class EntriesFilter < Docs::EntriesFilter3 class EntriesFilter < Docs::EntriesFilter
4
5 # Set the name to h1 content
6 def get_name4 def get_name
7 node = at_css("h1")5 if slug.start_with?('docs/')
8 node.content.strip6 name = at_css('.page-inner h1').content.strip
7
8 if slug.start_with?('docs/syntax_and_semantics')
9 name.prepend "#{slug.split('/')[2].titleize}: " if slug.split('/').length > 3
10 elsif slug.split('/').length > 1
11 chapter = slug.split('/')[1].titleize.capitalize
12 name.prepend "#{chapter}: " unless name == chapter
13 end
14
15 name
16 else
17 name = at_css('h1').children.last.content.strip
18 name.remove! %r{\(.*\)}
19 name
20 end
9 end21 end
10 22
11 # Crystal types from url slug
12 def get_type23 def get_type
13 slug["blob/master/"] = ""24 return if root_page?
14 object, method = *slug.split("/")25
15 object = object.capitalize26 if slug.start_with?('docs/syntax_and_semantics')
16 method ? object : "Index"27 'Book: Language'
28 elsif slug.start_with?('docs/')
29 'Book'
30 else
31 hierarchy = at_css('.superclass-hierarchy')
32 if hierarchy && hierarchy.content.include?('Exception')
33 'Exceptions'
34 else
35 type = at_css('#types-list > ul > .current > a').content
36 type = 'Float' if type.start_with?('Float')
37 type = 'Int' if type.start_with?('Int')
38 type = 'UInt' if type.start_with?('UInt')
39 type = 'TCP' if type.start_with?('TCP')
40 type
41 end
42 end
17 end43 end
18 44
45 def additional_entries
46 return [] unless slug.start_with?('api')
47 entries = []
48
49 css('.entry-detail[id$="class-method"]').each do |node|
50 name = node.at_css('.signature > strong').content.strip
51 name.prepend "#{self.name}." unless slug.end_with?('toplevel')
52 id = node['id'] = node['id'].remove(/<.+?>/)
53 entries << [name, id] unless entries.last && entries.last[0] == name
54 end
55
56 css('.entry-detail[id$="instance-method"]').each do |node|
57 name = node.at_css('.signature > strong').content.strip
58 name.prepend "#{self.name}#" unless slug.end_with?('toplevel')
59 id = node['id'] = node['id'].remove(/<.+?>/)
60 entries << [name, id] unless entries.last && entries.last[0] == name
61 end
62
63 css('.entry-detail[id$="macro"]').each do |node|
64 name = node.at_css('.signature > strong').content.strip
65 name.prepend "#{self.name} " unless slug.end_with?('toplevel')
66 id = node['id'] = node['id'].remove(/<.+?>/)
67 entries << [name, id] unless entries.last && entries.last[0] == name
68 end
69
70 entries
71 end
19 end72 end
20 end73 end
21end74end
lib/docs/scrapers/crystal.rb+28−13
@@ -1,22 +1,37 @@
1module Docs1module Docs
2 class Crystal < UrlScraper2 class Crystal < UrlScraper
3 self.name = "Crystal"3 self.type = 'crystal'
4 self.type = "crystal"4 self.release = '0.18.7'
5 self.base_url = "https://github.com/crystal-lang/crystal-book"5 self.base_url = 'https://crystal-lang.org/'
6 self.initial_paths = %w(/blob/master/SUMMARY.md)6 self.root_path = 'api/0.18.7/index.html'
7 self.initial_paths = %w(docs/index.html)
7 self.links = {8 self.links = {
8 home: "https://crystal-lang.org/",9 home: 'https://crystal-lang.org/',
9 code: "https://github.com/crystal-lang/crystal"10 code: 'https://github.com/crystal-lang/crystal'
10 }11 }
11 12
12 html_filters.push "crystal/clean_html", "crystal/entries"13 html_filters.push 'crystal/entries', 'crystal/clean_html'
13 14
14 options[:container] = ".entry-content"15 options[:only_patterns] = [/\Adocs\//, /\Aapi\/#{release}\//]
15 options[:only_patterns] = [/\/blob\/master\/.*\.md/]
16 options[:skip] = %w(/blob/master/README.md)
17 16
18 options[:attribution] = <<-HTML17 options[:replace_paths] = {
19 <a href="http://creativecommons.org/publicdomain/zero/1.0/">CC0</a>18 "api/#{release}/" => "api/#{release}/index.html",
20 HTML19 'docs/' => 'docs/index.html'
20 }
21
22 options[:attribution] = ->(filter) {
23 if filter.slug.start_with?('docs')
24 <<-HTML
25 To the extent possible under law, the persons who contributed to this work
26 have waived<br>all copyright and related or neighboring rights to this work
27 by associating CC0 with it.
28 HTML
29 else
30 <<-HTML
31 &copy; 2012&ndash;2016 Manas Technology Solutions.<br>
32 Licensed under the Apache License, Version 2.0.
33 HTML
34 end
35 }
21 end36 end
22end37end
public/icons/docs/crystal/16.png+0−0

Binary file not shown.

public/icons/docs/crystal/16@2x.png+0−0

Binary file not shown.