Finish Crystal scraper

Thibaut Courouble committed 10 years ago

Commit 3036c711 parent 0324bdc16 files · +156 −34Browse files

Unified · Split

16 files

assets/images/docs.png +0 −0
assets/images/docs@2x.png +0 −0
assets/javascripts/collections/types.coffee +1 −1
assets/javascripts/news.json +3 −0
assets/javascripts/templates/pages/about_tmpl.coffee +5 −0
assets/javascripts/views/pages/crystal.coffee +0 −6
assets/javascripts/views/pages/simple.coffee +1 −0
assets/stylesheets/application-dark.css.scss +1 −0
assets/stylesheets/application.css.scss +1 −0
assets/stylesheets/global/_icons.scss +1 −0
assets/stylesheets/pages/_crystal.scss +21 −2
lib/docs/filters/crystal/clean_html.rb +36 −7
lib/docs/filters/crystal/entries.rb +62 −9
lib/docs/scrapers/crystal.rb +28 −13
public/icons/docs/crystal/16.png +0 −0
public/icons/docs/crystal/16@2x.png +0 −0

assets/images/docs.png+0−0

Binary file not shown.

assets/images/docs@2x.png+0−0

Binary file not shown.

assets/javascripts/collections/types.coffee+1−1

@@ -7,7 +7,7 @@ class app.collections.Types extends app.Collection · 6 unmodified lines
7	7	(result[@_groupFor(type)] \|\|= []).push(type)
8	8	result.filter (e) -> e.length > 0
9	9
10		− GUIDES_RGX = /(^\|[\s$])(guides?\|tutorials?\|reference\|playbooks\|getting\ started\|manual)($\|[\s$:])/i
	10	+ GUIDES_RGX = /(^\|[\s$])(guides?\|tutorials?\|reference\|book\|getting\ started\|manual)($\|[\s$:])/i
11	11
12	12	_groupFor: (type) ->
13	13	if GUIDES_RGX.test(type.name)

assets/javascripts/news.json+3−0

@@ -1,5 +1,8 @@
1	1	[
2	2	[
	3	+ "2016-07-24",
	4	+ "New documentation: <a href=\"/crystal/\">Crystal</a>"
	5	+ ], [
3	6	"2016-07-03",
4	7	"New documentations: <a href=\"/cmake/\">CMake</a> and <a href=\"/matplotlib/\">Matplotlib</a>"
5	8	], [

assets/javascripts/templates/pages/about_tmpl.coffee+5−0

@@ -160,6 +160,11 @@ credits = [ · 159 unmodified lines
160	160	'2005-2016 Mozilla Developer Network and individual contributors',
161	161	'CC BY-SA',
162	162	'https://creativecommons.org/licenses/by-sa/2.5/'
	163	+ ], [
	164	+ 'Crystal',
	165	+ '2012-2016 Manas Technology Solutions',
	166	+ 'Apache',
	167	+ 'https://raw.githubusercontent.com/crystal-lang/crystal/master/LICENSE'
163	168	], [
164	169	'D3.js',
165	170	'2010-2016 Michael Bostock',

assets/javascripts/views/pages/crystal.coffee+0−6

@@ -1,6 +0,0 @@
1		−#= require views/pages/base
2		−
3		−class app.views.CrystalPage extends app.views.BasePage
4		− prepare: ->
5		− @highlightCode @findAllByTag('pre'), 'ruby'
6		− return

assets/javascripts/views/pages/simple.coffee+1−0

@@ -10,6 +10,7 @@ app.views.AngularPage = · 9 unmodified lines
10	10	app.views.AngularjsPage =
11	11	app.views.CakephpPage =
12	12	app.views.ChaiPage =
	13	+app.views.CrystalPage =
13	14	app.views.DrupalPage =
14	15	app.views.ElixirPage =
15	16	app.views.EmberPage =

assets/stylesheets/application-dark.css.scss+1−0

@@ -39,6 +39,7 @@ · 38 unmodified lines
39	39	'pages/cakephp',
40	40	'pages/clojure',
41	41	'pages/coffeescript',
	42	+ 'pages/crystal',
42	43	'pages/d3',
43	44	'pages/dojo',
44	45	'pages/drupal',

assets/stylesheets/application.css.scss+1−0

@@ -39,6 +39,7 @@ · 38 unmodified lines
39	39	'pages/cakephp',
40	40	'pages/clojure',
41	41	'pages/coffeescript',
	42	+ 'pages/crystal',
42	43	'pages/d3',
43	44	'pages/dojo',
44	45	'pages/drupal',

assets/stylesheets/global/_icons.scss+1−0

@@ -130,6 +130,7 @@ · 129 unmodified lines
130	130	._icon-vue:before { background-position: -3rem -8rem; }
131	131	._icon-opentsdb:before { background-position: -4rem -8rem; }
132	132	._icon-q:before { background-position: -5rem -8rem; }
	133	+._icon-crystal:before { background-position: -6rem -8rem; @extend %darkIconFix !optional; }
133	134	._icon-react_native:before { background-position: 0 -9rem; }
134	135	._icon-phalcon:before { background-position: -1rem -9rem; }
135	136	._icon-matplotlib:before { background-position: -2rem -9rem; }

assets/stylesheets/pages/_crystal.scss+21−2

@@ -1,7 +1,26 @@
1	1	._crystal {
2	2	@extend %simple;
3	3
4		− blockquote {
5		− @extend %note;
	4	+ .signature { @extend %code; }
	5	+ a.signature, .superclass > a { @extend %label; }
	6	+
	7	+ .entry-detail { margin-top: 1em; }
	8	+ .view-source { float: right; }
	9	+
	10	+ .superclass-hierarchy {
	11	+ list-style: none;
	12	+ padding: 0;
	13	+ overflow: hidden;
	14	+ }
	15	+
	16	+ li.superclass {
	17	+ float: left;
	18	+ margin: 0 .5em 0 0;
	19	+ padding: 0;
	20	+ }
	21	+
	22	+ li.superclass + li.superclass:before {
	23	+ content: '<';
	24	+ margin-right: .5em;
6	25	}
7	26	}

lib/docs/filters/crystal/clean_html.rb+36−7

@@ -2,18 +2,47 @@ module Docs · 1 unmodified line
2	2	class Crystal
3	3	class CleanHtmlFilter < Filter
4	4	def call
	5	+ slug.start_with?('docs') ? book : api
	6	+ doc
	7	+ end
5	8
6		− # Remove class attr from div and child nodes
7		− css("div").each do \|node\|
8		− node.xpath("//@class").remove
	9	+ def book
	10	+ @doc = at_css('.page-inner > section')
	11	+
	12	+ css('pre > code').each do \|node\|
	13	+ node.parent['data-language'] = node['class'][/lang-(\w+)/, 1] if node['class']
	14	+ node.parent.content = node.parent.content
9	15	end
	16	+ end
	17	+
	18	+ def api
	19	+ @doc = at_css('#main-content')
	20	+
	21	+ at_css('h1 + p').remove if root_page?
	22	+
	23	+ css('.method-permalink', '.doc + br', 'hr', 'a > br', 'div + br').remove
10	24
11		− # Set id attributes on <h1> instead of an empty <a>
12		− css("h1").each do \|node\|
13		− node["id"] = node.at_css("a")["id"]
	25	+ css('pre > code').each do \|node\|
	26	+ node.parent['data-language'] = 'crystal'
	27	+ node.parent.content = node.parent.content
14	28	end
15	29
16		− doc
	30	+ css('span').each do \|node\|
	31	+ node.before(node.children).remove
	32	+ end
	33	+
	34	+ css('div.signature').each do \|node\|
	35	+ node.name = 'h3'
	36	+ node.inner_html = node.inner_html.strip
	37	+ end
	38	+
	39	+ css('.entry-detail a:contains("View source")').each do \|node\|
	40	+ node['class'] = 'view-source'
	41	+ node.content = 'Source'
	42	+ parent = node.parent
	43	+ node.ancestors('.entry-detail').first.at_css('h3') << node
	44	+ parent.remove
	45	+ end
17	46	end
18	47	end
19	48	end

lib/docs/filters/crystal/entries.rb+62−9

@@ -1,21 +1,74 @@
1	1	module Docs
2	2	class Crystal
3	3	class EntriesFilter < Docs::EntriesFilter
4		−
5		− # Set the name to h1 content
6	4	def get_name
7		− node = at_css("h1")
8		− node.content.strip
	5	+ if slug.start_with?('docs/')
	6	+ name = at_css('.page-inner h1').content.strip
	7	+
	8	+ if slug.start_with?('docs/syntax_and_semantics')
	9	+ name.prepend "#{slug.split('/')[2].titleize}: " if slug.split('/').length > 3
	10	+ elsif slug.split('/').length > 1
	11	+ chapter = slug.split('/')[1].titleize.capitalize
	12	+ name.prepend "#{chapter}: " unless name == chapter
	13	+ end
	14	+
	15	+ name
	16	+ else
	17	+ name = at_css('h1').children.last.content.strip
	18	+ name.remove! %r{$.*$}
	19	+ name
	20	+ end
9	21	end
10	22
11		− # Crystal types from url slug
12	23	def get_type
13		− slug["blob/master/"] = ""
14		− object, method = *slug.split("/")
15		− object = object.capitalize
16		− method ? object : "Index"
	24	+ return if root_page?
	25	+
	26	+ if slug.start_with?('docs/syntax_and_semantics')
	27	+ 'Book: Language'
	28	+ elsif slug.start_with?('docs/')
	29	+ 'Book'
	30	+ else
	31	+ hierarchy = at_css('.superclass-hierarchy')
	32	+ if hierarchy && hierarchy.content.include?('Exception')
	33	+ 'Exceptions'
	34	+ else
	35	+ type = at_css('#types-list > ul > .current > a').content
	36	+ type = 'Float' if type.start_with?('Float')
	37	+ type = 'Int' if type.start_with?('Int')
	38	+ type = 'UInt' if type.start_with?('UInt')
	39	+ type = 'TCP' if type.start_with?('TCP')
	40	+ type
	41	+ end
	42	+ end
17	43	end
18	44
	45	+ def additional_entries
	46	+ return [] unless slug.start_with?('api')
	47	+ entries = []
	48	+
	49	+ css('.entry-detail[id$="class-method"]').each do \|node\|
	50	+ name = node.at_css('.signature > strong').content.strip
	51	+ name.prepend "#{self.name}." unless slug.end_with?('toplevel')
	52	+ id = node['id'] = node['id'].remove(/<.+?>/)
	53	+ entries << [name, id] unless entries.last && entries.last[0] == name
	54	+ end
	55	+
	56	+ css('.entry-detail[id$="instance-method"]').each do \|node\|
	57	+ name = node.at_css('.signature > strong').content.strip
	58	+ name.prepend "#{self.name}#" unless slug.end_with?('toplevel')
	59	+ id = node['id'] = node['id'].remove(/<.+?>/)
	60	+ entries << [name, id] unless entries.last && entries.last[0] == name
	61	+ end
	62	+
	63	+ css('.entry-detail[id$="macro"]').each do \|node\|
	64	+ name = node.at_css('.signature > strong').content.strip
	65	+ name.prepend "#{self.name} " unless slug.end_with?('toplevel')
	66	+ id = node['id'] = node['id'].remove(/<.+?>/)
	67	+ entries << [name, id] unless entries.last && entries.last[0] == name
	68	+ end
	69	+
	70	+ entries
	71	+ end
19	72	end
20	73	end
21	74	end

lib/docs/scrapers/crystal.rb+28−13

@@ -1,22 +1,37 @@
1	1	module Docs
2	2	class Crystal < UrlScraper
3		− self.name = "Crystal"
4		− self.type = "crystal"
5		− self.base_url = "https://github.com/crystal-lang/crystal-book"
6		− self.initial_paths = %w(/blob/master/SUMMARY.md)
	3	+ self.type = 'crystal'
	4	+ self.release = '0.18.7'
	5	+ self.base_url = 'https://crystal-lang.org/'
	6	+ self.root_path = 'api/0.18.7/index.html'
	7	+ self.initial_paths = %w(docs/index.html)
7	8	self.links = {
8		− home: "https://crystal-lang.org/",
9		− code: "https://github.com/crystal-lang/crystal"
	9	+ home: 'https://crystal-lang.org/',
	10	+ code: 'https://github.com/crystal-lang/crystal'
10	11	}
11	12
12		− html_filters.push "crystal/clean_html", "crystal/entries"
	13	+ html_filters.push 'crystal/entries', 'crystal/clean_html'
13	14
14		− options[:container] = ".entry-content"
15		− options[:only_patterns] = [/\/blob\/master\/.*\.md/]
16		− options[:skip] = %w(/blob/master/README.md)
	15	+ options[:only_patterns] = [/\Adocs\//, /\Aapi\/#{release}\//]
17	16
18		− options[:attribution] = <<-HTML
19		− <a href="http://creativecommons.org/publicdomain/zero/1.0/">CC0</a>
20		− HTML
	17	+ options[:replace_paths] = {
	18	+ "api/#{release}/" => "api/#{release}/index.html",
	19	+ 'docs/' => 'docs/index.html'
	20	+ }
	21	+
	22	+ options[:attribution] = ->(filter) {
	23	+ if filter.slug.start_with?('docs')
	24	+ <<-HTML
	25	+ To the extent possible under law, the persons who contributed to this work
	26	+ have waived<br>all copyright and related or neighboring rights to this work
	27	+ by associating CC0 with it.
	28	+ HTML
	29	+ else
	30	+ <<-HTML
	31	+ © 2012–2016 Manas Technology Solutions.<br>
	32	+ Licensed under the Apache License, Version 2.0.
	33	+ HTML
	34	+ end
	35	+ }
21	36	end
22	37	end

public/icons/docs/crystal/16.png+0−0

Binary file not shown.

public/icons/docs/crystal/16@2x.png+0−0

Binary file not shown.