Update Haskell scraper

Thibaut committed 12 years ago

Commit 1819d711 parent 9a43dfb13 files · +116 −293Browse files

Unified · Split

13 files

assets/images/icons.png +0 −0
assets/images/icons@2x.png +0 −0
assets/javascripts/templates/pages/about_tmpl.coffee +5 −0
assets/javascripts/templates/pages/news_tmpl.coffee +4 −1
assets/stylesheets/application.css.scss +1 −0
assets/stylesheets/global/_icons.scss +1 −0
assets/stylesheets/pages/_haskell.scss +18 −114
lib/docs/filters/haskell/clean_html.rb +31 −130
lib/docs/filters/haskell/entries.rb +40 −41
lib/docs/scrapers/haskell.rb +19 −11
public/icons/docs/haskell/16.png +0 −0
public/icons/docs/haskell/16@2x.png +0 −0
public/icons/docs/haskell/SOURCE +1 −0

assets/images/icons.png+0−0

Binary file not shown.

assets/images/icons@2x.png+0−0

Binary file not shown.

assets/javascripts/templates/pages/about_tmpl.coffee+5−0

     '2014 Grunt Team',
     'MIT',
     'https://raw.githubusercontent.com/gruntjs/gruntjs.com/master/LICENSE'
+  ], [
+    'Haskell',
+    'The University of Glasgow',
+    'BSD',
+    'http://www.haskell.org/ghc/license'
   ], [
     'HTTP',
     '1999 The Internet Society',

assets/javascripts/templates/pages/news_tmpl.coffee+4−1

   result
 app.news = [
-  [ 1400976000000, # May 25, 2014
+  [ 1402704000000, # June 14, 2014
+    """ New <a href="/haskell/">Haskell</a> documentation """,
+  ], [
+    1400976000000, # May 25, 2014
     """ New <a href="/laravel/">Laravel</a> documentation """,
   ], [
     1399161600000, # May 4, 2014

assets/stylesheets/application.css.scss+1−0

@@ -35,6 +35,7 @@ · 34 unmodified lines
35	'pages/ember',	35	'pages/ember',
36	'pages/express',	36	'pages/express',
37	'pages/go',	37	'pages/go',
		38	'pages/haskell',
38	'pages/jquery',	39	'pages/jquery',
39	'pages/knockout',	40	'pages/knockout',
40	'pages/git',	41	'pages/git',

assets/stylesheets/global/_icons.scss+1−0

@@ -57,3 +57,4 @@ · 56 unmodified lines
57	._icon-grunt:before { background-position: -3rem -8rem; }	57	._icon-grunt:before { background-position: -3rem -8rem; }
58	._icon-maxcdn:before { background-position: -4rem -8rem; }	58	._icon-maxcdn:before { background-position: -4rem -8rem; }
59	._icon-laravel:before { background-position: 0 -9rem; }	59	._icon-laravel:before { background-position: 0 -9rem; }
		60	._icon-haskell:before { background-position: -1rem -9rem; }

assets/stylesheets/pages/_haskell.scss+18−114

@@ -1,121 +1,25 @@
1	._icon-haskell:before {	1	._haskell {
2	background-image: image-url('/icons/docs/haskell/16.png');	2	> h2 { @extend %block-heading; }
3	background-size: cover;	3	> h3 { @extend %block-label; }
4	background-repeat: no-repeat;	4	h4 { font-size: 1em; }
5	}
6
7	.empty-table .empty {
8	display: none;
9	}
10
11	.arguments td.src {
12	background: #faf9e2;
13	width: 30%;
14	}
15
16	th.src,
17	td.src {
18	font-family: $monoFont;
19	font-weight: normal;
20	font-style: normal;
21	background: #f8f8f8;
22	}
23
24	caption {
25	font-weight: bold;
26	text-align: left;
27	font-style: italic;
28	font-size: 1.1em;
29	}
30
31	// remove margin in descript listing
32	dd > pre {
33	@extend %pre;
34	margin: 0;
35	background: #faf9e2;
36	border-color: #dddaaa #dddaaa #d7d7a9;
37	}
38
39	// warnings are red
40	.warning {
41	@extend %note;
42	@extend %note-red;
43	}
44
45
46	// complexity classes are blue boxes
47	.with-complexity {
48	display: flex;
49	display: -webkit-flex;
50
51	justify-content: space-between;
52	-webkit-justify-content: space-between;
53		5
54	align-items: flex-start;	6	.module + .package, p.src > .link { float: right; }
55	-webkit-align-items: flex-start;
56		7
57	align-content: stretch;	8	.src {
58	-webkit-align-content: stretch;	9	white-space: normal;
59	}	10	@extend %code;
60		11	}
61	.complexity {	12	p.src { @extend %block-label, %label-blue; }
62	@extend %note;	13	dt.src { white-space: normal; }
63	@extend %note-blue;
64	margin: 0;
65	margin-left: 1em;
66	margin-bottom: 0.75em;
67	font-style: italic;
68	white-space: nowrap;
69
70	flex-shrink: 0;
71	-webkit-flex-shrink: 0;
72
73	order: 2;
74	-webkit-order: 2;
75	}
76
77	.complexity + span {
78	order: 1;
79	-webkit-order: 1;
80	}
81
82	// add box type to "since: ..."
83	.added {
84	@extend %note;
85	@extend %note-gold;
86	}
87
88	.added-cell {
89	@extend %note-gold;
90	}
91
92	.fields h3 {
93	display: none;
94	}
95		14
96	// separate types more	15	.top > .subs { margin-left: 2em; }
97	.src {	16	.subs p.src { margin-top: 1em; }
98	margin-top: 2.5em;
99	}
100		17
101	h1 + .top .src,	18	dt > code, .complexity, .version { @extend %label; }
102	h2 + .top .src,	19	.complexity, .version { @extend %label-green; }
103	h3 + .top .src,
104	.caption + .top .src {
105	margin-top: 0;
106	}
107		20
108	// but not for first type	21	table { margin: 1em 0; }
109	h1 + .top,	22	td > pre { margin: 0; }
110	h2 + .top,
111	h3 + .top,
112	h4 + .top {
113	margin-top: 0;
114	}
115		23
116	// change color of example code	24	.warning { @extend %note; }
117	.example {
118	border: 1px solid;
119	background: #faf9e2;
120	border-color: #dddaaa #dddaaa #d7d7a9;
121	}	25	}

lib/docs/filters/haskell/clean_html.rb+31−130

@@ -2,146 +2,54 @@ module Docs · 1 unmodified line
2	class Haskell	2	class Haskell
3	class CleanHtmlFilter < Filter	3	class CleanHtmlFilter < Filter
4	def call	4	def call
		5	root_page? ? root : other
		6	doc
		7	end
5		8
6	# remove unwanted elements	9	def root
7	css('#footer', '#package-header', '#module-header', '#synopsis', '.link', '#table-of-contents', '.package').remove	10	css('#description', '#module-list').each do \|node\|
8		11	node.before(node.children).remove
9	# cpations in tables are h3
10	css('table .caption').each do \|node\|
11	node.name = 'h3'
12	end
13
14	# turn captions into real headers
15	css('.caption').each do \|node\|
16	node.name = 'h1'
17	end
18
19	# section
20	css('.top > .caption').each do \|node\|
21	node.name = 'h2'
22	end
23
24	# subsections
25	css('.top > .subs > .caption', '.fields > .caption').each do \|node\|
26	node.name = 'h3'
27	end
28
29	# subsubsections
30	css('.top > .subs > .subs > .caption').each do \|node\|
31	node.name = 'h4'
32	end
33
34	# ...
35	css('.top > .subs > .subs > .subs > .caption').each do \|node\|
36	node.name = 'h5'
37	end
38
39	# ......
40	css('.top > .subs > .subs > .subs > .subs > .caption').each do \|node\|
41	node.name = 'h6'
42	end
43
44	# all pre's are examples
45	css('pre').each do \|node\|
46	node.add_css_class('example')
47	end	12	end
		13	end
48		14
49	# turn source listing in to pre	15	def other
50	css('.src').each do \|node\|	16	css('h1').each do \|node\|
51	if node.name != "td"	17	node.remove if node.content == 'Documentation'
52	node.name = 'pre'
53	end
54	end	18	end
55		19
56	# check if second column of table is totally empty.	20	css('h1, h2, h3, h4').each do \|node\|
57	# and remove it if it is	21	node.name = node.name.sub(/\d/) { \|i\| i.to_i + 1 }
58	css('table').each do \|table\|
59	empty = true
60	table.css('td + td').each do \|snd\|
61	empty = empty && snd['class'] =~ /empty/
62	end
63	if empty
64	# remove empty column
65	table.css('td + td').remove
66	end
67	end	22	end
68		23
69	# move table captions into the tables	24	at_css('#module-header').tap do \|node\|
70	css(".caption + table").each do \|table\|	25	heading = at_css('.caption')
71	caption = table.previous	26	heading.name = 'h1'
72	caption.name = "caption"	27	node.before(heading)
73	caption.parent = table	28	node.before(node.children).remove
74	end	29	end
75		30
76	css(".caption + .show table").each do \|table\|	31	css('#synopsis').remove
77	caption = table.parent.parent.css('.caption')[0]
78	caption.name = 'caption'
79	caption.parent = table
80	end
81		32
82	# better arguments display:	33	css('#interface', 'h2 code').each do \|node\|
83	css('.src + .arguments table').each do \|table\|	34	node.before(node.children).remove
84	src = table.parent.previous # the function name
85	row = doc.document.create_element('tr')
86	table.css('tr')[0].before(row)
87	src.parent = row
88	src.name = "th"
89	src['colspan'] = 2
90	end	35	end
91		36
92	# remove root page title	37	css('a[name]').each do \|node\|
93	if root_page?	38	node['id'] = node['name']
94	at_css('h1').remove	39	node.remove_attribute('name')
95	end	40	end
96		41
97	# add id to links (based on name)	42	css('p.caption').each do \|node\|
98	css('a').each do \|node\|	43	node.name = 'h4'
99	if node['name']
100	node['id'] = node['name']
101	end
102	end
103
104	# make code in description into proper pre
105	css('dd > code').each do \|node\|
106	node.name = 'pre'
107	end	44	end
108		45
109	# add some informational boxes
110	css('em').each do \|node\|	46	css('em').each do \|node\|
111	if node.content == 'Deprecated.'	47	if node.content.start_with?('O(')
112	# Make deprecated messages red.	48	node.name = 'span'
113	node.parent.add_css_class('warning')	49	node['class'] = 'complexity'
114	elsif node.content =~ /O$.*$/	50	elsif node.content.start_with?('Since')
115	# this is big_O notation, but only apply the class if this is not	51	node.name = 'span'
116	# inside running text (it must be at the start of a paragraph)	52	node['class'] = 'version'
117	# from:
118	# <p><em>O(n)</em>. Koel ok</p>
119	# to:
120	# <p class="with-complexity">
121	# <span class="complexity">O(n)</span>
122	# <span>Koel ok</span>
123	# </p>
124	if node.previous == nil
125	node.add_css_class('complexity') # add css class
126	node.name="span" # just make it div
127	node.next.content = node.next.content.gsub(/^. /, "") # remove . if directly after em
128	node.content = node.content.gsub(/\.$/, "") # remove trailing . if it's inside em
129
130	# reparent the nodes
131	cont = doc.document.create_element "p", :class => "with-complexity"
132	node.parent.previous = cont
133	par = node.parent
134	node.parent = cont
135	par.parent = cont
136	par.name = "span"
137	end
138	elsif node.content =~ /Since: .*/
139	# add box to 'Since:' annotations
140	if node.parent.parent.name == "td"
141	node.parent.parent.add_css_class('added-cell')
142	else
143	node.add_css_class('added')
144	end
145	end	53	end
146	end	54	end
147		55
@@ -150,10 +58,3 @@ module Docs · 2 unmodified lines
150	end	58	end
151	end	59	end
152	end	60	end
153
154	class Nokogiri::XML::Node
155	def add_css_class( *classes )
156	existing = (self['class'] \|\| "").split(/\s+/)
157	self['class'] = existing.concat(classes).uniq.join(" ")
158	end
159	end

lib/docs/filters/haskell/entries.rb+40−41

@@ -1,55 +1,54 @@
1	module Docs	1	module Docs
2	class Haskell	2	class Haskell
3	class EntriesFilter < Docs::EntriesFilter	3	class EntriesFilter < Docs::EntriesFilter
		4	IGNORE_ENTRIES_PATHS = %w(
		5	bytestring-0.10.4.0/Data-ByteString-Lazy.html
		6	bytestring-0.10.4.0/Data-ByteString-Char8.html
		7	bytestring-0.10.4.0/Data-ByteString-Lazy-Char8.html
		8	array-0.5.0.0/Data-Array-IArray.html
		9	containers-0.5.5.1/Data-IntMap-Lazy.html
		10	containers-0.5.5.1/Data-Map-Lazy.html
		11	unix-2.7.0.1/System-Posix-Files-ByteString.html
		12	filepath-1.3.0.2/System-FilePath-Windows.html
		13	transformers-0.3.0.0/Control-Monad-Trans-RWS-Lazy.html
		14	transformers-0.3.0.0/Control-Monad-Trans-Writer-Lazy.html
		15	base-4.7.0.0/GHC-Conc-Sync.html
		16	base-4.7.0.0/GHC-IO-Encoding-UTF32.html
		17	unix-2.7.0.1/System-Posix-Terminal-ByteString.html)
4		18
5	# gets name and type in one fell swoop	19	def get_name
6	#	20	at_css('#module-header .caption').content.strip
7	# eg.	21	end
8	# Control.Monad > [Monad, Control]
9	# Control.Concurrent.Mvar > [Concurrent.MVar, Control]
10	# Array > [Array, nil]
11	def get_name_and_type
12	if at_css('h1') && at_css('h1').content == 'Haskell Hierarchical Libraries'
13	puts 'ok'
14	name = 'Haskell'
15	type = nil
16	else
17	# find full module identifier
18	caption = at_css('#module-header .caption')
19		22
20	if caption	23	def get_type
21	# split the module path	24	%w(System.Posix System.Win32 Control.Monad).each do \|type\|
22	parts = caption.content.split('.')	25	return type if name.start_with?(type)
		26	end
23		27
24	if parts.length > 1	28	if name.start_with?('Data')
25	# if more than one part then the	29	name.split('.')[0..1].join('.')
26	# first is the type and the rest is the name	30	else
27	type = parts[0]	31	name.split('.').first
28	name = parts.drop(1).join('.')
29	else
30	# if only one part, this is the name
31	name = parts[0]
32	type = nil
33	end
34	else
35	# no caption found -> no type / no name
36	name = 'no-name'
37	type = 'no-type'
38	end
39	end	32	end
40	[name, type]
41	end	33	end
42		34
43	# get the name	35	def additional_entries
44	def get_name	36	return [] if IGNORE_ENTRIES_PATHS.include?(subpath)
45	n, t = get_name_and_type()	37
46	n	38	css('#synopsis > ul > li').each_with_object [] do \|node, entries\|
		39	link = node.at_css('a')
		40	next unless link['href'].start_with?('#')
		41	name = node.content.strip
		42	name.remove! %r{\A(?:module\|data\|newtype\|class\|type family m\|type)\s+}
		43	name.sub! %r{\A$(.+?)$}, '\1'
		44	name.sub!(/ (?:\:\: (\w+))?.+\z/) { \|_\| $1 ? " (#{$1})" : '' }
		45	next if name == self.name
		46	entries << [name, link['href'].remove('#')]
		47	end
47	end	48	end
48		49
49	# get the type	50	def include_default_entry?
50	def get_type	51	at_css('#synopsis > ul > li')
51	n, t = get_name_and_type()
52	t
53	end	52	end
54	end	53	end
55	end	54	end

lib/docs/scrapers/haskell.rb+19−11

@@ -1,24 +1,32 @@
1	module Docs	1	module Docs
2	class Haskell < UrlScraper	2	class Haskell < UrlScraper
3	self.name = 'Haskell'	3	self.name = 'Haskell'
4	self.slug = 'haskell'
5	self.type = 'haskell'	4	self.type = 'haskell'
6	self.version = '7.8.2'	5	self.version = '7.8.2'
7	self.base_url = 'http://www.haskell.org/ghc/docs/7.8.2/html/libraries/'	6	self.base_url = 'http://www.haskell.org/ghc/docs/7.8.2/html/libraries/'
8	self.initial_paths = ['/index.html']	7	self.root_path = 'index.html'
9		8
10	html_filters.push 'haskell/entries'	9	html_filters.push 'haskell/entries', 'haskell/clean_html'
11	html_filters.push 'haskell/clean_html'
12	html_filters.push 'title'
13		10
		11	options[:container] = '#content'
14		12
15	options[:container] = '#content'	13	options[:skip] = %w(
16	options[:skip_patterns] = [/src/, /index/, /haskell2010/, /ghc-/, /Cabal-/] # skip source listings and index files	14	hoopl-3.10.0.1/Compiler-Hoopl-Internals.html
		15	base-4.7.0.0/Control-Exception-Base.html
		16	binary-0.7.1.0/Data-Binary-Get-Internal.html
		17	template-haskell-2.9.0.0/Language-Haskell-TH-Lib.html
		18	haskell98-2.0.0.3/Prelude.html
		19	pretty-1.1.1.1/Text-PrettyPrint.html
		20	base-4.7.0.0/Data-OldTypeable-Internal.html
		21	base-4.7.0.0/Data-Typeable-Internal.html
		22	base-4.7.0.0/GHC-IO-Encoding-Types.html
		23	unix-2.7.0.1/System-Posix-Process-Internals.html)
		24
		25	options[:skip_patterns] = [/src\//, /doc-index/, /haskell2010/, /ghc-/, /Cabal-/]
17		26
18	options[:attribution] = <<-HTML	27	options[:attribution] = <<-HTML
19	© The University Court of the University of Glasgow.<br>	28	© The University of Glasgow and others<br>
20	All rights reserved. <a href="http://www.haskell.org/ghc/license">See here for more info</a>	29	Licensed under a BSD-style license (see top of the page).
21	HTML	30	HTML
22		31	end
23	end
24	end	32	end

public/icons/docs/haskell/16.png+0−0

Binary file not shown.

public/icons/docs/haskell/16@2x.png+0−0

Binary file not shown.

public/icons/docs/haskell/SOURCE+1−0

@@ -0,0 +1 @@
		1	http://www.haskell.org/haskellwiki/Thompson-Wheeler_logo