History

Update Haskell scraper

Thibaut committed 12 years ago

Commit 1819d711 parent 9a43dfb13 files · +116 −293Browse files

Unified · Split

13 files
  1. assets/images/icons.png +0 −0
  2. assets/images/icons@2x.png +0 −0
  3. assets/javascripts/templates/pages/about_tmpl.coffee +5 −0
  4. assets/javascripts/templates/pages/news_tmpl.coffee +4 −1
  5. assets/stylesheets/application.css.scss +1 −0
  6. assets/stylesheets/global/_icons.scss +1 −0
  7. assets/stylesheets/pages/_haskell.scss +18 −114
  8. lib/docs/filters/haskell/clean_html.rb +31 −130
  9. lib/docs/filters/haskell/entries.rb +40 −41
  10. lib/docs/scrapers/haskell.rb +19 −11
  11. public/icons/docs/haskell/16.png +0 −0
  12. public/icons/docs/haskell/16@2x.png +0 −0
  13. public/icons/docs/haskell/SOURCE +1 −0
assets/images/icons.png+0−0

Binary file not shown.

assets/images/icons@2x.png+0−0

Binary file not shown.

assets/javascripts/templates/pages/about_tmpl.coffee+5−0
@@ -130,6 +130,11 @@ credits = [ · 129 unmodified lines
130130 '2014 Grunt Team',
131131 'MIT',
132132 'https://raw.githubusercontent.com/gruntjs/gruntjs.com/master/LICENSE'
133+ ], [
134+ 'Haskell',
135+ 'The University of Glasgow',
136+ 'BSD',
137+ 'http://www.haskell.org/ghc/license'
133138 ], [
134139 'HTTP',
135140 '1999 The Internet Society',
assets/javascripts/templates/pages/news_tmpl.coffee+4−1
@@ -24,7 +24,10 @@ newsItem = (date, news) -> · 23 unmodified lines
2424 result
2525
2626 app.news = [
27− [ 1400976000000, # May 25, 2014
27+ [ 1402704000000, # June 14, 2014
28+ """ New <a href="/haskell/">Haskell</a> documentation """,
29+ ], [
30+ 1400976000000, # May 25, 2014
2831 """ New <a href="/laravel/">Laravel</a> documentation """,
2932 ], [
3033 1399161600000, # May 4, 2014
assets/stylesheets/application.css.scss+1−0
@@ -35,6 +35,7 @@ · 34 unmodified lines
3535 'pages/ember',
3636 'pages/express',
3737 'pages/go',
38+ 'pages/haskell',
3839 'pages/jquery',
3940 'pages/knockout',
4041 'pages/git',
assets/stylesheets/global/_icons.scss+1−0
@@ -57,3 +57,4 @@ · 56 unmodified lines
5757 ._icon-grunt:before { background-position: -3rem -8rem; }
5858 ._icon-maxcdn:before { background-position: -4rem -8rem; }
5959 ._icon-laravel:before { background-position: 0 -9rem; }
60+._icon-haskell:before { background-position: -1rem -9rem; }
assets/stylesheets/pages/_haskell.scss+18−114
@@ -1,121 +1,25 @@
1−._icon-haskell:before {
2− background-image: image-url('/icons/docs/haskell/16.png');
3− background-size: cover;
4− background-repeat: no-repeat;
5−}
6−
7−.empty-table .empty {
8− display: none;
9−}
10−
11−.arguments td.src {
12− background: #faf9e2;
13− width: 30%;
14−}
15−
16−th.src,
17−td.src {
18− font-family: $monoFont;
19− font-weight: normal;
20− font-style: normal;
21− background: #f8f8f8;
22−}
23−
24−caption {
25− font-weight: bold;
26− text-align: left;
27− font-style: italic;
28− font-size: 1.1em;
29−}
30−
31−// remove margin in descript listing
32−dd > pre {
33− @extend %pre;
34− margin: 0;
35− background: #faf9e2;
36− border-color: #dddaaa #dddaaa #d7d7a9;
37−}
38−
39−// warnings are red
40−.warning {
41− @extend %note;
42− @extend %note-red;
43−}
44−
45−
46−// complexity classes are blue boxes
47−.with-complexity {
48− display: flex;
49− display: -webkit-flex;
50−
51− justify-content: space-between;
52− -webkit-justify-content: space-between;
1+._haskell {
2+ > h2 { @extend %block-heading; }
3+ > h3 { @extend %block-label; }
4+ h4 { font-size: 1em; }
535
54− align-items: flex-start;
55− -webkit-align-items: flex-start;
6+ .module + .package, p.src > .link { float: right; }
567
57− align-content: stretch;
58− -webkit-align-content: stretch;
59−}
60−
61−.complexity {
62− @extend %note;
63− @extend %note-blue;
64− margin: 0;
65− margin-left: 1em;
66− margin-bottom: 0.75em;
67− font-style: italic;
68− white-space: nowrap;
69−
70− flex-shrink: 0;
71− -webkit-flex-shrink: 0;
72−
73− order: 2;
74− -webkit-order: 2;
75−}
76−
77−.complexity + span {
78− order: 1;
79− -webkit-order: 1;
80−}
81−
82−// add box type to "since: ..."
83−.added {
84− @extend %note;
85− @extend %note-gold;
86−}
87−
88−.added-cell {
89− @extend %note-gold;
90−}
91−
92−.fields h3 {
93− display: none;
94−}
8+ .src {
9+ white-space: normal;
10+ @extend %code;
11+ }
12+ p.src { @extend %block-label, %label-blue; }
13+ dt.src { white-space: normal; }
9514
96−// separate types more
97−.src {
98− margin-top: 2.5em;
99−}
15+ .top > .subs { margin-left: 2em; }
16+ .subs p.src { margin-top: 1em; }
10017
101−h1 + .top .src,
102−h2 + .top .src,
103−h3 + .top .src,
104−.caption + .top .src {
105− margin-top: 0;
106−}
18+ dt > code, .complexity, .version { @extend %label; }
19+ .complexity, .version { @extend %label-green; }
10720
108−// but not for first type
109−h1 + .top,
110−h2 + .top,
111−h3 + .top,
112−h4 + .top {
113− margin-top: 0;
114−}
21+ table { margin: 1em 0; }
22+ td > pre { margin: 0; }
11523
116−// change color of example code
117−.example {
118− border: 1px solid;
119− background: #faf9e2;
120− border-color: #dddaaa #dddaaa #d7d7a9;
24+ .warning { @extend %note; }
12125 }
lib/docs/filters/haskell/clean_html.rb+31−130
@@ -2,146 +2,54 @@ module Docs · 1 unmodified line
22 class Haskell
33 class CleanHtmlFilter < Filter
44 def call
5+ root_page? ? root : other
6+ doc
7+ end
58
6− # remove unwanted elements
7− css('#footer', '#package-header', '#module-header', '#synopsis', '.link', '#table-of-contents', '.package').remove
8−
9− # cpations in tables are h3
10− css('table .caption').each do |node|
11− node.name = 'h3'
12− end
13−
14− # turn captions into real headers
15− css('.caption').each do |node|
16− node.name = 'h1'
17− end
18−
19− # section
20− css('.top > .caption').each do |node|
21− node.name = 'h2'
22− end
23−
24− # subsections
25− css('.top > .subs > .caption', '.fields > .caption').each do |node|
26− node.name = 'h3'
27− end
28−
29− # subsubsections
30− css('.top > .subs > .subs > .caption').each do |node|
31− node.name = 'h4'
32− end
33−
34− # ...
35− css('.top > .subs > .subs > .subs > .caption').each do |node|
36− node.name = 'h5'
37− end
38−
39− # ......
40− css('.top > .subs > .subs > .subs > .subs > .caption').each do |node|
41− node.name = 'h6'
42− end
43−
44− # all pre's are examples
45− css('pre').each do |node|
46− node.add_css_class('example')
9+ def root
10+ css('#description', '#module-list').each do |node|
11+ node.before(node.children).remove
4712 end
13+ end
4814
49− # turn source listing in to pre
50− css('.src').each do |node|
51− if node.name != "td"
52− node.name = 'pre'
53− end
15+ def other
16+ css('h1').each do |node|
17+ node.remove if node.content == 'Documentation'
5418 end
5519
56− # check if second column of table is totally empty.
57− # and remove it if it is
58− css('table').each do |table|
59− empty = true
60− table.css('td + td').each do |snd|
61− empty = empty && snd['class'] =~ /empty/
62− end
63− if empty
64− # remove empty column
65− table.css('td + td').remove
66− end
20+ css('h1, h2, h3, h4').each do |node|
21+ node.name = node.name.sub(/\d/) { |i| i.to_i + 1 }
6722 end
6823
69− # move table captions into the tables
70− css(".caption + table").each do |table|
71− caption = table.previous
72− caption.name = "caption"
73− caption.parent = table
24+ at_css('#module-header').tap do |node|
25+ heading = at_css('.caption')
26+ heading.name = 'h1'
27+ node.before(heading)
28+ node.before(node.children).remove
7429 end
7530
76− css(".caption + .show table").each do |table|
77− caption = table.parent.parent.css('.caption')[0]
78− caption.name = 'caption'
79− caption.parent = table
80− end
31+ css('#synopsis').remove
8132
82− # better arguments display:
83− css('.src + .arguments table').each do |table|
84− src = table.parent.previous # the function name
85− row = doc.document.create_element('tr')
86− table.css('tr')[0].before(row)
87− src.parent = row
88− src.name = "th"
89− src['colspan'] = 2
33+ css('#interface', 'h2 code').each do |node|
34+ node.before(node.children).remove
9035 end
9136
92− # remove root page title
93− if root_page?
94− at_css('h1').remove
37+ css('a[name]').each do |node|
38+ node['id'] = node['name']
39+ node.remove_attribute('name')
9540 end
9641
97− # add id to links (based on name)
98− css('a').each do |node|
99− if node['name']
100− node['id'] = node['name']
101− end
102− end
103−
104− # make code in description into proper pre
105− css('dd > code').each do |node|
106− node.name = 'pre'
42+ css('p.caption').each do |node|
43+ node.name = 'h4'
10744 end
10845
109− # add some informational boxes
11046 css('em').each do |node|
111− if node.content == 'Deprecated.'
112− # Make deprecated messages red.
113− node.parent.add_css_class('warning')
114− elsif node.content =~ /O\(.*\)/
115− # this is big_O notation, but only apply the class if this is not
116− # inside running text (it must be at the start of a paragraph)
117− # from:
118− # <p><em>O(n)</em>. Koel ok</p>
119− # to:
120− # <p class="with-complexity">
121− # <span class="complexity">O(n)</span>
122− # <span>Koel ok</span>
123− # </p>
124− if node.previous == nil
125− node.add_css_class('complexity') # add css class
126− node.name="span" # just make it div
127− node.next.content = node.next.content.gsub(/^. /, "") # remove . if directly after em
128− node.content = node.content.gsub(/\.$/, "") # remove trailing . if it's inside em
129−
130− # reparent the nodes
131− cont = doc.document.create_element "p", :class => "with-complexity"
132− node.parent.previous = cont
133− par = node.parent
134− node.parent = cont
135− par.parent = cont
136− par.name = "span"
137− end
138− elsif node.content =~ /Since: .*/
139− # add box to 'Since:' annotations
140− if node.parent.parent.name == "td"
141− node.parent.parent.add_css_class('added-cell')
142− else
143− node.add_css_class('added')
144− end
47+ if node.content.start_with?('O(')
48+ node.name = 'span'
49+ node['class'] = 'complexity'
50+ elsif node.content.start_with?('Since')
51+ node.name = 'span'
52+ node['class'] = 'version'
14553 end
14654 end
14755
@@ -150,10 +58,3 @@ module Docs · 2 unmodified lines
15058 end
15159 end
15260 end
153−
154−class Nokogiri::XML::Node
155− def add_css_class( *classes )
156− existing = (self['class'] || "").split(/\s+/)
157− self['class'] = existing.concat(classes).uniq.join(" ")
158− end
159−end
lib/docs/filters/haskell/entries.rb+40−41
@@ -1,55 +1,54 @@
11 module Docs
22 class Haskell
33 class EntriesFilter < Docs::EntriesFilter
4+ IGNORE_ENTRIES_PATHS = %w(
5+ bytestring-0.10.4.0/Data-ByteString-Lazy.html
6+ bytestring-0.10.4.0/Data-ByteString-Char8.html
7+ bytestring-0.10.4.0/Data-ByteString-Lazy-Char8.html
8+ array-0.5.0.0/Data-Array-IArray.html
9+ containers-0.5.5.1/Data-IntMap-Lazy.html
10+ containers-0.5.5.1/Data-Map-Lazy.html
11+ unix-2.7.0.1/System-Posix-Files-ByteString.html
12+ filepath-1.3.0.2/System-FilePath-Windows.html
13+ transformers-0.3.0.0/Control-Monad-Trans-RWS-Lazy.html
14+ transformers-0.3.0.0/Control-Monad-Trans-Writer-Lazy.html
15+ base-4.7.0.0/GHC-Conc-Sync.html
16+ base-4.7.0.0/GHC-IO-Encoding-UTF32.html
17+ unix-2.7.0.1/System-Posix-Terminal-ByteString.html)
418
5− # gets name and type in one fell swoop
6− #
7− # eg.
8− # Control.Monad > [Monad, Control]
9− # Control.Concurrent.Mvar > [Concurrent.MVar, Control]
10− # Array > [Array, nil]
11− def get_name_and_type
12− if at_css('h1') && at_css('h1').content == 'Haskell Hierarchical Libraries'
13− puts 'ok'
14− name = 'Haskell'
15− type = nil
16− else
17− # find full module identifier
18− caption = at_css('#module-header .caption')
19+ def get_name
20+ at_css('#module-header .caption').content.strip
21+ end
1922
20− if caption
21− # split the module path
22− parts = caption.content.split('.')
23+ def get_type
24+ %w(System.Posix System.Win32 Control.Monad).each do |type|
25+ return type if name.start_with?(type)
26+ end
2327
24− if parts.length > 1
25− # if more than one part then the
26− # first is the type and the rest is the name
27− type = parts[0]
28− name = parts.drop(1).join('.')
29− else
30− # if only one part, this is the name
31− name = parts[0]
32− type = nil
33− end
34− else
35− # no caption found -> no type / no name
36− name = 'no-name'
37− type = 'no-type'
38− end
28+ if name.start_with?('Data')
29+ name.split('.')[0..1].join('.')
30+ else
31+ name.split('.').first
3932 end
40− [name, type]
4133 end
4234
43− # get the name
44− def get_name
45− n, t = get_name_and_type()
46− n
35+ def additional_entries
36+ return [] if IGNORE_ENTRIES_PATHS.include?(subpath)
37+
38+ css('#synopsis > ul > li').each_with_object [] do |node, entries|
39+ link = node.at_css('a')
40+ next unless link['href'].start_with?('#')
41+ name = node.content.strip
42+ name.remove! %r{\A(?:module|data|newtype|class|type family m|type)\s+}
43+ name.sub! %r{\A\((.+?)\)}, '\1'
44+ name.sub!(/ (?:\:\: (\w+))?.+\z/) { |_| $1 ? " (#{$1})" : '' }
45+ next if name == self.name
46+ entries << [name, link['href'].remove('#')]
47+ end
4748 end
4849
49− # get the type
50− def get_type
51− n, t = get_name_and_type()
52− t
50+ def include_default_entry?
51+ at_css('#synopsis > ul > li')
5352 end
5453 end
5554 end
lib/docs/scrapers/haskell.rb+19−11
@@ -1,24 +1,32 @@
11 module Docs
22 class Haskell < UrlScraper
33 self.name = 'Haskell'
4− self.slug = 'haskell'
54 self.type = 'haskell'
65 self.version = '7.8.2'
76 self.base_url = 'http://www.haskell.org/ghc/docs/7.8.2/html/libraries/'
8− self.initial_paths = ['/index.html']
7+ self.root_path = 'index.html'
98
10− html_filters.push 'haskell/entries'
11− html_filters.push 'haskell/clean_html'
12− html_filters.push 'title'
9+ html_filters.push 'haskell/entries', 'haskell/clean_html'
1310
11+ options[:container] = '#content'
1412
15− options[:container] = '#content'
16− options[:skip_patterns] = [/src/, /index/, /haskell2010/, /ghc-/, /Cabal-/] # skip source listings and index files
13+ options[:skip] = %w(
14+ hoopl-3.10.0.1/Compiler-Hoopl-Internals.html
15+ base-4.7.0.0/Control-Exception-Base.html
16+ binary-0.7.1.0/Data-Binary-Get-Internal.html
17+ template-haskell-2.9.0.0/Language-Haskell-TH-Lib.html
18+ haskell98-2.0.0.3/Prelude.html
19+ pretty-1.1.1.1/Text-PrettyPrint.html
20+ base-4.7.0.0/Data-OldTypeable-Internal.html
21+ base-4.7.0.0/Data-Typeable-Internal.html
22+ base-4.7.0.0/GHC-IO-Encoding-Types.html
23+ unix-2.7.0.1/System-Posix-Process-Internals.html)
24+
25+ options[:skip_patterns] = [/src\//, /doc-index/, /haskell2010/, /ghc-/, /Cabal-/]
1726
1827 options[:attribution] = <<-HTML
19− &copy; The University Court of the University of Glasgow.<br>
20− All rights reserved. <a href="http://www.haskell.org/ghc/license">See here for more info</a>
28+ &copy; The University of Glasgow and others<br>
29+ Licensed under a BSD-style license (see top of the page).
2130 HTML
22−
23− end
31+ end
2432 end
public/icons/docs/haskell/16.png+0−0

Binary file not shown.

public/icons/docs/haskell/16@2x.png+0−0

Binary file not shown.

public/icons/docs/haskell/SOURCE+1−0
@@ -0,0 +1 @@
1+http://www.haskell.org/haskellwiki/Thompson-Wheeler_logo