History

Add a scraper for Scala 3

Nicolas Ettlin committed 4 years ago

Commit afe3a261 parent 7a9b9fe5 files · +348 −7Browse files

Unified · Split

5 files
  1. assets/javascripts/templates/pages/about_tmpl.coffee +2 −2
  2. assets/stylesheets/pages/_scala.scss +34 −0
  3. lib/docs/filters/scala/clean_html_v3.rb +181 −0
  4. lib/docs/filters/scala/entries_v3.rb +104 −0
  5. lib/docs/scrapers/scala.rb +27 −5
assets/javascripts/templates/pages/about_tmpl.coffee+2−2
@@ -794,9 +794,9 @@ credits = [ · 793 unmodified lines
794 'https://raw.githubusercontent.com/sass/sass/stable/MIT-LICENSE'794 'https://raw.githubusercontent.com/sass/sass/stable/MIT-LICENSE'
795 ], [795 ], [
796 'Scala',796 'Scala',
797 '2002-2019 EPFL, with contributions from Lightbend',797 '2002-2022 EPFL, with contributions from Lightbend',
798 'Apache',798 'Apache',
799 'https://raw.githubusercontent.com/scala/scala-lang/master/license.md'799 'https://www.scala-lang.org/license/'
800 ], [800 ], [
801 'scikit-image',801 'scikit-image',
802 '2019 the scikit-image team',802 '2019 the scikit-image team',
assets/stylesheets/pages/_scala.scss+34−0
@@ -1,4 +1,38 @@
1._scala {1._scala {
2 @extend %simple;2 @extend %simple;
3
3 .deprecated { @extend %label-red; }4 .deprecated { @extend %label-red; }
5
6 .related-types {
7 @extend %pre;
8 margin-top: 0;
9 white-space: normal;
10 }
11
12 .links {
13 @extend %box;
14 margin-left: -1rem;
15 text-align: center;
16 padding: .5em;
17
18 a { padding: .4em }
19
20 @include print {
21 display: none;
22 }
23 }
24
25 .source-link {
26 float: right;
27 font-size: .75rem;
28 color: var(--linkColor);
29 cursor: pointer;
30 @extend %user-select-none;
31
32 &:hover { text-decoration: underline; }
33
34 @include print {
35 display: none;
36 }
37 }
4}38}
lib/docs/filters/scala/clean_html_v3.rb+181−0
@@ -0,0 +1,181 @@
1# frozen_string_literal: true
2
3module Docs
4 class Scala
5 class CleanHtmlV3Filter < Filter
6 def call
7 # Remove unneeded elements
8 css('.documentableFilter, .documentableAnchor, .documentableBrief').remove
9
10 format_title
11 format_top_links
12 format_metadata
13 format_members
14
15 # Simplify the HTML structure
16 @doc = at_css('#content > div')
17 css('.documentableList > *').each do |element|
18 element.parent = doc
19 end
20 at_css('.membersList').remove
21
22 doc
23 end
24
25 def format_title
26 # Add the kind of page to the title
27 cover_header = at_css('.cover-header')
28 unless cover_header.nil?
29 icon = cover_header.at_css('.micon')
30 types = {
31 cl: 'Class',
32 ob: 'Object',
33 tr: 'Trait',
34 en: 'Enum',
35 ty: 'Type',
36 pa: 'Package',
37 }
38 type_id = cover_header.at_css('.micon')['class']
39 type_id.remove!('micon ')
40 type_id.remove!('-wc')
41 type = types[type_id.to_sym]
42 name = CGI.escapeHTML cover_header.at_css('h1').text
43
44 package = at_css('.breadcrumbs a:nth-of-type(3)').text
45 package = package + '.' unless name.empty? || package.empty?
46
47 title = root_page? ? 'Package root' : "#{type} #{package}#{name}".strip
48 cover_header.replace "<h1>#{title}</h1>"
49 end
50
51 # Signature
52 signature = at_css('.signature')
53 signature_annotations = signature.at_css('.annotations')
54 signature_annotations.name = 'small' unless signature_annotations.nil?
55 signature.replace "<h2 id=\"signature\">#{signature.inner_html}</h2>"
56 end
57
58 def format_top_links
59 # Companion page
60 links = []
61 at_css('.attributes').css('dt').each do |dt|
62 next if dt.content.strip != 'Companion:'
63 dd = dt.next_sibling
64
65 companion_link = dd.at_css('a')
66 companion_link.content = "Companion #{companion_link.content}"
67 links.append(companion_link.to_html)
68
69 dt.remove
70 dd.remove
71 end
72
73 # Source code
74 at_css('.attributes').css('dt').each do |dt|
75 next if dt.content.strip != 'Source:'
76 dd = dt.next_sibling
77
78 source_link = dd.at_css('a')
79 source_link.content = 'Source code'
80 links.append(source_link.to_html)
81
82 dt.remove
83 dd.remove
84 end
85
86 # Format the links
87 title = at_css('h1')
88 title.add_next_sibling("<div class=\"links\">#{links.join(' • ')}</div>")
89 end
90
91 def format_metadata
92 # Metadata (attributes)
93 css('.tabs.single .monospace').each do |node|
94 node['class'] = 'related-types'
95
96 if node.children.count > 15
97 node.replace "<details>
98 <summary>#{node.children.count} types</summary>
99 #{node.to_html}
100 </details>"
101 end
102 end
103
104 attributes = at_css('.attributes')
105 attributes.add_previous_sibling('<h3>Metadata</h3>')
106
107 tabs_names = css('.tabs.single .names .tab')
108 tabs_contents = css('.tabs.single .contents .tab')
109 tabs_names.zip(tabs_contents).each do |name, contents|
110 next if name.content == "Graph"
111
112 attributes.add_child("<dt>#{name.content}</dt>")
113 attributes.add_child("<dd>#{contents.inner_html.strip}</dd>")
114 end
115 at_css('.tabs').remove
116 end
117
118 def format_members
119 # Headings
120 css('.cover h2').each do |node|
121 node.name = 'h3'
122 end
123 css('h2:not(#signature)').remove
124 css(
125 '.membersList h3',
126
127 # Custom group headers for which Scaladoc generates invalid HTML
128 '.documentableList > h3:empty + p'
129 ).each do |node|
130 node.name = 'h2'
131 node.content = node.content
132 end
133
134 # Methods
135 css('.documentableElement').each do |element|
136 header = element.at_css('.header')
137 header.name = 'h3'
138
139 id = element['id']
140 element.remove_attribute('id')
141 header['id'] = id
142
143 annotations = element.at_css('.annotations')
144 annotations.name = 'small'
145 header.prepend_child(annotations)
146
147 # View source
148 element.css('dt').each do |dt|
149 next if dt.content.strip != 'Source:'
150 dd = dt.next_sibling
151
152 source_link = dd.at_css('a')
153 source_link.content = 'Source'
154 source_link['class'] = 'source-link'
155 header.prepend_child(source_link)
156
157 dt.remove
158 dd.remove
159 end
160
161 # Remove the unnecessary wrapper element
162 element.replace(element.inner_html)
163 end
164
165 # Remove deprecated sections
166 css('.documentableList').each do |list|
167 header = list.at_css('.groupHeader')
168 list.remove if (header.text.downcase.include? 'deprecate' rescue false)
169 end
170
171 # Code blocks
172 css('pre > code').each do |code|
173 pre = code.parent
174 pre['data-language'] = 'scala'
175 pre.inner_html = code.inner_html
176 end
177 end
178
179 end
180 end
181end
lib/docs/filters/scala/entries_v3.rb+104−0
@@ -0,0 +1,104 @@
1# frozen_string_literal: true
2
3module Docs
4 class Scala
5 class EntriesV3Filter < Docs::EntriesFilter
6 REPLACEMENTS = {
7 '$eq' => '=',
8 '$colon' => ':',
9 '$less' => '<',
10 }
11
12 def get_name
13 if is_package?
14 at_css('.cover-header h1').text
15 else
16 name = slug.split('/').last
17
18 # Some objects have inner objects, show ParentObject$.ChildObject$ instead of ParentObject$$ChildObject$
19 name = name.gsub('$$', '$.')
20
21 REPLACEMENTS.each do |key, value|
22 name = name.gsub(key, value)
23 end
24
25 # If a dollar sign is used as separator between two characters, replace it with a dot
26 name.gsub(/([^$.])\$([^$.])/, '\1.\2')
27 end
28 end
29
30 def get_type
31 # if this entry is for a package, we group the package under the parent package
32 if is_package?
33 parent_package
34 # otherwise, group it under the regular package name
35 else
36 package_name
37 end
38 end
39
40 def include_default_entry?
41 true
42 end
43
44 def additional_entries
45 entries = []
46 titles = []
47
48 css(".documentableElement").each do |node|
49 # Ignore elements without IDs
50 id = node['id']
51 next if id.nil?
52
53 # Ignore deprecated and inherited members
54 next unless node.at_css('.deprecated').nil?
55
56 member_name = node.at_css('.documentableName').content
57 title = "#{name}.#{member_name}"
58
59 # Add () to methods that take parameters, i.e. methods who have (…)
60 # in their signature, ignoring occurrences of (implicit …) and (using …)
61 signature = node.at_css('.signature').content
62 title += '()' if signature =~ /\((?!implicit)(?!using ).*\)/
63
64 next if titles.include?(title) # Ignore duplicates (function overloading)
65
66 entries << [title, id]
67 titles.push(title)
68 end
69
70 entries
71 end
72
73 private
74
75 # For the package name, we use the slug rather than parsing the package
76 # name from the HTML because companion object classes may be broken out into
77 # their own entries (by the source documentation). When that happens,
78 # we want to group these classes (like `scala.reflect.api.Annotations.Annotation`)
79 # under the package name, and not the fully-qualfied name which would
80 # include the companion object.
81 def package_name
82 name = package_drop_last(slug_parts)
83 name.empty? ? 'scala' : name
84 end
85
86 def parent_package
87 parent = package_drop_last(package_name.split('.'))
88 parent.empty? ? 'scala' : parent
89 end
90
91 def package_drop_last(parts)
92 parts[0...-1].join('.')
93 end
94
95 def slug_parts
96 slug.split('/')
97 end
98
99 def is_package?
100 !at_css('.cover-header .micon.pa').nil?
101 end
102 end
103 end
104end
lib/docs/scrapers/scala.rb+27−5
@@ -3,22 +3,41 @@ module Docs · 2 unmodified lines
3 self.name = 'Scala'3 self.name = 'Scala'
4 self.type = 'scala'4 self.type = 'scala'
5 self.links = {5 self.links = {
6 home: 'http://www.scala-lang.org/',6 home: 'https://www.scala-lang.org/',
7 code: 'https://github.com/scala/scala'7 code: 'https://github.com/scala/scala'
8 }8 }
9 9
10 options[:container] = '#content-container'
11 options[:attribution] = <<-HTML10 options[:attribution] = <<-HTML
12 &copy; 2002-2019 EPFL, with contributions from Lightbend.<br>11 &copy; 2002-2022 EPFL, with contributions from Lightbend.<br>
13 Licensed under the Apache License, Version 2.0.12 Licensed under the Apache License, Version 2.0.
14 HTML13 HTML
15 14
15 # For Scala 3, there is no official download link for the documentation
16 # (see https://contributors.scala-lang.org/t/5537).
17 #
18 # We currently need to build the docs ourselves. To do so:
19 # 1. Make sure that Scala 3 and sbt are installed
20 # (https://www.scala-lang.org/download/scala3.html)
21 # 2. Clone the Scala 3 (Dotty) repository (https://github.com/lampepfl/dotty)
22 # 3. From the Dotty folder, run this command in the terminal:
23 # $ sbt scaladoc/generateScalaDocumentation
24 # 4. Extract scaladoc/output/scala3/api/ into docs/scala~3.1
25 version '3.1' do
26 self.release = '3.1.1'
27 self.base_url = 'https://scala-lang.org/api/3.1.1/'
28 self.root_path = 'index.html'
29 # options[:container] = '#main-content'
30
31 html_filters.push 'scala/entries_v3', 'scala/clean_html_v3'
32 end
33
16 # https://downloads.lightbend.com/scala/2.13.0/scala-docs-2.13.0.zip34 # https://downloads.lightbend.com/scala/2.13.0/scala-docs-2.13.0.zip
17 # Extract api/scala-library into docs/scala~2.13_library35 # Extract api/scala-library into docs/scala~2.13_library
18 version '2.13 Library' do36 version '2.13 Library' do
19 self.release = '2.13.0'37 self.release = '2.13.0'
20 self.base_url = 'https://www.scala-lang.org/api/2.13.0/'38 self.base_url = 'https://www.scala-lang.org/api/2.13.0/'
21 self.root_path = 'index.html'39 self.root_path = 'index.html'
40 options[:container] = '#content-container'
22 41
23 html_filters.push 'scala/entries_v2', 'scala/clean_html_v2'42 html_filters.push 'scala/entries_v2', 'scala/clean_html_v2'
24 end43 end
@@ -29,6 +48,7 @@ module Docs · 4 unmodified lines
29 self.release = '2.13.0'48 self.release = '2.13.0'
30 self.base_url = 'https://www.scala-lang.org/api/2.13.0/scala-reflect/'49 self.base_url = 'https://www.scala-lang.org/api/2.13.0/scala-reflect/'
31 self.root_path = 'index.html'50 self.root_path = 'index.html'
51 options[:container] = '#content-container'
32 52
33 html_filters.push 'scala/entries_v2', 'scala/clean_html_v2'53 html_filters.push 'scala/entries_v2', 'scala/clean_html_v2'
34 end54 end
@@ -39,6 +59,7 @@ module Docs · 4 unmodified lines
39 self.release = '2.12.9'59 self.release = '2.12.9'
40 self.base_url = 'https://www.scala-lang.org/api/2.12.9/'60 self.base_url = 'https://www.scala-lang.org/api/2.12.9/'
41 self.root_path = 'index.html'61 self.root_path = 'index.html'
62 options[:container] = '#content-container'
42 63
43 html_filters.push 'scala/entries_v2', 'scala/clean_html_v2'64 html_filters.push 'scala/entries_v2', 'scala/clean_html_v2'
44 end65 end
@@ -49,13 +70,14 @@ module Docs · 4 unmodified lines
49 self.release = '2.12.9'70 self.release = '2.12.9'
50 self.base_url = 'https://www.scala-lang.org/api/2.12.9/scala-reflect/'71 self.base_url = 'https://www.scala-lang.org/api/2.12.9/scala-reflect/'
51 self.root_path = 'index.html'72 self.root_path = 'index.html'
73 options[:container] = '#content-container'
52 74
53 html_filters.push 'scala/entries_v2', 'scala/clean_html_v2'75 html_filters.push 'scala/entries_v2', 'scala/clean_html_v2'
54 end76 end
55 77
56 def get_latest_version(opts)78 def get_latest_version(opts)
57 doc = fetch_doc('https://www.scala-lang.org/api/current/', opts)79 doc = fetch_doc('https://www.scala-lang.org/api/3.x/', opts)
58 doc.at_css('#doc-version').content80 doc.at_css('.projectVersion').content
59 end81 end
60 end82 end
61end83end