History

Merge pull request #1708 from Nicolapps/scala-3-scraper

Add a scraper for Scala 3

Simon Legner committed 4 years ago

Commit 39d36962 parents 2584ecc 545806a8 files · +493 −17Browse files

Unified · Split

8 files
  1. assets/javascripts/templates/pages/about_tmpl.coffee +2 −2
  2. assets/javascripts/vendor/prism.js +52 −2
  3. assets/stylesheets/pages/_scala.scss +39 −0
  4. lib/docs/filters/scala/clean_html_v2.rb +1 −1
  5. lib/docs/filters/scala/clean_html_v3.rb +253 −0
  6. lib/docs/filters/scala/entries_v2.rb +3 −3
  7. lib/docs/filters/scala/entries_v3.rb +105 −0
  8. lib/docs/scrapers/scala.rb +38 −9
assets/javascripts/templates/pages/about_tmpl.coffee+2−2
@@ -794,9 +794,9 @@ credits = [ · 793 unmodified lines
794 'https://raw.githubusercontent.com/sass/sass/stable/MIT-LICENSE'794 'https://raw.githubusercontent.com/sass/sass/stable/MIT-LICENSE'
795 ], [795 ], [
796 'Scala',796 'Scala',
797 '2002-2019 EPFL, with contributions from Lightbend',797 '2002-2022 EPFL, with contributions from Lightbend',
798 'Apache',798 'Apache',
799 'https://raw.githubusercontent.com/scala/scala-lang/master/license.md'799 'https://www.scala-lang.org/license/'
800 ], [800 ], [
801 'scikit-image',801 'scikit-image',
802 '2019 the scikit-image team',802 '2019 the scikit-image team',
assets/javascripts/vendor/prism.js+52−2
@@ -1,5 +1,5 @@
1/* PrismJS 1.26.01/* PrismJS 1.27.0
2https://prismjs.com/download.html#themes=prism&languages=markup+css+clike+javascript+bash+c+cpp+cmake+coffeescript+crystal+d+dart+diff+django+elixir+erlang+go+groovy+java+json+julia+kotlin+latex+lua+markup-templating+matlab+nginx+nim+ocaml+perl+php+python+qml+r+jsx+ruby+rust+scss+shell-session+sql+typescript+yaml+zig */2https://prismjs.com/download.html#themes=prism&languages=markup+css+clike+javascript+bash+c+cpp+cmake+coffeescript+crystal+d+dart+diff+django+elixir+erlang+go+groovy+java+json+julia+kotlin+latex+lua+markup-templating+matlab+nginx+nim+ocaml+perl+php+python+qml+r+jsx+ruby+rust+scss+scala+shell-session+sql+typescript+yaml+zig */
3/// <reference lib="WebWorker"/>3/// <reference lib="WebWorker"/>
4 4
5var _self = (typeof window !== 'undefined')5var _self = (typeof window !== 'undefined')
@@ -4660,6 +4660,56 @@ Prism.languages.insertBefore('scss', 'function', { · 4654 unmodified lines
4660 4660
4661Prism.languages.scss['atrule'].inside.rest = Prism.languages.scss;4661Prism.languages.scss['atrule'].inside.rest = Prism.languages.scss;
4662 4662
4663Prism.languages.scala = Prism.languages.extend('java', {
4664 'triple-quoted-string': {
4665 pattern: /"""[\s\S]*?"""/,
4666 greedy: true,
4667 alias: 'string'
4668 },
4669 'string': {
4670 pattern: /("|')(?:\\.|(?!\1)[^\\\r\n])*\1/,
4671 greedy: true
4672 },
4673 'keyword': /<-|=>|\b(?:abstract|case|catch|class|def|do|else|extends|final|finally|for|forSome|if|implicit|import|lazy|match|new|null|object|override|package|private|protected|return|sealed|self|super|this|throw|trait|try|type|val|var|while|with|yield)\b/,
4674 'number': /\b0x(?:[\da-f]*\.)?[\da-f]+|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:e\d+)?[dfl]?/i,
4675 'builtin': /\b(?:Any|AnyRef|AnyVal|Boolean|Byte|Char|Double|Float|Int|Long|Nothing|Short|String|Unit)\b/,
4676 'symbol': /'[^\d\s\\]\w*/
4677});
4678
4679Prism.languages.insertBefore('scala', 'triple-quoted-string', {
4680 'string-interpolation': {
4681 pattern: /\b[a-z]\w*(?:"""(?:[^$]|\$(?:[^{]|\{(?:[^{}]|\{[^{}]*\})*\}))*?"""|"(?:[^$"\r\n]|\$(?:[^{]|\{(?:[^{}]|\{[^{}]*\})*\}))*")/i,
4682 greedy: true,
4683 inside: {
4684 'id': {
4685 pattern: /^\w+/,
4686 greedy: true,
4687 alias: 'function'
4688 },
4689 'escape': {
4690 pattern: /\\\$"|\$[$"]/,
4691 greedy: true,
4692 alias: 'symbol'
4693 },
4694 'interpolation': {
4695 pattern: /\$(?:\w+|\{(?:[^{}]|\{[^{}]*\})*\})/,
4696 greedy: true,
4697 inside: {
4698 'punctuation': /^\$\{?|\}$/,
4699 'expression': {
4700 pattern: /[\s\S]+/,
4701 inside: Prism.languages.scala
4702 }
4703 }
4704 },
4705 'string': /[\s\S]+/
4706 }
4707 }
4708});
4709
4710delete Prism.languages.scala['class-name'];
4711delete Prism.languages.scala['function'];
4712
4663(function (Prism) {4713(function (Prism) {
4664 4714
4665 // CAREFUL!4715 // CAREFUL!
assets/stylesheets/pages/_scala.scss+39−0
@@ -1,4 +1,43 @@
1._scala {1._scala {
2 @extend %simple;2 @extend %simple;
3
3 .deprecated { @extend %label-red; }4 .deprecated { @extend %label-red; }
5
6 .attributes dl,
7 .attributes pre {
8 margin: 0;
9 }
10
11 .related-types {
12 @extend %pre;
13 margin: 0;
14 white-space: normal;
15 }
16
17 .links {
18 @extend %box;
19 margin-left: -1rem;
20 text-align: center;
21 padding: .5em;
22
23 a { padding: .4em }
24
25 @include print {
26 display: none;
27 }
28 }
29
30 .source-link {
31 float: right;
32 font-size: .75rem;
33 color: var(--linkColor);
34 cursor: pointer;
35 @extend %user-select-none;
36
37 &:hover { text-decoration: underline; }
38
39 @include print {
40 display: none;
41 }
42 }
4}43}
lib/docs/filters/scala/clean_html_v2.rbrenamed from lib/docs/filters/scala/clean_html.rb+1−1
@@ -1,6 +1,6 @@
1module Docs1module Docs
2 class Scala2 class Scala
3 class CleanHtmlFilter < Filter3 class CleanHtmlV2Filter < Filter
4 def call4 def call
5 @doc = at_css('#content')5 @doc = at_css('#content')
6 6
lib/docs/filters/scala/clean_html_v3.rb+253−0
@@ -0,0 +1,253 @@
1# frozen_string_literal: true
2
3module Docs
4 class Scala
5 class CleanHtmlV3Filter < Filter
6 def call
7 # Remove unneeded elements
8 css('.documentableFilter, .documentableAnchor, .documentableBrief').remove
9
10 format_title
11 format_signature
12 format_top_links
13 format_metadata
14
15 # Remove the redundant long descriptions on the main page
16 if slug == 'index'
17 css('.contents').remove
18 else
19 format_members
20 end
21
22 simplify_html
23
24 doc
25 end
26
27 private
28
29 # Formats the title of the page
30 def format_title
31 cover_header = at_css('.cover-header')
32 return if cover_header.nil?
33
34 # Add the kind of page to the title
35 icon = cover_header.at_css('.micon')
36 types = {
37 cl: 'Class',
38 ob: 'Object',
39 tr: 'Trait',
40 en: 'Enum',
41 ty: 'Type',
42 pa: 'Package',
43 }
44 type_id = cover_header.at_css('.micon')['class']
45 type_id.remove!('micon ')
46 type_id.remove!('-wc')
47 type = types[type_id.to_sym]
48 name = CGI.escapeHTML cover_header.at_css('h1').text
49
50 # Add the package name
51 package = at_css('.breadcrumbs a:nth-of-type(3)').text
52 package = package + '.' unless name.empty? || package.empty?
53
54 # Replace the title
55 title = root_page? ? 'Package root' : "#{type} #{package}#{name}".strip
56 cover_header.replace "<h1>#{title}</h1>"
57 end
58
59 # Formats the signature block at the top of the page
60 def format_signature
61 signature = at_css('.signature')
62 signature_annotations = signature.at_css('.annotations')
63 signature_annotations.name = 'small' unless signature_annotations.nil?
64 signature.replace "<h2 id=\"signature\">#{signature.inner_html}</h2>"
65 end
66
67 # Formats the top links (companion page, source code)
68 def format_top_links
69 # Companion page (e.g. List ↔ List$)
70 links = []
71 at_css('.attributes').css('dt').each do |dt|
72 next if dt.content.strip != 'Companion:'
73 dd = dt.next_sibling
74
75 companion_link = dd.at_css('a')
76 companion_link.content = "Companion #{companion_link.content}"
77 links.append(companion_link.to_html)
78
79 dt.remove
80 dd.remove
81 end
82
83 # Source code
84 at_css('.attributes').css('dt').each do |dt|
85 next if dt.content.strip != 'Source:'
86 dd = dt.next_sibling
87
88 source_link = dd.at_css('a')
89 source_link.content = 'Source code'
90 links.append(source_link.to_html)
91
92 dt.remove
93 dd.remove
94 end
95
96 # Format the links
97 title = at_css('h1')
98 title.add_next_sibling("<div class=\"links\">#{links.join(' • ')}</div>")
99 end
100
101 # Metadata about the whole file (e.g. supertypes)
102 def format_metadata
103 # Format the values
104 css('.tabs.single .monospace').each do |node|
105 node.css('> div').each do |div|
106 div['class'] = 'member'
107 end
108
109 node['class'] = 'related-types'
110
111 if node.children.count > 15 # Hide too large lists
112 node.replace "<details>
113 <summary>#{node.children.count} types</summary>
114 #{node.to_html}
115 </details>"
116 end
117 end
118
119 attributes = at_css('.attributes')
120
121 # Change the HTML structure
122 tabs_names = css('.tabs.single .names .tab')
123 tabs_contents = css('.tabs.single .contents .tab')
124 tabs_names.zip(tabs_contents).each do |name, contents|
125 next if name.content == "Graph"
126
127 attributes.add_child("<dt>#{name.content}</dt>")
128 attributes.add_child("<dd>#{contents.inner_html.strip}</dd>")
129 end
130
131 convert_dl_to_table(attributes)
132
133 tabs = at_css('.tabs')
134 tabs.remove unless tabs.nil? || tabs.parent['class'] == 'membersList'
135 end
136
137 # Format the members (methods, values…)
138 def format_members
139 # Section headings
140 css('.cover h2').each do |node|
141 node.name = 'h3'
142 end
143 css('h2:not(#signature)').remove
144 css(
145 '.membersList h3',
146
147 # Custom group headers for which Scaladoc generates invalid HTML
148 # (<h3><p>…</p></h3>)
149 '.documentableList > h3:empty + p'
150 ).each do |node|
151 node.name = 'h2'
152 node.content = node.content
153 end
154
155 # Individual members
156 css('.documentableElement').each do |element|
157 header = element.at_css('.header')
158 header.name = 'h3'
159
160 id = element['id']
161 element.remove_attribute('id')
162 header['id'] = id unless id.nil?
163
164 annotations = element.at_css('.annotations')
165 annotations.name = 'small'
166 header.prepend_child(annotations)
167
168 # View source
169 element.css('dt').each do |dt|
170 next if dt.content.strip != 'Source:'
171 dd = dt.next_sibling
172
173 source_link = dd.at_css('a')
174 source_link.content = 'Source'
175 source_link['class'] = 'source-link'
176 header.prepend_child(source_link)
177
178 dt.remove
179 dd.remove
180 end
181
182 # Format attributes as a table
183 dl = element.at_css('.attributes')
184 convert_dl_to_table(dl) unless dl.nil?
185
186 # Remove the unnecessary wrapper element
187 element.replace(element.inner_html)
188 end
189
190 # Remove deprecated sections
191 css('.documentableList').each do |list|
192 header = list.at_css('.groupHeader')
193 list.remove if (header.text.downcase.include? 'deprecate' rescue false)
194 end
195
196 # Code blocks
197 css('pre > code').each do |code|
198 pre = code.parent
199 pre['data-language'] = 'scala'
200 pre.inner_html = code.inner_html
201 end
202 end
203
204 # Simplify the HTML structure by removing useless elements
205 def simplify_html
206 # Remove unneeded parts of the document
207 @doc = at_css('#content > div')
208
209 # Remove the useless elements around members
210 css('.documentableList > *').each do |element|
211 element.parent = doc
212 end
213 at_css('.membersList').remove
214
215 # Remove useless classes
216 css('.header, .groupHeader, .cover, .documentableName').each do |element|
217 element.remove_attribute('class')
218 end
219
220 # Remove useless attributes
221 css('[t]').each do |element|
222 element.remove_attribute('t')
223 end
224
225 # Remove useless wrapper elements
226 css('.docs, .doc, .memberDocumentation, span, div:not([class])').each do |element|
227 element.replace(element.children)
228 end
229 end
230
231 def convert_dl_to_table(dl)
232 table = Nokogiri::XML::Node.new('table', doc)
233 table['class'] = 'attributes'
234
235 dl.css('> dt').each do |dt|
236 dd = dt.next_element
237 has_dd = dd.name == 'dd' rescue false
238
239 tr = Nokogiri::XML::Node.new('tr', doc)
240 colspan = has_dd ? '' : ' colspan="2"' # handle <dt> without following <dt>
241 tr.add_child("<th#{colspan}>#{dt.inner_html.sub(/:$/, '')}</th>")
242
243 tr.add_child("<td>#{dd.inner_html}</td>") if has_dd
244
245 table.add_child(tr)
246 end
247
248 dl.replace(table)
249 end
250
251 end
252 end
253end
lib/docs/filters/scala/entries_v2.rbrenamed from lib/docs/filters/scala/entries.rb+3−3
@@ -1,6 +1,6 @@
1module Docs1module Docs
2 class Scala2 class Scala
3 class EntriesFilter < Docs::EntriesFilter3 class EntriesV2Filter < Docs::EntriesFilter
4 REPLACEMENTS = {4 REPLACEMENTS = {
5 '$eq' => '=',5 '$eq' => '=',
6 '$colon' => ':',6 '$colon' => ':',
@@ -75,12 +75,12 @@ module Docs · 68 unmodified lines
75 # include the companion object.75 # include the companion object.
76 def package_name76 def package_name
77 name = package_drop_last(slug_parts)77 name = package_drop_last(slug_parts)
78 name.empty? ? '_root_' : name78 name.empty? ? 'scala' : name
79 end79 end
80 80
81 def parent_package81 def parent_package
82 parent = package_drop_last(package_name.split('.'))82 parent = package_drop_last(package_name.split('.'))
83 parent.empty? ? '_root_' : parent83 parent.empty? ? 'scala' : parent
84 end84 end
85 85
86 def package_drop_last(parts)86 def package_drop_last(parts)
lib/docs/filters/scala/entries_v3.rb+105−0
@@ -0,0 +1,105 @@
1# frozen_string_literal: true
2
3module Docs
4 class Scala
5 class EntriesV3Filter < Docs::EntriesFilter
6 REPLACEMENTS = {
7 '$eq' => '=',
8 '$colon' => ':',
9 '$less' => '<',
10 }
11
12 def get_name
13 if is_package?
14 at_css('.cover-header h1').text
15 else
16 name = slug.split('/').last
17
18 # Some objects have inner objects, show ParentObject$.ChildObject$ instead of ParentObject$$ChildObject$
19 name = name.gsub('$$', '$.')
20
21 REPLACEMENTS.each do |key, value|
22 name = name.gsub(key, value)
23 end
24
25 # If a dollar sign is used as separator between two characters, replace it with a dot
26 name.gsub(/([^$.])\$([^$.])/, '\1.\2')
27 end
28 end
29
30 def get_type
31 # if this entry is for a package, we group the package under the parent package
32 if is_package?
33 parent_package
34 # otherwise, group it under the regular package name
35 else
36 package_name
37 end
38 end
39
40 def include_default_entry?
41 # Ignore package pages
42 at_css('.cover-header .micon.pa').nil?
43 end
44
45 def additional_entries
46 entries = []
47 titles = []
48
49 css(".documentableElement").each do |node|
50 # Ignore elements without IDs
51 id = node['id']
52 next if id.nil?
53
54 # Ignore deprecated and inherited members
55 next unless node.at_css('.deprecated').nil?
56
57 member_name = node.at_css('.documentableName').content
58 title = "#{name}.#{member_name}"
59
60 # Add () to methods that take parameters, i.e. methods who have (…)
61 # in their signature, ignoring occurrences of (implicit …) and (using …)
62 signature = node.at_css('.signature').content
63 title += '()' if signature =~ /\((?!implicit)(?!using ).*\)/
64
65 next if titles.include?(title) # Ignore duplicates (function overloading)
66
67 entries << [title, id]
68 titles.push(title)
69 end
70
71 entries
72 end
73
74 private
75
76 # For the package name, we use the slug rather than parsing the package
77 # name from the HTML because companion object classes may be broken out into
78 # their own entries (by the source documentation). When that happens,
79 # we want to group these classes (like `scala.reflect.api.Annotations.Annotation`)
80 # under the package name, and not the fully-qualfied name which would
81 # include the companion object.
82 def package_name
83 name = package_drop_last(slug_parts)
84 name.empty? ? 'scala' : name
85 end
86
87 def parent_package
88 parent = package_drop_last(package_name.split('.'))
89 parent.empty? ? 'scala' : parent
90 end
91
92 def package_drop_last(parts)
93 parts[0...-1].join('.')
94 end
95
96 def slug_parts
97 slug.split('/')
98 end
99
100 def is_package?
101 !at_css('.cover-header .micon.pa').nil?
102 end
103 end
104 end
105end
lib/docs/scrapers/scala.rb+38−9
@@ -3,24 +3,50 @@ module Docs · 2 unmodified lines
3 self.name = 'Scala'3 self.name = 'Scala'
4 self.type = 'scala'4 self.type = 'scala'
5 self.links = {5 self.links = {
6 home: 'http://www.scala-lang.org/',6 home: 'https://www.scala-lang.org/',
7 code: 'https://github.com/scala/scala'7 code: 'https://github.com/scala/scala'
8 }8 }
9 9
10 options[:container] = '#content-container'
11 options[:attribution] = <<-HTML10 options[:attribution] = <<-HTML
12 &copy; 2002-2019 EPFL, with contributions from Lightbend.<br>11 &copy; 2002-2022 EPFL, with contributions from Lightbend.<br>
13 Licensed under the Apache License, Version 2.0.12 Licensed under the Apache License, Version 2.0.
14 HTML13 HTML
15 14
15 # For Scala 3, there is no official download link for the documentation
16 # (see https://contributors.scala-lang.org/t/5537).
17 #
18 # We currently need to build the docs ourselves. To do so:
19 # 1. Make sure that Scala 3 and sbt are installed
20 # (https://www.scala-lang.org/download/scala3.html)
21 # 2. Clone the Scala 3 (Dotty) repository (https://github.com/lampepfl/dotty)
22 # 3. From the Dotty folder, run this command in the terminal:
23 # $ sbt scaladoc/generateScalaDocumentation
24 # 4. Extract scaladoc/output/scala3/api/ into docs/scala~3.1
25 version '3.1' do
26 self.release = '3.1.1'
27 self.base_url = 'https://scala-lang.org/api/3.1.1/'
28 self.root_path = 'index.html'
29
30 options[:skip_patterns] = [
31 # Ignore class names with include “#”, which cause issues with the scraper
32 /%23/,
33
34 # Ignore local links to the Java documentation created by a Scaladoc bug
35 /java\/lang/,
36 ]
37
38 html_filters.push 'scala/entries_v3', 'scala/clean_html_v3'
39 end
40
16 # https://downloads.lightbend.com/scala/2.13.0/scala-docs-2.13.0.zip41 # https://downloads.lightbend.com/scala/2.13.0/scala-docs-2.13.0.zip
17 # Extract api/scala-library into docs/scala~2.13_library42 # Extract api/scala-library into docs/scala~2.13_library
18 version '2.13 Library' do43 version '2.13 Library' do
19 self.release = '2.13.0'44 self.release = '2.13.0'
20 self.base_url = 'https://www.scala-lang.org/api/2.13.0/'45 self.base_url = 'https://www.scala-lang.org/api/2.13.0/'
21 self.root_path = 'index.html'46 self.root_path = 'index.html'
47 options[:container] = '#content-container'
22 48
23 html_filters.push 'scala/entries', 'scala/clean_html'49 html_filters.push 'scala/entries_v2', 'scala/clean_html_v2'
24 end50 end
25 51
26 # https://downloads.lightbend.com/scala/2.13.0/scala-docs-2.13.0.zip52 # https://downloads.lightbend.com/scala/2.13.0/scala-docs-2.13.0.zip
@@ -29,8 +55,9 @@ module Docs · 2 unmodified lines
29 self.release = '2.13.0'55 self.release = '2.13.0'
30 self.base_url = 'https://www.scala-lang.org/api/2.13.0/scala-reflect/'56 self.base_url = 'https://www.scala-lang.org/api/2.13.0/scala-reflect/'
31 self.root_path = 'index.html'57 self.root_path = 'index.html'
58 options[:container] = '#content-container'
32 59
33 html_filters.push 'scala/entries', 'scala/clean_html'60 html_filters.push 'scala/entries_v2', 'scala/clean_html_v2'
34 end61 end
35 62
36 # https://downloads.lightbend.com/scala/2.12.9/scala-docs-2.12.9.zip63 # https://downloads.lightbend.com/scala/2.12.9/scala-docs-2.12.9.zip
@@ -39,8 +66,9 @@ module Docs · 2 unmodified lines
39 self.release = '2.12.9'66 self.release = '2.12.9'
40 self.base_url = 'https://www.scala-lang.org/api/2.12.9/'67 self.base_url = 'https://www.scala-lang.org/api/2.12.9/'
41 self.root_path = 'index.html'68 self.root_path = 'index.html'
69 options[:container] = '#content-container'
42 70
43 html_filters.push 'scala/entries', 'scala/clean_html'71 html_filters.push 'scala/entries_v2', 'scala/clean_html_v2'
44 end72 end
45 73
46 # https://downloads.lightbend.com/scala/2.12.9/scala-docs-2.12.9.zip74 # https://downloads.lightbend.com/scala/2.12.9/scala-docs-2.12.9.zip
@@ -49,13 +77,14 @@ module Docs · 2 unmodified lines
49 self.release = '2.12.9'77 self.release = '2.12.9'
50 self.base_url = 'https://www.scala-lang.org/api/2.12.9/scala-reflect/'78 self.base_url = 'https://www.scala-lang.org/api/2.12.9/scala-reflect/'
51 self.root_path = 'index.html'79 self.root_path = 'index.html'
80 options[:container] = '#content-container'
52 81
53 html_filters.push 'scala/entries', 'scala/clean_html'82 html_filters.push 'scala/entries_v2', 'scala/clean_html_v2'
54 end83 end
55 84
56 def get_latest_version(opts)85 def get_latest_version(opts)
57 doc = fetch_doc('https://www.scala-lang.org/api/current/', opts)86 doc = fetch_doc('https://www.scala-lang.org/api/3.x/', opts)
58 doc.at_css('#doc-version').content87 doc.at_css('.projectVersion').content
59 end88 end
60 end89 end
61end90end