Add a scraper for Scala 3

Nicolas Ettlin committed 4 years ago

Commit afe3a261 parent 7a9b9fe5 files · +348 −7Browse files

Unified · Split

5 files

assets/javascripts/templates/pages/about_tmpl.coffee +2 −2
assets/stylesheets/pages/_scala.scss +34 −0
lib/docs/filters/scala/clean_html_v3.rb +181 −0
lib/docs/filters/scala/entries_v3.rb +104 −0
lib/docs/scrapers/scala.rb +27 −5

assets/javascripts/templates/pages/about_tmpl.coffee+2−2

@@ -794,9 +794,9 @@ credits = [ · 793 unmodified lines
794	794	'https://raw.githubusercontent.com/sass/sass/stable/MIT-LICENSE'
795	795	], [
796	796	'Scala',
797		− '2002-2019 EPFL, with contributions from Lightbend',
	797	+ '2002-2022 EPFL, with contributions from Lightbend',
798	798	'Apache',
799		− 'https://raw.githubusercontent.com/scala/scala-lang/master/license.md'
	799	+ 'https://www.scala-lang.org/license/'
800	800	], [
801	801	'scikit-image',
802	802	'2019 the scikit-image team',

assets/stylesheets/pages/_scala.scss+34−0

@@ -1,4 +1,38 @@
1	1	._scala {
2	2	@extend %simple;
	3	+
3	4	.deprecated { @extend %label-red; }
	5	+
	6	+ .related-types {
	7	+ @extend %pre;
	8	+ margin-top: 0;
	9	+ white-space: normal;
	10	+ }
	11	+
	12	+ .links {
	13	+ @extend %box;
	14	+ margin-left: -1rem;
	15	+ text-align: center;
	16	+ padding: .5em;
	17	+
	18	+ a { padding: .4em }
	19	+
	20	+ @include print {
	21	+ display: none;
	22	+ }
	23	+ }
	24	+
	25	+ .source-link {
	26	+ float: right;
	27	+ font-size: .75rem;
	28	+ color: var(--linkColor);
	29	+ cursor: pointer;
	30	+ @extend %user-select-none;
	31	+
	32	+ &:hover { text-decoration: underline; }
	33	+
	34	+ @include print {
	35	+ display: none;
	36	+ }
	37	+ }
4	38	}

lib/docs/filters/scala/clean_html_v3.rb+181−0

@@ -0,0 +1,181 @@
	1	+# frozen_string_literal: true
	2	+
	3	+module Docs
	4	+ class Scala
	5	+ class CleanHtmlV3Filter < Filter
	6	+ def call
	7	+ # Remove unneeded elements
	8	+ css('.documentableFilter, .documentableAnchor, .documentableBrief').remove
	9	+
	10	+ format_title
	11	+ format_top_links
	12	+ format_metadata
	13	+ format_members
	14	+
	15	+ # Simplify the HTML structure
	16	+ @doc = at_css('#content > div')
	17	+ css('.documentableList > *').each do \|element\|
	18	+ element.parent = doc
	19	+ end
	20	+ at_css('.membersList').remove
	21	+
	22	+ doc
	23	+ end
	24	+
	25	+ def format_title
	26	+ # Add the kind of page to the title
	27	+ cover_header = at_css('.cover-header')
	28	+ unless cover_header.nil?
	29	+ icon = cover_header.at_css('.micon')
	30	+ types = {
	31	+ cl: 'Class',
	32	+ ob: 'Object',
	33	+ tr: 'Trait',
	34	+ en: 'Enum',
	35	+ ty: 'Type',
	36	+ pa: 'Package',
	37	+ }
	38	+ type_id = cover_header.at_css('.micon')['class']
	39	+ type_id.remove!('micon ')
	40	+ type_id.remove!('-wc')
	41	+ type = types[type_id.to_sym]
	42	+ name = CGI.escapeHTML cover_header.at_css('h1').text
	43	+
	44	+ package = at_css('.breadcrumbs a:nth-of-type(3)').text
	45	+ package = package + '.' unless name.empty? \|\| package.empty?
	46	+
	47	+ title = root_page? ? 'Package root' : "#{type} #{package}#{name}".strip
	48	+ cover_header.replace "<h1>#{title}</h1>"
	49	+ end
	50	+
	51	+ # Signature
	52	+ signature = at_css('.signature')
	53	+ signature_annotations = signature.at_css('.annotations')
	54	+ signature_annotations.name = 'small' unless signature_annotations.nil?
	55	+ signature.replace "<h2 id=\"signature\">#{signature.inner_html}</h2>"
	56	+ end
	57	+
	58	+ def format_top_links
	59	+ # Companion page
	60	+ links = []
	61	+ at_css('.attributes').css('dt').each do \|dt\|
	62	+ next if dt.content.strip != 'Companion:'
	63	+ dd = dt.next_sibling
	64	+
	65	+ companion_link = dd.at_css('a')
	66	+ companion_link.content = "Companion #{companion_link.content}"
	67	+ links.append(companion_link.to_html)
	68	+
	69	+ dt.remove
	70	+ dd.remove
	71	+ end
	72	+
	73	+ # Source code
	74	+ at_css('.attributes').css('dt').each do \|dt\|
	75	+ next if dt.content.strip != 'Source:'
	76	+ dd = dt.next_sibling
	77	+
	78	+ source_link = dd.at_css('a')
	79	+ source_link.content = 'Source code'
	80	+ links.append(source_link.to_html)
	81	+
	82	+ dt.remove
	83	+ dd.remove
	84	+ end
	85	+
	86	+ # Format the links
	87	+ title = at_css('h1')
	88	+ title.add_next_sibling("<div class=\"links\">#{links.join(' • ')}</div>")
	89	+ end
	90	+
	91	+ def format_metadata
	92	+ # Metadata (attributes)
	93	+ css('.tabs.single .monospace').each do \|node\|
	94	+ node['class'] = 'related-types'
	95	+
	96	+ if node.children.count > 15
	97	+ node.replace "<details>
	98	+ <summary>#{node.children.count} types</summary>
	99	+ #{node.to_html}
	100	+ </details>"
	101	+ end
	102	+ end
	103	+
	104	+ attributes = at_css('.attributes')
	105	+ attributes.add_previous_sibling('<h3>Metadata</h3>')
	106	+
	107	+ tabs_names = css('.tabs.single .names .tab')
	108	+ tabs_contents = css('.tabs.single .contents .tab')
	109	+ tabs_names.zip(tabs_contents).each do \|name, contents\|
	110	+ next if name.content == "Graph"
	111	+
	112	+ attributes.add_child("<dt>#{name.content}</dt>")
	113	+ attributes.add_child("<dd>#{contents.inner_html.strip}</dd>")
	114	+ end
	115	+ at_css('.tabs').remove
	116	+ end
	117	+
	118	+ def format_members
	119	+ # Headings
	120	+ css('.cover h2').each do \|node\|
	121	+ node.name = 'h3'
	122	+ end
	123	+ css('h2:not(#signature)').remove
	124	+ css(
	125	+ '.membersList h3',
	126	+
	127	+ # Custom group headers for which Scaladoc generates invalid HTML
	128	+ '.documentableList > h3:empty + p'
	129	+ ).each do \|node\|
	130	+ node.name = 'h2'
	131	+ node.content = node.content
	132	+ end
	133	+
	134	+ # Methods
	135	+ css('.documentableElement').each do \|element\|
	136	+ header = element.at_css('.header')
	137	+ header.name = 'h3'
	138	+
	139	+ id = element['id']
	140	+ element.remove_attribute('id')
	141	+ header['id'] = id
	142	+
	143	+ annotations = element.at_css('.annotations')
	144	+ annotations.name = 'small'
	145	+ header.prepend_child(annotations)
	146	+
	147	+ # View source
	148	+ element.css('dt').each do \|dt\|
	149	+ next if dt.content.strip != 'Source:'
	150	+ dd = dt.next_sibling
	151	+
	152	+ source_link = dd.at_css('a')
	153	+ source_link.content = 'Source'
	154	+ source_link['class'] = 'source-link'
	155	+ header.prepend_child(source_link)
	156	+
	157	+ dt.remove
	158	+ dd.remove
	159	+ end
	160	+
	161	+ # Remove the unnecessary wrapper element
	162	+ element.replace(element.inner_html)
	163	+ end
	164	+
	165	+ # Remove deprecated sections
	166	+ css('.documentableList').each do \|list\|
	167	+ header = list.at_css('.groupHeader')
	168	+ list.remove if (header.text.downcase.include? 'deprecate' rescue false)
	169	+ end
	170	+
	171	+ # Code blocks
	172	+ css('pre > code').each do \|code\|
	173	+ pre = code.parent
	174	+ pre['data-language'] = 'scala'
	175	+ pre.inner_html = code.inner_html
	176	+ end
	177	+ end
	178	+
	179	+ end
	180	+ end
	181	+end

lib/docs/filters/scala/entries_v3.rb+104−0

@@ -0,0 +1,104 @@
	1	+# frozen_string_literal: true
	2	+
	3	+module Docs
	4	+ class Scala
	5	+ class EntriesV3Filter < Docs::EntriesFilter
	6	+ REPLACEMENTS = {
	7	+ '$eq' => '=',
	8	+ '$colon' => ':',
	9	+ '$less' => '<',
	10	+ }
	11	+
	12	+ def get_name
	13	+ if is_package?
	14	+ at_css('.cover-header h1').text
	15	+ else
	16	+ name = slug.split('/').last
	17	+
	18	+ # Some objects have inner objects, show ParentObject$.ChildObject$ instead of ParentObject$$ChildObject$
	19	+ name = name.gsub('$$', '$.')
	20	+
	21	+ REPLACEMENTS.each do \|key, value\|
	22	+ name = name.gsub(key, value)
	23	+ end
	24	+
	25	+ # If a dollar sign is used as separator between two characters, replace it with a dot
	26	+ name.gsub(/([^$.])\$([^$.])/, '\1.\2')
	27	+ end
	28	+ end
	29	+
	30	+ def get_type
	31	+ # if this entry is for a package, we group the package under the parent package
	32	+ if is_package?
	33	+ parent_package
	34	+ # otherwise, group it under the regular package name
	35	+ else
	36	+ package_name
	37	+ end
	38	+ end
	39	+
	40	+ def include_default_entry?
	41	+ true
	42	+ end
	43	+
	44	+ def additional_entries
	45	+ entries = []
	46	+ titles = []
	47	+
	48	+ css(".documentableElement").each do \|node\|
	49	+ # Ignore elements without IDs
	50	+ id = node['id']
	51	+ next if id.nil?
	52	+
	53	+ # Ignore deprecated and inherited members
	54	+ next unless node.at_css('.deprecated').nil?
	55	+
	56	+ member_name = node.at_css('.documentableName').content
	57	+ title = "#{name}.#{member_name}"
	58	+
	59	+ # Add () to methods that take parameters, i.e. methods who have (…)
	60	+ # in their signature, ignoring occurrences of (implicit …) and (using …)
	61	+ signature = node.at_css('.signature').content
	62	+ title += '()' if signature =~ /$(?!implicit)(?!using ).*$/
	63	+
	64	+ next if titles.include?(title) # Ignore duplicates (function overloading)
	65	+
	66	+ entries << [title, id]
	67	+ titles.push(title)
	68	+ end
	69	+
	70	+ entries
	71	+ end
	72	+
	73	+ private
	74	+
	75	+ # For the package name, we use the slug rather than parsing the package
	76	+ # name from the HTML because companion object classes may be broken out into
	77	+ # their own entries (by the source documentation). When that happens,
	78	+ # we want to group these classes (like `scala.reflect.api.Annotations.Annotation`)
	79	+ # under the package name, and not the fully-qualfied name which would
	80	+ # include the companion object.
	81	+ def package_name
	82	+ name = package_drop_last(slug_parts)
	83	+ name.empty? ? 'scala' : name
	84	+ end
	85	+
	86	+ def parent_package
	87	+ parent = package_drop_last(package_name.split('.'))
	88	+ parent.empty? ? 'scala' : parent
	89	+ end
	90	+
	91	+ def package_drop_last(parts)
	92	+ parts[0...-1].join('.')
	93	+ end
	94	+
	95	+ def slug_parts
	96	+ slug.split('/')
	97	+ end
	98	+
	99	+ def is_package?
	100	+ !at_css('.cover-header .micon.pa').nil?
	101	+ end
	102	+ end
	103	+ end
	104	+end

lib/docs/scrapers/scala.rb+27−5

@@ -3,22 +3,41 @@ module Docs · 2 unmodified lines
3	3	self.name = 'Scala'
4	4	self.type = 'scala'
5	5	self.links = {
6		− home: 'http://www.scala-lang.org/',
	6	+ home: 'https://www.scala-lang.org/',
7	7	code: 'https://github.com/scala/scala'
8	8	}
9	9
10		− options[:container] = '#content-container'
11	10	options[:attribution] = <<-HTML
12		− © 2002-2019 EPFL, with contributions from Lightbend.<br>
	11	+ © 2002-2022 EPFL, with contributions from Lightbend.<br>
13	12	Licensed under the Apache License, Version 2.0.
14	13	HTML
15	14
	15	+ # For Scala 3, there is no official download link for the documentation
	16	+ # (see https://contributors.scala-lang.org/t/5537).
	17	+ #
	18	+ # We currently need to build the docs ourselves. To do so:
	19	+ # 1. Make sure that Scala 3 and sbt are installed
	20	+ # (https://www.scala-lang.org/download/scala3.html)
	21	+ # 2. Clone the Scala 3 (Dotty) repository (https://github.com/lampepfl/dotty)
	22	+ # 3. From the Dotty folder, run this command in the terminal:
	23	+ # $ sbt scaladoc/generateScalaDocumentation
	24	+ # 4. Extract scaladoc/output/scala3/api/ into docs/scala~3.1
	25	+ version '3.1' do
	26	+ self.release = '3.1.1'
	27	+ self.base_url = 'https://scala-lang.org/api/3.1.1/'
	28	+ self.root_path = 'index.html'
	29	+ # options[:container] = '#main-content'
	30	+
	31	+ html_filters.push 'scala/entries_v3', 'scala/clean_html_v3'
	32	+ end
	33	+
16	34	# https://downloads.lightbend.com/scala/2.13.0/scala-docs-2.13.0.zip
17	35	# Extract api/scala-library into docs/scala~2.13_library
18	36	version '2.13 Library' do
19	37	self.release = '2.13.0'
20	38	self.base_url = 'https://www.scala-lang.org/api/2.13.0/'
21	39	self.root_path = 'index.html'
	40	+ options[:container] = '#content-container'
22	41
23	42	html_filters.push 'scala/entries_v2', 'scala/clean_html_v2'
24	43	end
@@ -29,6 +48,7 @@ module Docs · 4 unmodified lines
29	48	self.release = '2.13.0'
30	49	self.base_url = 'https://www.scala-lang.org/api/2.13.0/scala-reflect/'
31	50	self.root_path = 'index.html'
	51	+ options[:container] = '#content-container'
32	52
33	53	html_filters.push 'scala/entries_v2', 'scala/clean_html_v2'
34	54	end
@@ -39,6 +59,7 @@ module Docs · 4 unmodified lines
39	59	self.release = '2.12.9'
40	60	self.base_url = 'https://www.scala-lang.org/api/2.12.9/'
41	61	self.root_path = 'index.html'
	62	+ options[:container] = '#content-container'
42	63
43	64	html_filters.push 'scala/entries_v2', 'scala/clean_html_v2'
44	65	end
@@ -49,13 +70,14 @@ module Docs · 4 unmodified lines
49	70	self.release = '2.12.9'
50	71	self.base_url = 'https://www.scala-lang.org/api/2.12.9/scala-reflect/'
51	72	self.root_path = 'index.html'
	73	+ options[:container] = '#content-container'
52	74
53	75	html_filters.push 'scala/entries_v2', 'scala/clean_html_v2'
54	76	end
55	77
56	78	def get_latest_version(opts)
57		− doc = fetch_doc('https://www.scala-lang.org/api/current/', opts)
58		− doc.at_css('#doc-version').content
	79	+ doc = fetch_doc('https://www.scala-lang.org/api/3.x/', opts)
	80	+ doc.at_css('.projectVersion').content
59	81	end
60	82	end
61	83	end