krisyotam/devdocs

docs/file-scrapers.md+8−15

 ```
 ## OpenJDK
+Search 'Openjdk' in https://www.debian.org/distrib/packages, find the `openjdk-$VERSION-doc` package,
+download it, extract it with `dpkg -x $PACKAGE ./` and move `./usr/share/doc/openjdk-16-jre-headless/api/`
+to `path/to/devdocs/docs/openjdk~$VERSION`
-https://packages.debian.org/sid/openjdk-11-doc
+If you use or have access to a Debian-based GNU/Linux distribution you can run the following command:
-```sh
-mkdir docs/openjdk~11
-curl --remote-name http://ftp.debian.org/debian/pool/main/o/openjdk-11/openjdk-11-doc_11.0.9.1+1-1_all.deb
-bsdtar --extract --to-stdout --file openjdk-11-doc_11.0.9.1+1-1_all.deb data.tar.xz | \
-bsdtar --extract --xz --file - --strip-components=6 --directory=docs/openjdk\~11/ ./usr/share/doc/openjdk-11-jre-headless/api/
-```
-https://packages.debian.org/sid/openjdk-8-doc
 ```sh
-mkdir docs/openjdk~8
+apt download openjdk-$VERSION-doc
-curl --remote-name http://ftp.debian.org/debian/pool/main/o/openjdk-8/openjdk-8-doc_8u272-b10-1_all.deb
+dpkg -x $PACKAGE ./
-bsdtar --extract --to-stdout --file openjdk-8-doc_8u272-b10-1_all.deb data.tar.xz | \
+# previous command makes a directory called 'usr' in the current directory
-bsdtar --extract --xz --file - --strip-components=6 --directory=docs/openjdk\~8/ ./usr/share/doc/openjdk-8-jre-headless/api/
+mv ./usr/share/doc/openjdk-16-jre-headless/api/ path/to/devdocs/docs/openjdk~$VERSION
 ```
 ## PHP

lib/docs/filters/openjdk/clean_html.rb+8−0

+# coding: utf-8
 # frozen_string_literal: true
 module Docs
           node.remove_attribute('class') unless node['class'] == 'inheritance'
         end
+        # fix ul section that contains summaries or tables
+        css('ul').each do |node|
+          node.css('section').each do |subnode|
+            node.add_previous_sibling(subnode)
+          end
+        end
         doc
       end
     end

lib/docs/filters/openjdk/clean_html_new.rb+31−0

@@ -0,0 +1,31 @@
	1	module Docs
	2	class Openjdk
	3	class CleanHtmlNewFilter < Filter
	4	def call
	5
	6	if root_page?
	7	at_css('h1').content = "OpenJDK #{release} Documentation"
	8	end
	9
	10	css('.header .sub-title').remove
	11
	12	css('blockquote pre').each do \|node\|
	13	node.parent.name = 'pre'
	14	node.parent['class'] = 'highlight'
	15	node.parent['data-language'] = 'java'
	16	node.parent.content = node.content
	17	node.remove
	18	end
	19
	20	# fix ul section that contains summaries or tables
	21	css('ul').each do \|node\|
	22	node.css('section').each do \|subnode\|
	23	node.add_previous_sibling(subnode)
	24	end
	25	end
	26
	27	doc
	28	end
	29	end
	30	end
	31	end

lib/docs/filters/openjdk/entries_new.rb+45−0

@@ -0,0 +1,45 @@
	1	module Docs
	2	class Openjdk
	3	class EntriesNewFilter < Docs::EntriesFilter
	4
	5	def get_name
	6	name = at_css('.header > .title').content.strip
	7	name.remove! 'Package '
	8	name.remove! 'Class '
	9	name.remove! 'Interface '
	10	name.remove! 'Annotation Type '
	11	name.remove! 'Enum '
	12	name.remove! %r{<.*}
	13	name
	14	end
	15
	16	def get_type
	17	return 'Packages' if slug.end_with?('package-summary')
	18	return 'Modules' if slug.end_with?('module-summary')
	19
	20	if subtitle = at_css('.header > .sub-title:last-of-type')
	21	type = subtitle.content.strip
	22	else
	23	type = at_css('.header > .title').content.strip.remove 'Package '
	24	type.remove!('Module ')
	25	end
	26	type = type.split('.')[0..2].join('.')
	27	type
	28	end
	29
	30	def additional_entries
	31	css('a[name$=".summary"]').each_with_object({}) do \|summary, entries\|
	32	next if summary['name'].include?('nested') \|\| summary['name'].include?('constructor') \|\|
	33	summary['name'].include?('field') \|\| summary['name'].include?('constant')
	34	summary.parent.css('.memberNameLink a').each do \|node\|
	35	name = node.parent.parent.content.strip
	36	name.sub! %r{$.+?$}m, '()'
	37	id = node['href'].remove(%r{.*#})
	38	entries[name] \|\|= ["#{self.name}.#{name}", id]
	39	end
	40	end.values
	41	end
	42
	43	end
	44	end
	45	end

lib/docs/scrapers/openjdk.rb+60−7

@@ -1,7 +1,6 @@
1	module Docs	1	module Docs
2	class Openjdk < FileScraper	2	class Openjdk < FileScraper
3	# Downloaded from packages.debian.org/sid/openjdk-8-doc	3
4	# Extracting subdirectory /usr/share/doc/openjdk-8-jre-headless/api
5	self.name = 'OpenJDK'	4	self.name = 'OpenJDK'
6	self.type = 'openjdk'	5	self.type = 'openjdk'
7	self.root_path = 'overview-summary.html'	6	self.root_path = 'overview-summary.html'
@@ -11,7 +10,6 @@ module Docs · 3 unmodified lines
11	}	10	}
12		11
13	html_filters.insert_after 'internal_urls', 'openjdk/clean_urls'	12	html_filters.insert_after 'internal_urls', 'openjdk/clean_urls'
14	html_filters.push 'openjdk/entries', 'openjdk/clean_html'
15		13
16	options[:skip_patterns] = [	14	options[:skip_patterns] = [
17	/compact[123]-/,	15	/compact[123]-/,
@@ -19,7 +17,12 @@ module Docs · 1 unmodified line
19	/package-tree\.html/,	17	/package-tree\.html/,
20	/package-use\.html/,	18	/package-use\.html/,
21	/class-use\//,	19	/class-use\//,
22	/doc-files\//]	20	/doc-files\//,
		21	/\.svg/,
		22	/\.png/
		23	]
		24
		25	options[:only_patterns] = [/\Ajava\./]
23		26
24	options[:attribution] = <<-HTML	27	options[:attribution] = <<-HTML
25	© 1993, 2020, Oracle and/or its affiliates. All rights reserved.<br>	28	© 1993, 2020, Oracle and/or its affiliates. All rights reserved.<br>
@@ -29,16 +32,59 @@ module Docs · 3 unmodified lines
29	Java and OpenJDK are trademarks or registered trademarks of Oracle and/or its affiliates.	32	Java and OpenJDK are trademarks or registered trademarks of Oracle and/or its affiliates.
30	HTML	33	HTML
31		34
		35	NEWFILTERS = ['openjdk/entries_new', 'openjdk/clean_html_new']
		36
		37	version '16' do
		38	self.release = '16'
		39	self.root_path = 'index.html'
		40
		41	html_filters.push NEWFILTERS
		42
		43	options[:container] = 'main'
		44	end
		45
		46	version '15' do
		47	self.release = '15.0.1'
		48	self.root_path = 'index.html'
		49
		50	html_filters.push NEWFILTERS
		51
		52	options[:container] = 'main'
		53	end
		54
		55	OLDFILTERS = ['openjdk/entries', 'openjdk/clean_html']
		56
		57	version '14' do
		58	self.release = '14.0.2'
		59	self.root_path = 'index.html'
		60
		61	html_filters.push OLDFILTERS
		62
		63	options[:container] = 'main'
		64	end
		65
		66	version '13' do
		67	self.release = '13.0.5'
		68	self.root_path = 'index.html'
		69
		70	html_filters.push OLDFILTERS
		71
		72	options[:container] = 'main'
		73	end
		74
32	version '11' do	75	version '11' do
33	self.release = '11.0.9'	76	self.release = '11.0.9'
34	self.root_path = 'index.html'	77	self.root_path = 'index.html'
35	self.base_url = 'https://docs.oracle.com/en/java/javase/11/docs/api/'	78	self.base_url = 'https://docs.oracle.com/en/java/javase/11/docs/api/'
36	options[:only_patterns] = [/\Ajava\./]	79
		80	html_filters.push OLDFILTERS
37	end	81	end
38		82
39	version '8' do	83	version '8' do
40	self.release = '8'	84	self.release = '8'
41		85
		86	html_filters.push OLDFILTERS
		87
42	options[:only_patterns] = [	88	options[:only_patterns] = [
43	/\Ajava\/beans\//,	89	/\Ajava\/beans\//,
44	/\Ajava\/io\//,	90	/\Ajava\/io\//,
@@ -61,20 +107,27 @@ module Docs · 16 unmodified lines
61	/\Ajavax\/script\//,	107	/\Ajavax\/script\//,
62	/\Ajavax\/security\//,	108	/\Ajavax\/security\//,
63	/\Ajavax\/sound\//,	109	/\Ajavax\/sound\//,
64	/\Ajavax\/tools\//]	110	/\Ajavax\/tools\//
		111	]
65	end	112	end
66		113
67	version '8 GUI' do	114	version '8 GUI' do
68	self.release = '8'	115	self.release = '8'
69		116
		117	html_filters.push OLDFILTERS
		118
70	options[:only_patterns] = [	119	options[:only_patterns] = [
71	/\Ajava\/awt\//,	120	/\Ajava\/awt\//,
72	/\Ajavax\/swing\//]	121	/\Ajavax\/swing\//
		122	]
		123
73	end	124	end
74		125
75	version '8 Web' do	126	version '8 Web' do
76	self.release = '8'	127	self.release = '8'
77		128
		129	html_filters.push OLDFILTERS
		130
78	options[:only_patterns] = [	131	options[:only_patterns] = [
79	/\Ajava\/applet\//,	132	/\Ajava\/applet\//,
80	/\Ajava\/rmi\//,	133	/\Ajava\/rmi\//,