Simplify file scraper setup; scrape files in the "docs/[slug]" directory

Thibaut Courouble committed 7 years ago

Commit bf003661 parent 91753ce22 files · +124 −104Browse files

Showing the changes to lib/docs (19 of 22 files). Show all

Unified · Split

19 files

lib/docs/core/doc.rb +6 −0
lib/docs/core/scrapers/file_scraper.rb +18 −7
lib/docs/scrapers/c.rb +0 −1
lib/docs/scrapers/cpp.rb +0 −1
lib/docs/scrapers/dart.rb +0 −2
lib/docs/scrapers/django.rb +0 −6
lib/docs/scrapers/erlang.rb +0 −4
lib/docs/scrapers/gnu/gcc.rb +0 −8
lib/docs/scrapers/gnu/gnu_fortran.rb +0 −4
lib/docs/scrapers/nokogiri2.rb +0 −1
lib/docs/scrapers/numpy.rb +0 −1
lib/docs/scrapers/openjdk.rb +2 −3
lib/docs/scrapers/perl.rb +0 −1
lib/docs/scrapers/php.rb +2 −3
lib/docs/scrapers/python.rb +4 −8
lib/docs/scrapers/rdoc/minitest.rb +1 −1
lib/docs/scrapers/rdoc/rails.rb +0 −1
lib/docs/scrapers/rdoc/ruby.rb +0 −4
lib/docs/scrapers/sqlite.rb +0 −1

@@ -95,6 +95,9 @@ module Docs · 94 unmodified lines
95	95	false
96	96	end
97	97	end
	98	+ rescue Docs::SetupError => error
	99	+ puts "ERROR: #{error.message}"
	100	+ false
98	101	end
99	102
100	103	def store_pages(store)
@@ -118,6 +121,9 @@ module Docs · 17 unmodified lines
118	121	false
119	122	end
120	123	end
	124	+ rescue Docs::SetupError => error
	125	+ puts "ERROR: #{error.message}"
	126	+ false
121	127	end
122	128
123	129	private

lib/docs/core/scrapers/file_scraper.rb+18−7

@@ -1,14 +1,13 @@
1	1	module Docs
2	2	class FileScraper < Scraper
	3	+ SOURCE_DIRECTORY = File.expand_path '../../../../../docs', __FILE__
	4	+
3	5	Response = Struct.new :body, :url
4	6
5	7	class << self
6		− attr_accessor :dir
7		−
8	8	def inherited(subclass)
9	9	super
10	10	subclass.base_url = base_url
11		− subclass.dir = dir
12	11	end
13	12	end
14	13
@@ -16,13 +15,25 @@ module Docs · 1 unmodified line
16	15
17	16	html_filters.push 'clean_local_urls'
18	17
	18	+ def source_directory
	19	+ @source_directory \|\|= File.join(SOURCE_DIRECTORY, self.class.path)
	20	+ end
	21	+
19	22	private
20	23
	24	+ def assert_source_directory_exists
	25	+ unless Dir.exists?(source_directory)
	26	+ raise SetupError, "The #{self.class.name} scraper requires the original documentation files to be stored in the \"#{source_directory}\" directory."
	27	+ end
	28	+ end
	29	+
21	30	def request_one(url)
22		− Response.new read_file(file_path_for(url)), URL.parse(url)
	31	+ assert_source_directory_exists
	32	+ Response.new read_file(url_to_path(url)), URL.parse(url)
23	33	end
24	34
25	35	def request_all(urls)
	36	+ assert_source_directory_exists
26	37	queue = [urls].flatten
27	38	until queue.empty?
28	39	result = yield request_one(queue.shift)
@@ -34,12 +45,12 @@ module Docs · 5 unmodified lines
34	45	response.body.present?
35	46	end
36	47
37		− def file_path_for(url)
38		− File.join self.class.dir, url.remove(base_url.to_s)
	48	+ def url_to_path(url)
	49	+ url.remove(base_url.to_s)
39	50	end
40	51
41	52	def read_file(path)
42		− File.read(path)
	53	+ File.read(File.join(source_directory, path))
43	54	rescue
44	55	instrument 'warn.doc', msg: "Failed to open file: #{path}"
45	56	nil

lib/docs/scrapers/c.rb+0−1

@@ -1,7 +1,6 @@
1	1	module Docs
2	2	class C < FileScraper
3	3	self.type = 'c'
4		− self.dir = '/Users/Thibaut/DevDocs/Docs/c'
5	4	self.base_url = 'http://en.cppreference.com/w/c/'
6	5	self.root_path = 'header.html'
7	6

lib/docs/scrapers/cpp.rb+0−1

@@ -3,7 +3,6 @@ module Docs · 2 unmodified lines
3	3	self.name = 'C++'
4	4	self.slug = 'cpp'
5	5	self.type = 'c'
6		− self.dir = '/Users/Thibaut/DevDocs/Docs/cpp'
7	6	self.base_url = 'http://en.cppreference.com/w/cpp/'
8	7	self.root_path = 'header.html'
9	8

lib/docs/scrapers/dart.rb+0−2

@@ -24,13 +24,11 @@ module Docs · 23 unmodified lines
24	24
25	25	version '2' do
26	26	self.release = '2.0.0'
27		− self.dir = '/Users/Thibaut/DevDocs/Docs/Dart2'
28	27	self.base_url = "https://api.dartlang.org/stable/#{release}/"
29	28	end
30	29
31	30	version '1' do
32	31	self.release = '1.24.3'
33		− self.dir = '/Users/Thibaut/DevDocs/Docs/Dart1'
34	32	self.base_url = "https://api.dartlang.org/stable/#{release}/"
35	33	end
36	34	end

lib/docs/scrapers/django.rb+0−6

@@ -36,37 +36,31 @@ module Docs · 35 unmodified lines
36	36
37	37	version '2.1' do
38	38	self.release = '2.1.0'
39		− self.dir = '/Users/Thibaut/DevDocs/Docs/Django21'
40	39	self.base_url = 'https://docs.djangoproject.com/en/2.1/'
41	40	end
42	41
43	42	version '2.0' do
44	43	self.release = '2.0.7'
45		− self.dir = '/Users/Thibaut/DevDocs/Docs/Django20'
46	44	self.base_url = 'https://docs.djangoproject.com/en/2.0/'
47	45	end
48	46
49	47	version '1.11' do
50	48	self.release = '1.11.9'
51		− self.dir = '/Users/Thibaut/DevDocs/Docs/Django111'
52	49	self.base_url = 'https://docs.djangoproject.com/en/1.11/'
53	50	end
54	51
55	52	version '1.10' do
56	53	self.release = '1.10.8'
57		− self.dir = '/Users/Thibaut/DevDocs/Docs/Django110'
58	54	self.base_url = 'https://docs.djangoproject.com/en/1.10/'
59	55	end
60	56
61	57	version '1.9' do
62	58	self.release = '1.9.13'
63		− self.dir = '/Users/Thibaut/DevDocs/Docs/Django19'
64	59	self.base_url = 'https://docs.djangoproject.com/en/1.9/'
65	60	end
66	61
67	62	version '1.8' do
68	63	self.release = '1.8.18'
69		− self.dir = '/Users/Thibaut/DevDocs/Docs/Django18'
70	64	self.base_url = 'https://docs.djangoproject.com/en/1.8/'
71	65	end
72	66	end

lib/docs/scrapers/erlang.rb+0−4

@@ -42,22 +42,18 @@ module Docs · 41 unmodified lines
42	42
43	43	version '21' do
44	44	self.release = '21.0'
45		− self.dir = '/Users/Thibaut/DevDocs/Docs/Erlang21'
46	45	end
47	46
48	47	version '20' do
49	48	self.release = '20.3'
50		− self.dir = '/Users/Thibaut/DevDocs/Docs/Erlang20'
51	49	end
52	50
53	51	version '19' do
54	52	self.release = '19.3'
55		− self.dir = '/Users/Thibaut/DevDocs/Docs/Erlang19'
56	53	end
57	54
58	55	version '18' do
59	56	self.release = '18.3'
60		− self.dir = '/Users/Thibaut/DevDocs/Docs/Erlang18'
61	57	end
62	58	end
63	59	end

lib/docs/scrapers/gnu/gcc.rb+0−8

@@ -48,13 +48,11 @@ module Docs · 47 unmodified lines
48	48
49	49	version '7' do
50	50	self.release = '7.3.0'
51		− self.dir = '/Users/Thibaut/DevDocs/Docs/gcc7'
52	51	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gcc/"
53	52	end
54	53
55	54	version '7 CPP' do
56	55	self.release = '7.3.0'
57		− self.dir = '/Users/Thibaut/DevDocs/Docs/gcpp7'
58	56	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/cpp/"
59	57
60	58	options[:replace_paths] = CPP_PATHS
@@ -62,7 +60,6 @@ module Docs · 1 unmodified line
62	60
63	61	version '6' do
64	62	self.release = '6.4.0'
65		− self.dir = '/Users/Thibaut/DevDocs/Docs/gcc6'
66	63	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gcc/"
67	64
68	65	options[:root_title] = 'Using the GNU Compiler Collection (GCC)'
@@ -70,7 +67,6 @@ module Docs · 1 unmodified line
70	67
71	68	version '6 CPP' do
72	69	self.release = '6.4.0'
73		− self.dir = '/Users/Thibaut/DevDocs/Docs/gcpp6'
74	70	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/cpp/"
75	71
76	72	options[:replace_paths] = CPP_PATHS
@@ -78,7 +74,6 @@ module Docs · 1 unmodified line
78	74
79	75	version '5' do
80	76	self.release = '5.4.0'
81		− self.dir = '/Users/Thibaut/DevDocs/Docs/gcc5'
82	77	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gcc/"
83	78
84	79	options[:root_title] = 'Using the GNU Compiler Collection (GCC)'
@@ -86,7 +81,6 @@ module Docs · 1 unmodified line
86	81
87	82	version '5 CPP' do
88	83	self.release = '5.4.0'
89		− self.dir = '/Users/Thibaut/DevDocs/Docs/gcpp5'
90	84	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/cpp/"
91	85
92	86	options[:replace_paths] = CPP_PATHS
@@ -94,7 +88,6 @@ module Docs · 1 unmodified line
94	88
95	89	version '4' do
96	90	self.release = '4.9.3'
97		− self.dir = '/Users/Thibaut/DevDocs/Docs/gcc4'
98	91	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gcc/"
99	92
100	93	options[:root_title] = 'Using the GNU Compiler Collection (GCC)'
@@ -102,7 +95,6 @@ module Docs · 1 unmodified line
102	95
103	96	version '4 CPP' do
104	97	self.release = '4.9.3'
105		− self.dir = '/Users/Thibaut/DevDocs/Docs/gcpp4'
106	98	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/cpp/"
107	99
108	100	options[:replace_paths] = CPP_PATHS

lib/docs/scrapers/gnu/gnu_fortran.rb+0−4

@@ -8,25 +8,21 @@ module Docs · 7 unmodified lines
8	8
9	9	version '7' do
10	10	self.release = '7.3.0'
11		− self.dir = '/Users/Thibaut/DevDocs/Docs/gfortran7'
12	11	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
13	12	end
14	13
15	14	version '6' do
16	15	self.release = '6.4.0'
17		− self.dir = '/Users/Thibaut/DevDocs/Docs/gfortran6'
18	16	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
19	17	end
20	18
21	19	version '5' do
22	20	self.release = '5.4.0'
23		− self.dir = '/Users/Thibaut/DevDocs/Docs/gfortran5'
24	21	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
25	22	end
26	23
27	24	version '4' do
28	25	self.release = '4.9.3'
29		− self.dir = '/Users/Thibaut/DevDocs/Docs/gfortran4'
30	26	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
31	27	end
32	28	end

lib/docs/scrapers/nokogiri2.rb+0−1

@@ -3,7 +3,6 @@ module Docs · 2 unmodified lines
3	3	self.name = 'Nokogiri'
4	4	self.slug = 'nokogiri'
5	5	self.release = '1.8.1'
6		− self.dir = '/Users/Thibaut/DevDocs/Docs/RDoc/Nokogiri'
7	6
8	7	html_filters.replace 'rdoc/entries', 'nokogiri2/entries'
9	8

lib/docs/scrapers/numpy.rb+0−1

@@ -2,7 +2,6 @@ module Docs · 1 unmodified line
2	2	class Numpy < FileScraper
3	3	self.name = 'NumPy'
4	4	self.type = 'sphinx'
5		− self.dir = '/Users/Thibaut/DevDocs/Docs/numpy/reference/'
6	5	self.root_path = 'index.html'
7	6	self.links = {
8	7	home: 'http://www.numpy.org/',

lib/docs/scrapers/openjdk.rb+2−3

@@ -1,11 +1,10 @@
1	1	module Docs
2	2	class Openjdk < FileScraper
	3	+ # Downloaded from packages.debian.org/sid/openjdk-8-doc
	4	+ # Extracting subdirectory /usr/share/doc/openjdk-8-jre-headless/api
3	5	self.name = 'OpenJDK'
4	6	self.type = 'openjdk'
5	7	self.root_path = 'overview-summary.html'
6		− # Downloaded from packages.debian.org/sid/openjdk-8-doc
7		− # Extracting subdirectory /usr/share/doc/openjdk-8-jre-headless/api
8		− self.dir = '/Users/Thibaut/DevDocs/Docs/OpenJDK'
9	8
10	9	html_filters.insert_after 'internal_urls', 'openjdk/clean_urls'
11	10	html_filters.push 'openjdk/entries', 'openjdk/clean_html'

lib/docs/scrapers/perl.rb+0−1

@@ -2,7 +2,6 @@ module Docs · 1 unmodified line
2	2	class Perl < FileScraper
3	3	self.name = 'Perl'
4	4	self.type = 'perl'
5		− self.dir = '/Users/Thibaut/DevDocs/Docs/Perl'
6	5	self.root_path = 'index.html'
7	6	self.links = {
8	7	home: 'https://www.perl.org/'

lib/docs/scrapers/php.rb+2−3

@@ -1,5 +1,7 @@
1	1	module Docs
2	2	class Php < FileScraper
	3	+ # Downloaded from php.net/download-docs.php
	4	+
3	5	include FixInternalUrlsBehavior
4	6
5	7	self.name = 'PHP'
@@ -23,9 +25,6 @@ module Docs · 17 unmodified lines
23	25	code: 'https://git.php.net/?p=php-src.git;a=summary'
24	26	}
25	27
26		− # Downloaded from php.net/download-docs.php
27		− self.dir = '/Users/Thibaut/DevDocs/Docs/PHP'
28		−
29	28	html_filters.push 'php/internal_urls', 'php/entries', 'php/clean_html', 'title'
30	29	text_filters.push 'php/fix_urls'
31	30

lib/docs/scrapers/python.rb+4−8

@@ -23,33 +23,29 @@ module Docs · 22 unmodified lines
23	23	Licensed under the PSF License.
24	24	HTML
25	25
26		− version '3.7' do
	26	+ version '3.7' do # docs.python.org/3.7/download.html
27	27	self.release = '3.7.0'
28		− self.dir = '/Users/Thibaut/DevDocs/Docs/Python37' # docs.python.org/3.7/download.html
29	28	self.base_url = 'https://docs.python.org/3.7/'
30	29
31	30	html_filters.push 'python/entries_v3', 'sphinx/clean_html', 'python/clean_html'
32	31	end
33	32
34		− version '3.6' do
	33	+ version '3.6' do # docs.python.org/3.6/download.html
35	34	self.release = '3.6.6'
36		− self.dir = '/Users/Thibaut/DevDocs/Docs/Python36' # docs.python.org/3.6/download.html
37	35	self.base_url = 'https://docs.python.org/3.6/'
38	36
39	37	html_filters.push 'python/entries_v3', 'sphinx/clean_html', 'python/clean_html'
40	38	end
41	39
42		− version '3.5' do
	40	+ version '3.5' do # docs.python.org/3.5/download.html
43	41	self.release = '3.5.3'
44		− self.dir = '/Users/Thibaut/DevDocs/Docs/Python35' # docs.python.org/3.5/download.html
45	42	self.base_url = 'https://docs.python.org/3.5/'
46	43
47	44	html_filters.push 'python/entries_v3', 'sphinx/clean_html', 'python/clean_html'
48	45	end
49	46
50		− version '2.7' do
	47	+ version '2.7' do # docs.python.org/2.7/download.html
51	48	self.release = '2.7.13'
52		− self.dir = '/Users/Thibaut/DevDocs/Docs/Python27' # docs.python.org/2.7/download.html
53	49	self.base_url = 'https://docs.python.org/2.7/'
54	50
55	51	html_filters.push 'python/entries_v2', 'sphinx/clean_html', 'python/clean_html'

lib/docs/scrapers/rdoc/minitest.rb+1−1

@@ -1,9 +1,9 @@
1	1	module Docs
2	2	class Minitest < Rdoc
	3	+ # Run "rake docs" in the gem directory
3	4	self.name = 'Ruby / Minitest'
4	5	self.slug = 'minitest'
5	6	self.release = '5.10.3'
6		− self.dir = '/Users/Thibaut/DevDocs/Docs/RDoc/Minitest' # rake docs
7	7	self.links = {
8	8	code: 'https://github.com/seattlerb/minitest'
9	9	}

lib/docs/scrapers/rdoc/rails.rb+0−1

@@ -4,7 +4,6 @@ module Docs · 3 unmodified lines
4	4
5	5	self.name = 'Ruby on Rails'
6	6	self.slug = 'rails'
7		− self.dir = '/Users/Thibaut/DevDocs/Docs/RDoc/Rails'
8	7	self.initial_paths = %w(guides/index.html)
9	8	self.links = {
10	9	home: 'http://rubyonrails.org/',

lib/docs/scrapers/rdoc/ruby.rb+0−4

@@ -78,22 +78,18 @@ module Docs · 77 unmodified lines
78	78
79	79	version '2.5' do
80	80	self.release = '2.5.0'
81		− self.dir = '/Users/Thibaut/DevDocs/Docs/RDoc/Ruby25'
82	81	end
83	82
84	83	version '2.4' do
85	84	self.release = '2.4.3'
86		− self.dir = '/Users/Thibaut/DevDocs/Docs/RDoc/Ruby24'
87	85	end
88	86
89	87	version '2.3' do
90	88	self.release = '2.3.6'
91		− self.dir = '/Users/Thibaut/DevDocs/Docs/RDoc/Ruby23'
92	89	end
93	90
94	91	version '2.2' do
95	92	self.release = '2.2.9'
96		− self.dir = '/Users/Thibaut/DevDocs/Docs/RDoc/Ruby22'
97	93	end
98	94	end
99	95	end

lib/docs/scrapers/sqlite.rb+0−1

@@ -3,7 +3,6 @@ module Docs · 2 unmodified lines
3	3	self.name = 'SQLite'
4	4	self.type = 'sqlite'
5	5	self.release = '3.25.2'
6		− self.dir = '/Users/Thibaut/DevDocs/Docs/sqlite/'
7	6	self.base_url = 'https://sqlite.org/'
8	7	self.root_path = 'docs.html'
9	8	self.initial_paths = %w(keyword_index.html)