History

Simplify file scraper setup; scrape files in the "docs/[slug]" directory

Thibaut Courouble committed 7 years ago

Commit bf003661 parent 91753ce22 files · +124 −104Browse files

Unified · Split

22 files
  1. .gitignore +1 −0
  2. lib/docs.rb +1 −0
  3. lib/docs/core/doc.rb +6 −0
  4. lib/docs/core/scrapers/file_scraper.rb +18 −7
  5. lib/docs/scrapers/c.rb +0 −1
  6. lib/docs/scrapers/cpp.rb +0 −1
  7. lib/docs/scrapers/dart.rb +0 −2
  8. lib/docs/scrapers/django.rb +0 −6
  9. lib/docs/scrapers/erlang.rb +0 −4
  10. lib/docs/scrapers/gnu/gcc.rb +0 −8
  11. lib/docs/scrapers/gnu/gnu_fortran.rb +0 −4
  12. lib/docs/scrapers/nokogiri2.rb +0 −1
  13. lib/docs/scrapers/numpy.rb +0 −1
  14. lib/docs/scrapers/openjdk.rb +2 −3
  15. lib/docs/scrapers/perl.rb +0 −1
  16. lib/docs/scrapers/php.rb +2 −3
  17. lib/docs/scrapers/python.rb +4 −8
  18. lib/docs/scrapers/rdoc/minitest.rb +1 −1
  19. lib/docs/scrapers/rdoc/rails.rb +0 −1
  20. lib/docs/scrapers/rdoc/ruby.rb +0 −4
  21. lib/docs/scrapers/sqlite.rb +0 −1
  22. test/lib/docs/core/scrapers/file_scraper_test.rb +89 −47
.gitignore+1−0
@@ -8,3 +8,4 @@ public/fonts · 7 unmodified lines
8public/docs/**/*8public/docs/**/*
9!public/docs/docs.json9!public/docs/docs.json
10!public/docs/**/index.json10!public/docs/**/index.json
11/docs/
lib/docs.rb+1−0
@@ -29,6 +29,7 @@ module Docs · 28 unmodified lines
29 self.rescue_errors = false29 self.rescue_errors = false
30 30
31 class DocNotFound < NameError; end31 class DocNotFound < NameError; end
32 class SetupError < StandardError; end
32 33
33 def self.all34 def self.all
34 Dir["#{root_path}/docs/scrapers/**/*.rb"].35 Dir["#{root_path}/docs/scrapers/**/*.rb"].
lib/docs/core/doc.rb+6−0
@@ -95,6 +95,9 @@ module Docs · 94 unmodified lines
95 false95 false
96 end96 end
97 end97 end
98 rescue Docs::SetupError => error
99 puts "ERROR: #{error.message}"
100 false
98 end101 end
99 102
100 def store_pages(store)103 def store_pages(store)
@@ -118,6 +121,9 @@ module Docs · 17 unmodified lines
118 false121 false
119 end122 end
120 end123 end
124 rescue Docs::SetupError => error
125 puts "ERROR: #{error.message}"
126 false
121 end127 end
122 128
123 private129 private
lib/docs/core/scrapers/file_scraper.rb+18−7
@@ -1,14 +1,13 @@
1module Docs1module Docs
2 class FileScraper < Scraper2 class FileScraper < Scraper
3 SOURCE_DIRECTORY = File.expand_path '../../../../../docs', __FILE__
4
3 Response = Struct.new :body, :url5 Response = Struct.new :body, :url
4 6
5 class << self7 class << self
6 attr_accessor :dir
7
8 def inherited(subclass)8 def inherited(subclass)
9 super9 super
10 subclass.base_url = base_url10 subclass.base_url = base_url
11 subclass.dir = dir
12 end11 end
13 end12 end
14 13
@@ -16,13 +15,25 @@ module Docs · 1 unmodified line
16 15
17 html_filters.push 'clean_local_urls'16 html_filters.push 'clean_local_urls'
18 17
18 def source_directory
19 @source_directory ||= File.join(SOURCE_DIRECTORY, self.class.path)
20 end
21
19 private22 private
20 23
24 def assert_source_directory_exists
25 unless Dir.exists?(source_directory)
26 raise SetupError, "The #{self.class.name} scraper requires the original documentation files to be stored in the \"#{source_directory}\" directory."
27 end
28 end
29
21 def request_one(url)30 def request_one(url)
22 Response.new read_file(file_path_for(url)), URL.parse(url)31 assert_source_directory_exists
32 Response.new read_file(url_to_path(url)), URL.parse(url)
23 end33 end
24 34
25 def request_all(urls)35 def request_all(urls)
36 assert_source_directory_exists
26 queue = [urls].flatten37 queue = [urls].flatten
27 until queue.empty?38 until queue.empty?
28 result = yield request_one(queue.shift)39 result = yield request_one(queue.shift)
@@ -34,12 +45,12 @@ module Docs · 5 unmodified lines
34 response.body.present?45 response.body.present?
35 end46 end
36 47
37 def file_path_for(url)48 def url_to_path(url)
38 File.join self.class.dir, url.remove(base_url.to_s)49 url.remove(base_url.to_s)
39 end50 end
40 51
41 def read_file(path)52 def read_file(path)
42 File.read(path)53 File.read(File.join(source_directory, path))
43 rescue54 rescue
44 instrument 'warn.doc', msg: "Failed to open file: #{path}"55 instrument 'warn.doc', msg: "Failed to open file: #{path}"
45 nil56 nil
lib/docs/scrapers/c.rb+0−1
@@ -1,7 +1,6 @@
1module Docs1module Docs
2 class C < FileScraper2 class C < FileScraper
3 self.type = 'c'3 self.type = 'c'
4 self.dir = '/Users/Thibaut/DevDocs/Docs/c'
5 self.base_url = 'http://en.cppreference.com/w/c/'4 self.base_url = 'http://en.cppreference.com/w/c/'
6 self.root_path = 'header.html'5 self.root_path = 'header.html'
7 6
lib/docs/scrapers/cpp.rb+0−1
@@ -3,7 +3,6 @@ module Docs · 2 unmodified lines
3 self.name = 'C++'3 self.name = 'C++'
4 self.slug = 'cpp'4 self.slug = 'cpp'
5 self.type = 'c'5 self.type = 'c'
6 self.dir = '/Users/Thibaut/DevDocs/Docs/cpp'
7 self.base_url = 'http://en.cppreference.com/w/cpp/'6 self.base_url = 'http://en.cppreference.com/w/cpp/'
8 self.root_path = 'header.html'7 self.root_path = 'header.html'
9 8
lib/docs/scrapers/dart.rb+0−2
@@ -24,13 +24,11 @@ module Docs · 23 unmodified lines
24 24
25 version '2' do25 version '2' do
26 self.release = '2.0.0'26 self.release = '2.0.0'
27 self.dir = '/Users/Thibaut/DevDocs/Docs/Dart2'
28 self.base_url = "https://api.dartlang.org/stable/#{release}/"27 self.base_url = "https://api.dartlang.org/stable/#{release}/"
29 end28 end
30 29
31 version '1' do30 version '1' do
32 self.release = '1.24.3'31 self.release = '1.24.3'
33 self.dir = '/Users/Thibaut/DevDocs/Docs/Dart1'
34 self.base_url = "https://api.dartlang.org/stable/#{release}/"32 self.base_url = "https://api.dartlang.org/stable/#{release}/"
35 end33 end
36 end34 end
lib/docs/scrapers/django.rb+0−6
@@ -36,37 +36,31 @@ module Docs · 35 unmodified lines
36 36
37 version '2.1' do37 version '2.1' do
38 self.release = '2.1.0'38 self.release = '2.1.0'
39 self.dir = '/Users/Thibaut/DevDocs/Docs/Django21'
40 self.base_url = 'https://docs.djangoproject.com/en/2.1/'39 self.base_url = 'https://docs.djangoproject.com/en/2.1/'
41 end40 end
42 41
43 version '2.0' do42 version '2.0' do
44 self.release = '2.0.7'43 self.release = '2.0.7'
45 self.dir = '/Users/Thibaut/DevDocs/Docs/Django20'
46 self.base_url = 'https://docs.djangoproject.com/en/2.0/'44 self.base_url = 'https://docs.djangoproject.com/en/2.0/'
47 end45 end
48 46
49 version '1.11' do47 version '1.11' do
50 self.release = '1.11.9'48 self.release = '1.11.9'
51 self.dir = '/Users/Thibaut/DevDocs/Docs/Django111'
52 self.base_url = 'https://docs.djangoproject.com/en/1.11/'49 self.base_url = 'https://docs.djangoproject.com/en/1.11/'
53 end50 end
54 51
55 version '1.10' do52 version '1.10' do
56 self.release = '1.10.8'53 self.release = '1.10.8'
57 self.dir = '/Users/Thibaut/DevDocs/Docs/Django110'
58 self.base_url = 'https://docs.djangoproject.com/en/1.10/'54 self.base_url = 'https://docs.djangoproject.com/en/1.10/'
59 end55 end
60 56
61 version '1.9' do57 version '1.9' do
62 self.release = '1.9.13'58 self.release = '1.9.13'
63 self.dir = '/Users/Thibaut/DevDocs/Docs/Django19'
64 self.base_url = 'https://docs.djangoproject.com/en/1.9/'59 self.base_url = 'https://docs.djangoproject.com/en/1.9/'
65 end60 end
66 61
67 version '1.8' do62 version '1.8' do
68 self.release = '1.8.18'63 self.release = '1.8.18'
69 self.dir = '/Users/Thibaut/DevDocs/Docs/Django18'
70 self.base_url = 'https://docs.djangoproject.com/en/1.8/'64 self.base_url = 'https://docs.djangoproject.com/en/1.8/'
71 end65 end
72 end66 end
lib/docs/scrapers/erlang.rb+0−4
@@ -42,22 +42,18 @@ module Docs · 41 unmodified lines
42 42
43 version '21' do43 version '21' do
44 self.release = '21.0'44 self.release = '21.0'
45 self.dir = '/Users/Thibaut/DevDocs/Docs/Erlang21'
46 end45 end
47 46
48 version '20' do47 version '20' do
49 self.release = '20.3'48 self.release = '20.3'
50 self.dir = '/Users/Thibaut/DevDocs/Docs/Erlang20'
51 end49 end
52 50
53 version '19' do51 version '19' do
54 self.release = '19.3'52 self.release = '19.3'
55 self.dir = '/Users/Thibaut/DevDocs/Docs/Erlang19'
56 end53 end
57 54
58 version '18' do55 version '18' do
59 self.release = '18.3'56 self.release = '18.3'
60 self.dir = '/Users/Thibaut/DevDocs/Docs/Erlang18'
61 end57 end
62 end58 end
63end59end
lib/docs/scrapers/gnu/gcc.rb+0−8
@@ -48,13 +48,11 @@ module Docs · 47 unmodified lines
48 48
49 version '7' do49 version '7' do
50 self.release = '7.3.0'50 self.release = '7.3.0'
51 self.dir = '/Users/Thibaut/DevDocs/Docs/gcc7'
52 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gcc/"51 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gcc/"
53 end52 end
54 53
55 version '7 CPP' do54 version '7 CPP' do
56 self.release = '7.3.0'55 self.release = '7.3.0'
57 self.dir = '/Users/Thibaut/DevDocs/Docs/gcpp7'
58 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/cpp/"56 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/cpp/"
59 57
60 options[:replace_paths] = CPP_PATHS58 options[:replace_paths] = CPP_PATHS
@@ -62,7 +60,6 @@ module Docs · 1 unmodified line
62 60
63 version '6' do61 version '6' do
64 self.release = '6.4.0'62 self.release = '6.4.0'
65 self.dir = '/Users/Thibaut/DevDocs/Docs/gcc6'
66 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gcc/"63 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gcc/"
67 64
68 options[:root_title] = 'Using the GNU Compiler Collection (GCC)'65 options[:root_title] = 'Using the GNU Compiler Collection (GCC)'
@@ -70,7 +67,6 @@ module Docs · 1 unmodified line
70 67
71 version '6 CPP' do68 version '6 CPP' do
72 self.release = '6.4.0'69 self.release = '6.4.0'
73 self.dir = '/Users/Thibaut/DevDocs/Docs/gcpp6'
74 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/cpp/"70 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/cpp/"
75 71
76 options[:replace_paths] = CPP_PATHS72 options[:replace_paths] = CPP_PATHS
@@ -78,7 +74,6 @@ module Docs · 1 unmodified line
78 74
79 version '5' do75 version '5' do
80 self.release = '5.4.0'76 self.release = '5.4.0'
81 self.dir = '/Users/Thibaut/DevDocs/Docs/gcc5'
82 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gcc/"77 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gcc/"
83 78
84 options[:root_title] = 'Using the GNU Compiler Collection (GCC)'79 options[:root_title] = 'Using the GNU Compiler Collection (GCC)'
@@ -86,7 +81,6 @@ module Docs · 1 unmodified line
86 81
87 version '5 CPP' do82 version '5 CPP' do
88 self.release = '5.4.0'83 self.release = '5.4.0'
89 self.dir = '/Users/Thibaut/DevDocs/Docs/gcpp5'
90 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/cpp/"84 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/cpp/"
91 85
92 options[:replace_paths] = CPP_PATHS86 options[:replace_paths] = CPP_PATHS
@@ -94,7 +88,6 @@ module Docs · 1 unmodified line
94 88
95 version '4' do89 version '4' do
96 self.release = '4.9.3'90 self.release = '4.9.3'
97 self.dir = '/Users/Thibaut/DevDocs/Docs/gcc4'
98 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gcc/"91 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gcc/"
99 92
100 options[:root_title] = 'Using the GNU Compiler Collection (GCC)'93 options[:root_title] = 'Using the GNU Compiler Collection (GCC)'
@@ -102,7 +95,6 @@ module Docs · 1 unmodified line
102 95
103 version '4 CPP' do96 version '4 CPP' do
104 self.release = '4.9.3'97 self.release = '4.9.3'
105 self.dir = '/Users/Thibaut/DevDocs/Docs/gcpp4'
106 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/cpp/"98 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/cpp/"
107 99
108 options[:replace_paths] = CPP_PATHS100 options[:replace_paths] = CPP_PATHS
lib/docs/scrapers/gnu/gnu_fortran.rb+0−4
@@ -8,25 +8,21 @@ module Docs · 7 unmodified lines
8 8
9 version '7' do9 version '7' do
10 self.release = '7.3.0'10 self.release = '7.3.0'
11 self.dir = '/Users/Thibaut/DevDocs/Docs/gfortran7'
12 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"11 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
13 end12 end
14 13
15 version '6' do14 version '6' do
16 self.release = '6.4.0'15 self.release = '6.4.0'
17 self.dir = '/Users/Thibaut/DevDocs/Docs/gfortran6'
18 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"16 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
19 end17 end
20 18
21 version '5' do19 version '5' do
22 self.release = '5.4.0'20 self.release = '5.4.0'
23 self.dir = '/Users/Thibaut/DevDocs/Docs/gfortran5'
24 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"21 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
25 end22 end
26 23
27 version '4' do24 version '4' do
28 self.release = '4.9.3'25 self.release = '4.9.3'
29 self.dir = '/Users/Thibaut/DevDocs/Docs/gfortran4'
30 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"26 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
31 end27 end
32 end28 end
lib/docs/scrapers/nokogiri2.rb+0−1
@@ -3,7 +3,6 @@ module Docs · 2 unmodified lines
3 self.name = 'Nokogiri'3 self.name = 'Nokogiri'
4 self.slug = 'nokogiri'4 self.slug = 'nokogiri'
5 self.release = '1.8.1'5 self.release = '1.8.1'
6 self.dir = '/Users/Thibaut/DevDocs/Docs/RDoc/Nokogiri'
7 6
8 html_filters.replace 'rdoc/entries', 'nokogiri2/entries'7 html_filters.replace 'rdoc/entries', 'nokogiri2/entries'
9 8
lib/docs/scrapers/numpy.rb+0−1
@@ -2,7 +2,6 @@ module Docs · 1 unmodified line
2 class Numpy < FileScraper2 class Numpy < FileScraper
3 self.name = 'NumPy'3 self.name = 'NumPy'
4 self.type = 'sphinx'4 self.type = 'sphinx'
5 self.dir = '/Users/Thibaut/DevDocs/Docs/numpy/reference/'
6 self.root_path = 'index.html'5 self.root_path = 'index.html'
7 self.links = {6 self.links = {
8 home: 'http://www.numpy.org/',7 home: 'http://www.numpy.org/',
lib/docs/scrapers/openjdk.rb+2−3
@@ -1,11 +1,10 @@
1module Docs1module Docs
2 class Openjdk < FileScraper2 class Openjdk < FileScraper
3 # Downloaded from packages.debian.org/sid/openjdk-8-doc
4 # Extracting subdirectory /usr/share/doc/openjdk-8-jre-headless/api
3 self.name = 'OpenJDK'5 self.name = 'OpenJDK'
4 self.type = 'openjdk'6 self.type = 'openjdk'
5 self.root_path = 'overview-summary.html'7 self.root_path = 'overview-summary.html'
6 # Downloaded from packages.debian.org/sid/openjdk-8-doc
7 # Extracting subdirectory /usr/share/doc/openjdk-8-jre-headless/api
8 self.dir = '/Users/Thibaut/DevDocs/Docs/OpenJDK'
9 8
10 html_filters.insert_after 'internal_urls', 'openjdk/clean_urls'9 html_filters.insert_after 'internal_urls', 'openjdk/clean_urls'
11 html_filters.push 'openjdk/entries', 'openjdk/clean_html'10 html_filters.push 'openjdk/entries', 'openjdk/clean_html'
lib/docs/scrapers/perl.rb+0−1
@@ -2,7 +2,6 @@ module Docs · 1 unmodified line
2 class Perl < FileScraper2 class Perl < FileScraper
3 self.name = 'Perl'3 self.name = 'Perl'
4 self.type = 'perl'4 self.type = 'perl'
5 self.dir = '/Users/Thibaut/DevDocs/Docs/Perl'
6 self.root_path = 'index.html'5 self.root_path = 'index.html'
7 self.links = {6 self.links = {
8 home: 'https://www.perl.org/'7 home: 'https://www.perl.org/'
lib/docs/scrapers/php.rb+2−3
@@ -1,5 +1,7 @@
1module Docs1module Docs
2 class Php < FileScraper2 class Php < FileScraper
3 # Downloaded from php.net/download-docs.php
4
3 include FixInternalUrlsBehavior5 include FixInternalUrlsBehavior
4 6
5 self.name = 'PHP'7 self.name = 'PHP'
@@ -23,9 +25,6 @@ module Docs · 17 unmodified lines
23 code: 'https://git.php.net/?p=php-src.git;a=summary'25 code: 'https://git.php.net/?p=php-src.git;a=summary'
24 }26 }
25 27
26 # Downloaded from php.net/download-docs.php
27 self.dir = '/Users/Thibaut/DevDocs/Docs/PHP'
28
29 html_filters.push 'php/internal_urls', 'php/entries', 'php/clean_html', 'title'28 html_filters.push 'php/internal_urls', 'php/entries', 'php/clean_html', 'title'
30 text_filters.push 'php/fix_urls'29 text_filters.push 'php/fix_urls'
31 30
lib/docs/scrapers/python.rb+4−8
@@ -23,33 +23,29 @@ module Docs · 22 unmodified lines
23 Licensed under the PSF License.23 Licensed under the PSF License.
24 HTML24 HTML
25 25
26 version '3.7' do26 version '3.7' do # docs.python.org/3.7/download.html
27 self.release = '3.7.0'27 self.release = '3.7.0'
28 self.dir = '/Users/Thibaut/DevDocs/Docs/Python37' # docs.python.org/3.7/download.html
29 self.base_url = 'https://docs.python.org/3.7/'28 self.base_url = 'https://docs.python.org/3.7/'
30 29
31 html_filters.push 'python/entries_v3', 'sphinx/clean_html', 'python/clean_html'30 html_filters.push 'python/entries_v3', 'sphinx/clean_html', 'python/clean_html'
32 end31 end
33 32
34 version '3.6' do33 version '3.6' do # docs.python.org/3.6/download.html
35 self.release = '3.6.6'34 self.release = '3.6.6'
36 self.dir = '/Users/Thibaut/DevDocs/Docs/Python36' # docs.python.org/3.6/download.html
37 self.base_url = 'https://docs.python.org/3.6/'35 self.base_url = 'https://docs.python.org/3.6/'
38 36
39 html_filters.push 'python/entries_v3', 'sphinx/clean_html', 'python/clean_html'37 html_filters.push 'python/entries_v3', 'sphinx/clean_html', 'python/clean_html'
40 end38 end
41 39
42 version '3.5' do40 version '3.5' do # docs.python.org/3.5/download.html
43 self.release = '3.5.3'41 self.release = '3.5.3'
44 self.dir = '/Users/Thibaut/DevDocs/Docs/Python35' # docs.python.org/3.5/download.html
45 self.base_url = 'https://docs.python.org/3.5/'42 self.base_url = 'https://docs.python.org/3.5/'
46 43
47 html_filters.push 'python/entries_v3', 'sphinx/clean_html', 'python/clean_html'44 html_filters.push 'python/entries_v3', 'sphinx/clean_html', 'python/clean_html'
48 end45 end
49 46
50 version '2.7' do47 version '2.7' do # docs.python.org/2.7/download.html
51 self.release = '2.7.13'48 self.release = '2.7.13'
52 self.dir = '/Users/Thibaut/DevDocs/Docs/Python27' # docs.python.org/2.7/download.html
53 self.base_url = 'https://docs.python.org/2.7/'49 self.base_url = 'https://docs.python.org/2.7/'
54 50
55 html_filters.push 'python/entries_v2', 'sphinx/clean_html', 'python/clean_html'51 html_filters.push 'python/entries_v2', 'sphinx/clean_html', 'python/clean_html'
lib/docs/scrapers/rdoc/minitest.rb+1−1
@@ -1,9 +1,9 @@
1module Docs1module Docs
2 class Minitest < Rdoc2 class Minitest < Rdoc
3 # Run "rake docs" in the gem directory
3 self.name = 'Ruby / Minitest'4 self.name = 'Ruby / Minitest'
4 self.slug = 'minitest'5 self.slug = 'minitest'
5 self.release = '5.10.3'6 self.release = '5.10.3'
6 self.dir = '/Users/Thibaut/DevDocs/Docs/RDoc/Minitest' # rake docs
7 self.links = {7 self.links = {
8 code: 'https://github.com/seattlerb/minitest'8 code: 'https://github.com/seattlerb/minitest'
9 }9 }
lib/docs/scrapers/rdoc/rails.rb+0−1
@@ -4,7 +4,6 @@ module Docs · 3 unmodified lines
4 4
5 self.name = 'Ruby on Rails'5 self.name = 'Ruby on Rails'
6 self.slug = 'rails'6 self.slug = 'rails'
7 self.dir = '/Users/Thibaut/DevDocs/Docs/RDoc/Rails'
8 self.initial_paths = %w(guides/index.html)7 self.initial_paths = %w(guides/index.html)
9 self.links = {8 self.links = {
10 home: 'http://rubyonrails.org/',9 home: 'http://rubyonrails.org/',
lib/docs/scrapers/rdoc/ruby.rb+0−4
@@ -78,22 +78,18 @@ module Docs · 77 unmodified lines
78 78
79 version '2.5' do79 version '2.5' do
80 self.release = '2.5.0'80 self.release = '2.5.0'
81 self.dir = '/Users/Thibaut/DevDocs/Docs/RDoc/Ruby25'
82 end81 end
83 82
84 version '2.4' do83 version '2.4' do
85 self.release = '2.4.3'84 self.release = '2.4.3'
86 self.dir = '/Users/Thibaut/DevDocs/Docs/RDoc/Ruby24'
87 end85 end
88 86
89 version '2.3' do87 version '2.3' do
90 self.release = '2.3.6'88 self.release = '2.3.6'
91 self.dir = '/Users/Thibaut/DevDocs/Docs/RDoc/Ruby23'
92 end89 end
93 90
94 version '2.2' do91 version '2.2' do
95 self.release = '2.2.9'92 self.release = '2.2.9'
96 self.dir = '/Users/Thibaut/DevDocs/Docs/RDoc/Ruby22'
97 end93 end
98 end94 end
99end95end
lib/docs/scrapers/sqlite.rb+0−1
@@ -3,7 +3,6 @@ module Docs · 2 unmodified lines
3 self.name = 'SQLite'3 self.name = 'SQLite'
4 self.type = 'sqlite'4 self.type = 'sqlite'
5 self.release = '3.25.2'5 self.release = '3.25.2'
6 self.dir = '/Users/Thibaut/DevDocs/Docs/sqlite/'
7 self.base_url = 'https://sqlite.org/'6 self.base_url = 'https://sqlite.org/'
8 self.root_path = 'docs.html'7 self.root_path = 'docs.html'
9 self.initial_paths = %w(keyword_index.html)8 self.initial_paths = %w(keyword_index.html)
test/lib/docs/core/scrapers/file_scraper_test.rb+89−47
@@ -2,16 +2,23 @@ require 'test_helper' · 1 unmodified line
2require 'docs'2require 'docs'
3 3
4class FileScraperTest < MiniTest::Spec4class FileScraperTest < MiniTest::Spec
5 ROOT_PATH = File.expand_path('../../../../../../', __FILE__)
6
5 class Scraper < Docs::FileScraper7 class Scraper < Docs::FileScraper
6 self.dir = '/'
7 self.html_filters = Docs::FilterStack.new8 self.html_filters = Docs::FilterStack.new
8 self.text_filters = Docs::FilterStack.new9 self.text_filters = Docs::FilterStack.new
10
11 version 'version' do; end
9 end12 end
10 13
11 let :scraper do14 let :scraper do
12 Scraper.new15 Scraper.new
13 end16 end
14 17
18 let :versioned_scraper do
19 Scraper.versions.first.new
20 end
21
15 let :response do22 let :response do
16 OpenStruct.new body: 'body', url: Docs::URL.parse(Scraper.base_url)23 OpenStruct.new body: 'body', url: Docs::URL.parse(Scraper.base_url)
17 end24 end
@@ -22,9 +29,16 @@ class FileScraperTest < MiniTest::Spec · 4 unmodified lines
22 end29 end
23 end30 end
24 31
32 describe "#source_directory" do
33 it "returns the directory at docs/[slug]" do
34 assert_equal File.join(ROOT_PATH, 'docs', 'scraper'), scraper.source_directory
35 assert_equal File.join(ROOT_PATH, 'docs', 'scraper~version'), versioned_scraper.source_directory
36 end
37 end
38
25 describe "#request_one" do39 describe "#request_one" do
26 let :path do40 let :path do
27 File.join(Scraper.dir, 'path')41 'path'
28 end42 end
29 43
30 let :result do44 let :result do
@@ -35,20 +49,34 @@ class FileScraperTest < MiniTest::Spec · 4 unmodified lines
35 stub(scraper).read_file49 stub(scraper).read_file
36 end50 end
37 51
38 it "reads a file" do52 context "when the source directory doesn't exist" do
39 mock(scraper).read_file(path)53 it "raises an error" do
40 result54 assert_raises Docs::SetupError do
55 result
56 end
57 end
41 end58 end
42 59
43 describe "the returned response object" do60 context "when the source directory exists" do
44 it "has a #body" do61 before do
45 stub(scraper).read_file { 'body' }62 stub(scraper).assert_source_directory_exists
46 assert_equal 'body', result.body63 end
64
65 it "reads a file" do
66 mock(scraper).read_file(path)
67 result
47 end68 end
48 69
49 it "has a #url" do70 describe "the returned response object" do
50 assert_equal path, result.url.to_s71 it "has a #body" do
51 assert_instance_of Docs::URL, result.url72 stub(scraper).read_file { 'body' }
73 assert_equal 'body', result.body
74 end
75
76 it "has a #url" do
77 assert_equal path, result.url.to_s
78 assert_instance_of Docs::URL, result.url
79 end
52 end80 end
53 end81 end
54 end82 end
@@ -58,49 +86,63 @@ class FileScraperTest < MiniTest::Spec · 3 unmodified lines
58 %w(one two)86 %w(one two)
59 end87 end
60 88
61 it "requests the given url" do89 context "when the source directory doesn't exist" do
62 mock(scraper).request_one('url')90 it "raises an error" do
63 scraper.send(:request_all, 'url') {}91 assert_raises Docs::SetupError do
64 end92 scraper.send(:request_all, urls) {}
65 93 end
66 it "requests the given urls" do94 end
67 requests = []
68 stub(scraper).request_one { |url| requests << url; nil }
69 scraper.send(:request_all, urls) {}
70 assert_equal urls, requests
71 end
72
73 it "yields the responses" do
74 responses = []
75 stub(scraper).request_one { |url| urls.index(url) }
76 scraper.send(:request_all, urls) { |response| responses << response; nil }
77 assert_equal (0...urls.length).to_a, responses
78 end95 end
79 96
80 context "when the block returns an array" do97 context "when the source directory exists" do
81 let :next_urls do98 before do
82 %w(three four)99 stub(scraper).assert_source_directory_exists
83 end100 end
84 101
85 let :all_urls do102 it "requests the given url" do
86 urls + %w(three four)103 mock(scraper).request_one('url')
104 scraper.send(:request_all, 'url') {}
87 end105 end
88 106
89 it "requests the returned urls" do107 it "requests the given urls" do
90 requests = []108 requests = []
91 stub(scraper).request_one { |url| requests << url; url }109 stub(scraper).request_one { |url| requests << url; nil }
92 scraper.send(:request_all, urls) { [next_urls.shift].compact }110 scraper.send(:request_all, urls) {}
93 assert_equal all_urls, requests111 assert_equal urls, requests
94 end112 end
95 113
96 it "yields their responses" do114 it "yields the responses" do
97 responses = []115 responses = []
98 stub(scraper).request_one { |url| all_urls.index(url) }116 stub(scraper).request_one { |url| urls.index(url) }
99 scraper.send :request_all, urls do |response|117 scraper.send(:request_all, urls) { |response| responses << response; nil }
100 responses << response118 assert_equal (0...urls.length).to_a, responses
101 [next_urls.shift].compact119 end
120
121 context "when the block returns an array" do
122 let :next_urls do
123 %w(three four)
124 end
125
126 let :all_urls do
127 urls + %w(three four)
128 end
129
130 it "requests the returned urls" do
131 requests = []
132 stub(scraper).request_one { |url| requests << url; url }
133 scraper.send(:request_all, urls) { [next_urls.shift].compact }
134 assert_equal all_urls, requests
135 end
136
137 it "yields their responses" do
138 responses = []
139 stub(scraper).request_one { |url| all_urls.index(url) }
140 scraper.send :request_all, urls do |response|
141 responses << response
142 [next_urls.shift].compact
143 end
144 assert_equal (0...all_urls.length).to_a, responses
102 end145 end
103 assert_equal (0...all_urls.length).to_a, responses
104 end146 end
105 end147 end
106 end148 end
@@ -126,13 +168,13 @@ class FileScraperTest < MiniTest::Spec · 19 unmodified lines
126 scraper.send :read_file, 'file'168 scraper.send :read_file, 'file'
127 end169 end
128 170
129 it "returns the file's content when the file exists" do171 it "returns the file's content when the file exists in the source directory" do
130 stub(File).read('file') { 'content' }172 stub(File).read(File.join(ROOT_PATH, 'docs', 'scraper', 'file')) { 'content' }
131 assert_equal 'content', result173 assert_equal 'content', result
132 end174 end
133 175
134 it "returns nil when the file doesn't exist" do176 it "returns nil when the file doesn't exist" do
135 stub(File).read('file') { raise }177 stub(File).read(File.join(ROOT_PATH, 'docs', 'scraper', 'file')) { raise }
136 assert_nil result178 assert_nil result
137 end179 end
138 end180 end