History

Merge branch 'master' into master

Simon Legner committed 6 years ago

Commit 44eb7ec2 parents 830af65 0f8a44651 files · +257 −94Browse files

Showing the changes to lib/docs (43 of 51 files). Show all

Unified · Split

43 files
  1. lib/docs/core/doc.rb +2 −0
  2. lib/docs/filters/ansible/entries.rb +13 −0
  3. lib/docs/filters/haxe/clean_html.rb +4 −2
  4. lib/docs/filters/jest/entries.rb +1 −0
  5. lib/docs/filters/leaflet/clean_html.rb +7 −7
  6. lib/docs/filters/openjdk/clean_html.rb +5 −2
  7. lib/docs/scrapers/angular.rb +62 −2
  8. lib/docs/scrapers/angularjs.rb +7 −2
  9. lib/docs/scrapers/ansible.rb +9 −4
  10. lib/docs/scrapers/babel.rb +1 −2
  11. lib/docs/scrapers/chef.rb +2 −2
  12. lib/docs/scrapers/codeigniter.rb +2 −3
  13. lib/docs/scrapers/dart.rb +1 −1
  14. lib/docs/scrapers/docker.rb +3 −3
  15. lib/docs/scrapers/elixir.rb +1 −1
  16. lib/docs/scrapers/erlang.rb +1 −2
  17. lib/docs/scrapers/fish.rb +1 −2
  18. lib/docs/scrapers/gnu_cobol.rb +5 −3
  19. lib/docs/scrapers/gnuplot.rb +5 −0
  20. lib/docs/scrapers/go.rb +1 −1
  21. lib/docs/scrapers/godot.rb +1 −2
  22. lib/docs/scrapers/haxe.rb +7 −3
  23. lib/docs/scrapers/influxdata.rb +1 −3
  24. lib/docs/scrapers/jest.rb +2 −2
  25. lib/docs/scrapers/leaflet.rb +5 −0
  26. lib/docs/scrapers/lua.rb +7 −2
  27. lib/docs/scrapers/moment.rb +1 −2
  28. lib/docs/scrapers/nginx.rb +4 −3
  29. lib/docs/scrapers/node.rb +6 −1
  30. lib/docs/scrapers/openjdk.rb +8 −1
  31. lib/docs/scrapers/pandas.rb +1 −3
  32. lib/docs/scrapers/perl.rb +1 −2
  33. lib/docs/scrapers/php.rb +2 −3
  34. lib/docs/scrapers/postgresql.rb +6 −1
  35. lib/docs/scrapers/pug.rb +5 −1
  36. lib/docs/scrapers/python.rb +12 −4
  37. lib/docs/scrapers/rdoc/ruby.rb +4 −0
  38. lib/docs/scrapers/react.rb +1 −1
  39. lib/docs/scrapers/react_native.rb +2 −2
  40. lib/docs/scrapers/scikit_learn.rb +1 −2
  41. lib/docs/scrapers/sqlite.rb +1 −1
  42. lib/docs/scrapers/statsmodels.rb +1 −2
  43. lib/docs/scrapers/vagrant.rb +1 −2
lib/docs/core/doc.rb+2−0
@@ -220,6 +220,8 @@ module Docs · 219 unmodified lines
220 220
221 if opts.key?(:github_token) and url.start_with?('https://api.github.com/')221 if opts.key?(:github_token) and url.start_with?('https://api.github.com/')
222 headers['Authorization'] = "token #{opts[:github_token]}"222 headers['Authorization'] = "token #{opts[:github_token]}"
223 elsif ENV['GITHUB_TOKEN'] and url.start_with?('https://api.github.com/')
224 headers['Authorization'] = "token #{ENV['GITHUB_TOKEN']}"
223 end225 end
224 226
225 opts[:logger].debug("Fetching #{url}")227 opts[:logger].debug("Fetching #{url}")
lib/docs/filters/ansible/entries.rb+13−0
@@ -7,6 +7,13 @@ module Docs · 6 unmodified lines
7 name.remove! %r{ \- .*}7 name.remove! %r{ \- .*}
8 name.remove! 'Introduction To '8 name.remove! 'Introduction To '
9 name.remove! %r{ Guide\z}9 name.remove! %r{ Guide\z}
10
11 if version == "2.10"
12 if slug =~ /\Acollections\// and slug !~ /index$/
13 name = name.split('.')[2]
14 end
15 end
16
10 name17 name
11 end18 end
12 19
@@ -21,6 +28,12 @@ module Docs · 8 unmodified lines
21 end28 end
22 end29 end
23 30
31 if version == "2.10"
32 if slug =~ /\Acollections\//
33 return "Collection #{slug.split('/')[1..-2].join(".")}"
34 end
35 end
36
24 if slug =~ /\Acli\//37 if slug =~ /\Acli\//
25 'CLI Reference'38 'CLI Reference'
26 elsif slug =~ /\Anetwork\//39 elsif slug =~ /\Anetwork\//
lib/docs/filters/haxe/clean_html.rb+4−2
@@ -2,7 +2,7 @@ module Docs · 1 unmodified line
2 class Haxe2 class Haxe
3 class CleanHtmlFilter < Filter3 class CleanHtmlFilter < Filter
4 def call4 def call
5 css('.viewsource', 'hr', 'h1 > small', '.inherited-fields').remove5 css('.viewsource', 'hr', 'h1 > small', '.inherited-fields', '.label-meta').remove
6 6
7 css('h4 + h1').each do |node|7 css('h4 + h1').each do |node|
8 node.after(node.previous_element)8 node.after(node.previous_element)
@@ -28,8 +28,10 @@ module Docs · 19 unmodified lines
28 end28 end
29 29
30 css('.field').each do |node|30 css('.field').each do |node|
31 h3 = node.at_css('h3:not(:empty)')
32 next unless h3.present?
31 link = node.at_css('a[name]')33 link = node.at_css('a[name]')
32 node.at_css('h3:not(:empty)')['id'] = link['name']34 h3['id'] = link['name']
33 link.before(link.children).remove35 link.before(link.children).remove
34 node.before(node.children).remove36 node.before(node.children).remove
35 end37 end
lib/docs/filters/jest/entries.rb+1−0
@@ -19,6 +19,7 @@ module Docs · 18 unmodified lines
19 19
20 def additional_entries20 def additional_entries
21 return [] unless !root_page? && self.type == self.name # api page21 return [] unless !root_page? && self.type == self.name # api page
22 return [] if self.slug == 'environment-variables'
22 23
23 entries = []24 entries = []
24 25
lib/docs/filters/leaflet/clean_html.rb+7−7
@@ -8,25 +8,25 @@ module Docs · 7 unmodified lines
8 node.name = 'h2'8 node.name = 'h2'
9 end9 end
10 10
11 at_css('> h2:first-child').name = 'h1'11 # remove "This reference reflects Leaflet"
12 12 css('p:contains("This reference reflects Leaflet")').each do |node|
13 # remove "This reference reflects Leaflet 1.2.0."
14 css('h1 ~ p').each do |node|
15 node.remove13 node.remove
16 break14 break
17 end15 end
18 16
17 at_css('> h2:first-child').name = 'h1'
18
19 css('section', 'code b', '.accordion', '.accordion-overflow', '.accordion-content').each do |node|19 css('section', 'code b', '.accordion', '.accordion-overflow', '.accordion-content').each do |node|
20 node.before(node.children).remove20 node.before(node.children).remove
21 end21 end
22 22
23 css('pre > code').each do |node|23 css('pre > code').each do |node|
24 node['class'] ||= ''24 node['class'] ||= ''
25 lang = if node['class'].include?('lang-html') || node.content =~ /\A</25 lang = if node['class'].include?('lang-html') || node['class'].include?('language-html') || node.content =~ /\A</
26 'html'26 'html'
27 elsif node['class'].include?('lang-css')27 elsif node['class'].include?('lang-css') || node['class'].include?('language-css')
28 'css'28 'css'
29 elsif node['class'].include?('lang-js') || node['class'].include?('lang-javascript')29 elsif node['class'].include?('lang-js') || node['class'].include?('language-js') || node['class'].include?('lang-javascript')
30 'javascript'30 'javascript'
31 end31 end
32 node.parent['data-language'] = lang if lang32 node.parent['data-language'] = lang if lang
lib/docs/filters/openjdk/clean_html.rb+5−2
@@ -33,7 +33,7 @@ module Docs · 32 unmodified lines
33 node.content = node.content.remove(' Summary').remove(' Detail').pluralize33 node.content = node.content.remove(' Summary').remove(' Detail').pluralize
34 end34 end
35 35
36 if root_page?36 if root_page? && version == '8'
37 css('.header')[1].remove37 css('.header')[1].remove
38 css('.contentContainer')[0].remove38 css('.contentContainer')[0].remove
39 css('.contentContainer')[-1].remove39 css('.contentContainer')[-1].remove
@@ -45,7 +45,10 @@ module Docs · 5 unmodified lines
45 end45 end
46 end46 end
47 47
48 at_css('h1').content = "OpenJDK #{release} Documentation" + (version != release ? " (#{version.split(' ').last})" : '')48 end
49
50 if root_page?
51 at_css('h1').content = "OpenJDK #{release} Documentation"
49 end52 end
50 53
51 css('table').each do |node|54 css('table').each do |node|
lib/docs/scrapers/angular.rb+62−2
@@ -11,7 +11,7 @@ module Docs · 10 unmodified lines
11 options[:max_image_size] = 256_00011 options[:max_image_size] = 256_000
12 12
13 options[:attribution] = <<-HTML13 options[:attribution] = <<-HTML
14 &copy; 2010&ndash;2019 Google, Inc.<br>14 &copy; 2010&ndash;2020 Google, Inc.<br>
15 Licensed under the Creative Commons Attribution License 4.0.15 Licensed under the Creative Commons Attribution License 4.0.
16 HTML16 HTML
17 17
@@ -59,7 +59,7 @@ module Docs · 41 unmodified lines
59 end59 end
60 60
61 version do61 version do
62 self.release = '8.2.14'62 self.release = '11.0.0'
63 self.base_url = 'https://angular.io/'63 self.base_url = 'https://angular.io/'
64 self.root_path = 'docs'64 self.root_path = 'docs'
65 65
@@ -78,6 +78,66 @@ module Docs · 12 unmodified lines
78 include Docs::Angular::Common78 include Docs::Angular::Common
79 end79 end
80 80
81 version '10' do
82 self.release = '10.2.3'
83 self.base_url = 'https://v10.angular.io/'
84 self.root_path = 'docs'
85
86 html_filters.push 'angular/clean_html', 'angular/entries'
87
88 options[:follow_links] = false
89 options[:only_patterns] = [/\Aguide/, /\Atutorial/, /\Aapi/]
90 options[:fix_urls_before_parse] = ->(url) do
91 url.sub! %r{\Aguide/}, '/guide/'
92 url.sub! %r{\Atutorial/}, '/tutorial/'
93 url.sub! %r{\Aapi/}, '/api/'
94 url.sub! %r{\Agenerated/}, '/generated/'
95 url
96 end
97
98 include Docs::Angular::Common
99 end
100
101 version '9' do
102 self.release = '9.1.12'
103 self.base_url = 'https://v9.angular.io/'
104 self.root_path = 'docs'
105
106 html_filters.push 'angular/clean_html', 'angular/entries'
107
108 options[:follow_links] = false
109 options[:only_patterns] = [/\Aguide/, /\Atutorial/, /\Aapi/]
110 options[:fix_urls_before_parse] = ->(url) do
111 url.sub! %r{\Aguide/}, '/guide/'
112 url.sub! %r{\Atutorial/}, '/tutorial/'
113 url.sub! %r{\Aapi/}, '/api/'
114 url.sub! %r{\Agenerated/}, '/generated/'
115 url
116 end
117
118 include Docs::Angular::Common
119 end
120
121 version '8' do
122 self.release = '8.2.14'
123 self.base_url = 'https://v8.angular.io/'
124 self.root_path = 'docs'
125
126 html_filters.push 'angular/clean_html', 'angular/entries'
127
128 options[:follow_links] = false
129 options[:only_patterns] = [/\Aguide/, /\Atutorial/, /\Aapi/]
130 options[:fix_urls_before_parse] = ->(url) do
131 url.sub! %r{\Aguide/}, '/guide/'
132 url.sub! %r{\Atutorial/}, '/tutorial/'
133 url.sub! %r{\Aapi/}, '/api/'
134 url.sub! %r{\Agenerated/}, '/generated/'
135 url
136 end
137
138 include Docs::Angular::Common
139 end
140
81 version '7' do141 version '7' do
82 self.release = '7.2.15'142 self.release = '7.2.15'
83 self.base_url = 'https://v7.angular.io/'143 self.base_url = 'https://v7.angular.io/'
lib/docs/scrapers/angularjs.rb+7−2
@@ -38,8 +38,8 @@ module Docs · 37 unmodified lines
38 ]38 ]
39 39
40 options[:attribution] = <<-HTML40 options[:attribution] = <<-HTML
41 &copy; 2010&ndash;2018 Google, Inc.<br>41 &copy; 2010&ndash;2020 Google, Inc.<br>
42 Licensed under the Creative Commons Attribution License 4.0.42 Licensed under the Creative Commons Attribution License 3.0.
43 HTML43 HTML
44 44
45 stub '' do45 stub '' do
@@ -49,6 +49,11 @@ module Docs · 3 unmodified lines
49 capybara.execute_script("return document.querySelector('.side-navigation').innerHTML")49 capybara.execute_script("return document.querySelector('.side-navigation').innerHTML")
50 end50 end
51 51
52 version '1.8' do
53 self.release = '1.8.2'
54 self.base_url = "https://code.angularjs.org/#{release}/docs/partials/"
55 end
56
52 version '1.7' do57 version '1.7' do
53 self.release = '1.7.8'58 self.release = '1.7.8'
54 self.base_url = "https://code.angularjs.org/#{release}/docs/partials/"59 self.base_url = "https://code.angularjs.org/#{release}/docs/partials/"
lib/docs/scrapers/ansible.rb+9−4
@@ -30,18 +30,23 @@ module Docs · 29 unmodified lines
30 /\Aroadmap.*/i,30 /\Aroadmap.*/i,
31 ]31 ]
32 32
33 version '2.10' do
34 self.release = '2.10.3'
35 self.base_url = "https://docs.ansible.com/ansible/#{version}/"
36 end
37
33 version '2.9' do38 version '2.9' do
34 self.release = '2.9.1'39 self.release = '2.9.15'
35 self.base_url = "https://docs.ansible.com/ansible/#{version}/"40 self.base_url = "https://docs.ansible.com/ansible/#{version}/"
36 end41 end
37 42
38 version '2.8' do43 version '2.8' do
39 self.release = '2.8.7'44 self.release = '2.8.16'
40 self.base_url = "https://docs.ansible.com/ansible/#{version}/"45 self.base_url = "https://docs.ansible.com/ansible/#{version}/"
41 end46 end
42 47
43 version '2.7' do48 version '2.7' do
44 self.release = '2.7.15'49 self.release = '2.7.17'
45 self.base_url = "https://docs.ansible.com/ansible/#{version}/"50 self.base_url = "https://docs.ansible.com/ansible/#{version}/"
46 end51 end
47 52
@@ -71,7 +76,7 @@ module Docs · 23 unmodified lines
71 76
72 def get_latest_version(opts)77 def get_latest_version(opts)
73 doc = fetch_doc('https://docs.ansible.com/ansible/latest/index.html', opts)78 doc = fetch_doc('https://docs.ansible.com/ansible/latest/index.html', opts)
74 doc.at_css('.DocSiteProduct-CurrentVersion').content.strip79 doc.at_css('.version').content.strip
75 end80 end
76 end81 end
77end82end
lib/docs/scrapers/babel.rb+1−2
@@ -24,8 +24,7 @@ module Docs · 23 unmodified lines
24 end24 end
25 25
26 def get_latest_version(opts)26 def get_latest_version(opts)
27 doc = fetch_doc('https://babeljs.io/docs/en/', opts)27 get_latest_github_release('babel', 'babel', opts)
28 doc.at_css('a[href="/versions"] > h3').content
29 end28 end
30 end29 end
31end30end
lib/docs/scrapers/chef.rb+2−2
@@ -49,8 +49,8 @@ module Docs · 48 unmodified lines
49 end49 end
50 50
51 def get_latest_version(opts)51 def get_latest_version(opts)
52 doc = fetch_doc('https://downloads.chef.io/chef', opts)52 doc = fetch_doc('https://downloads.chef.io/products/infra', opts)
53 doc.at_css('h1.product-heading > span').content.strip53 doc.at_css('#versions > option').content.strip
54 end54 end
55 end55 end
56end56end
lib/docs/scrapers/codeigniter.rb+2−3
@@ -40,9 +40,8 @@ module Docs · 39 unmodified lines
40 end40 end
41 41
42 def get_latest_version(opts)42 def get_latest_version(opts)
43 doc = fetch_doc('https://codeigniter.com/userguide3/changelog.html', opts)43 tags = get_github_tags('codeigniter4', 'codeigniter4', opts)
44 header = doc.at_css('#change-log h2')44 tags[0]['name'][1..-1]
45 header.content.scan(/([0-9.]+)/)[0][0]
46 end45 end
47 end46 end
48end47end
lib/docs/scrapers/dart.rb+1−1
@@ -33,7 +33,7 @@ module Docs · 32 unmodified lines
33 def get_latest_version(opts)33 def get_latest_version(opts)
34 doc = fetch_doc('https://api.dartlang.org/', opts)34 doc = fetch_doc('https://api.dartlang.org/', opts)
35 label = doc.at_css('footer > span').content.strip35 label = doc.at_css('footer > span').content.strip
36 label.sub(/Dart /, '')36 label.sub(/Dart\s*/, '')
37 end37 end
38 end38 end
39end39end
lib/docs/scrapers/docker.rb+3−3
@@ -260,9 +260,9 @@ module Docs · 259 unmodified lines
260 end260 end
261 261
262 def get_latest_version(opts)262 def get_latest_version(opts)
263 doc = fetch_doc('https://docs.docker.com/', opts)263 doc = fetch_doc('https://docs.docker.com/engine/release-notes/', opts)
264 label = doc.at_css('.nav-container button.dropdown-toggle').content.strip264 latest_version = doc.at_css('.content > section > h1[id^="version-"]').content.strip
265 label.scan(/([0-9.]+)/)[0][0]265 latest_version.rpartition(' ')[-1]
266 end266 end
267 end267 end
268end268end
lib/docs/scrapers/elixir.rb+1−1
@@ -126,7 +126,7 @@ module Docs · 125 unmodified lines
126 126
127 def get_latest_version(opts)127 def get_latest_version(opts)
128 doc = fetch_doc('https://hexdocs.pm/elixir/api-reference.html', opts)128 doc = fetch_doc('https://hexdocs.pm/elixir/api-reference.html', opts)
129 doc.at_css('h2.sidebar-projectVersion').content.strip[1..-1]129 doc.at_css('.sidebar-projectVersion').content.strip[1..-1]
130 end130 end
131 end131 end
132end132end
lib/docs/scrapers/erlang.rb+1−2
@@ -57,8 +57,7 @@ module Docs · 56 unmodified lines
57 end57 end
58 58
59 def get_latest_version(opts)59 def get_latest_version(opts)
60 doc = fetch_doc('https://www.erlang.org/downloads', opts)60 get_latest_github_release('erlang', 'otp', opts)[4..-1]
61 doc.at_css('.col-lg-3 > ul > li').content.strip.sub(/OTP /, '')
62 end61 end
63 end62 end
64end63end
lib/docs/scrapers/fish.rb+1−2
@@ -54,8 +54,7 @@ module Docs · 53 unmodified lines
54 end54 end
55 55
56 def get_latest_version(opts)56 def get_latest_version(opts)
57 doc = fetch_doc('http://fishshell.com/docs/current/index.html', opts)57 get_latest_github_release('fish-shell', 'fish-shell', opts)
58 doc.at_css('#toc-index').content.scan(/([0-9.]+)/)[0][0]
59 end58 end
60 end59 end
61end60end
lib/docs/scrapers/gnu_cobol.rb+5−3
@@ -18,9 +18,11 @@ module Docs · 17 unmodified lines
18 HTML18 HTML
19 19
20 def get_latest_version(opts)20 def get_latest_version(opts)
21 doc = fetch_doc('https://open-cobol.sourceforge.io/HTML/gnucobpg.html', opts)21 fetch_doc('https://sourceforge.net/projects/gnucobol/files/gnucobol/', opts)
22 title = doc.at_css('h1').content22 .css('#files_list > tbody > tr')
23 title.scan(/([0-9.]+)/)[0][0]23 .map { |file| file['title'] }
24 .sort_by { |version| version.to_f }
25 .last
24 end26 end
25 end27 end
26end28end
lib/docs/scrapers/gnuplot.rb+5−0
@@ -37,5 +37,10 @@ module Docs · 36 unmodified lines
37 Distributed under the <a href="https://sourceforge.net/p/gnuplot/gnuplot-main/ci/master/tree/Copyright">gnuplot license</a> (rights to distribute modified versions are withheld).37 Distributed under the <a href="https://sourceforge.net/p/gnuplot/gnuplot-main/ci/master/tree/Copyright">gnuplot license</a> (rights to distribute modified versions are withheld).
38 HTML38 HTML
39 39
40 def get_latest_version(opts)
41 doc = fetch_doc('http://www.gnuplot.info/download.html', opts)
42 label = doc.at_css('h2').content.strip
43 label.sub(/[^0-9.]*/, '')
44 end
40 end45 end
41end46end
lib/docs/scrapers/go.rb+1−1
@@ -1,7 +1,7 @@
1module Docs1module Docs
2 class Go < UrlScraper2 class Go < UrlScraper
3 self.type = 'go'3 self.type = 'go'
4 self.release = '1.13'4 self.release = '1.15'
5 self.base_url = 'https://golang.org/pkg/'5 self.base_url = 'https://golang.org/pkg/'
6 self.links = {6 self.links = {
7 home: 'https://golang.org/',7 home: 'https://golang.org/',
lib/docs/scrapers/godot.rb+1−2
@@ -44,8 +44,7 @@ module Docs · 43 unmodified lines
44 end44 end
45 45
46 def get_latest_version(opts)46 def get_latest_version(opts)
47 doc = fetch_doc('https://docs.godotengine.org/', opts)47 get_latest_github_release('godotengine', 'godot', opts).split('-')[0]
48 doc.at_css('.version').content.strip
49 end48 end
50 end49 end
51end50end
lib/docs/scrapers/haxe.rb+7−3
@@ -2,7 +2,7 @@ module Docs · 1 unmodified line
2 class Haxe < UrlScraper2 class Haxe < UrlScraper
3 self.name = 'Haxe'3 self.name = 'Haxe'
4 self.type = 'simple'4 self.type = 'simple'
5 self.release = '3.4.7'5 self.release = '4.1.3'
6 self.base_url = 'https://api.haxe.org/'6 self.base_url = 'https://api.haxe.org/'
7 7
8 html_filters.push 'haxe/clean_html', 'haxe/entries'8 html_filters.push 'haxe/clean_html', 'haxe/entries'
@@ -10,7 +10,7 @@ module Docs · 1 unmodified line
10 options[:container] = '.span9'10 options[:container] = '.span9'
11 11
12 options[:attribution] = <<-HTML12 options[:attribution] = <<-HTML
13 &copy; 2005&ndash;2018 Haxe Foundation<br>13 &copy; 2005&ndash;2020 Haxe Foundation<br>
14 Licensed under a MIT license.14 Licensed under a MIT license.
15 HTML15 HTML
16 16
@@ -20,7 +20,7 @@ module Docs · 3 unmodified lines
20 code: 'https://github.com/HaxeFoundation/haxe'20 code: 'https://github.com/HaxeFoundation/haxe'
21 }21 }
22 22
23 options[:skip_patterns] = [/\A(?:cpp|cs|flash|java|js|neko|php|python|lua|hl|sys)/i]23 options[:skip_patterns] = [/\A(?:cpp|cs|flash|java|js|neko|php|python|lua|hl|sys|eval)/i]
24 end24 end
25 25
26 version 'C++' do26 version 'C++' do
@@ -67,6 +67,10 @@ module Docs · 40 unmodified lines
67 self.base_url = 'https://api.haxe.org/python/'67 self.base_url = 'https://api.haxe.org/python/'
68 end68 end
69 69
70 version 'Eval' do
71 self.base_url = 'https://api.haxe.org/eval/'
72 end
73
70 def get_latest_version(opts)74 def get_latest_version(opts)
71 doc = fetch_doc('https://api.haxe.org/', opts)75 doc = fetch_doc('https://api.haxe.org/', opts)
72 label = doc.at_css('.container.main-content h1 > small').content76 label = doc.at_css('.container.main-content h1 > small').content
lib/docs/scrapers/influxdata.rb+1−3
@@ -48,9 +48,7 @@ module Docs · 47 unmodified lines
48 HTML48 HTML
49 49
50 def get_latest_version(opts)50 def get_latest_version(opts)
51 doc = fetch_doc('https://docs.influxdata.com/influxdb/', opts)51 get_latest_github_release('influxdata', 'influxdb', opts)
52 label = doc.at_css('.navbar--current-product').content.strip
53 label.scan(/([0-9.]+)/)[0][0]
54 end52 end
55 end53 end
56end54end
lib/docs/scrapers/jest.rb+2−2
@@ -1,7 +1,7 @@
1module Docs1module Docs
2 class Jest < UrlScraper2 class Jest < UrlScraper
3 self.type = 'simple'3 self.type = 'simple'
4 self.release = '24.9'4 self.release = '26.6'
5 self.base_url = 'https://jestjs.io/docs/en/'5 self.base_url = 'https://jestjs.io/docs/en/'
6 self.root_path = 'getting-started'6 self.root_path = 'getting-started'
7 self.links = {7 self.links = {
@@ -14,7 +14,7 @@ module Docs · 6 unmodified lines
14 options[:container] = '.docMainWrapper'14 options[:container] = '.docMainWrapper'
15 15
16 options[:attribution] = <<-HTML16 options[:attribution] = <<-HTML
17 &copy; 2019 Facebook, Inc. and its affiliates.<br>17 &copy; 2020 Facebook, Inc.<br>
18 Licensed under the MIT License.18 Licensed under the MIT License.
19 HTML19 HTML
20 20
lib/docs/scrapers/leaflet.rb+5−0
@@ -19,6 +19,11 @@ module Docs · 18 unmodified lines
19 Maps &copy; OpenStreetMap contributors.19 Maps &copy; OpenStreetMap contributors.
20 HTML20 HTML
21 21
22 version '1.7' do
23 self.release = '1.7.1'
24 self.base_url = "https://leafletjs.com/reference-#{release}.html"
25 end
26
22 version '1.6' do27 version '1.6' do
23 self.release = '1.6.0'28 self.release = '1.6.0'
24 self.base_url = "https://leafletjs.com/reference-#{release}.html"29 self.base_url = "https://leafletjs.com/reference-#{release}.html"
lib/docs/scrapers/lua.rb+7−2
@@ -8,12 +8,17 @@ module Docs · 7 unmodified lines
8 options[:skip_links] = true8 options[:skip_links] = true
9 9
10 options[:attribution] = <<-HTML10 options[:attribution] = <<-HTML
11 &copy; 1994&ndash;2017 Lua.org, PUC-Rio.<br>11 &copy; 1994&ndash;2020 Lua.org, PUC-Rio.<br>
12 Licensed under the MIT License.12 Licensed under the MIT License.
13 HTML13 HTML
14 14
15 version '5.4' do
16 self.release = '5.4.1'
17 self.base_url = 'https://www.lua.org/manual/5.4/'
18 end
19
15 version '5.3' do20 version '5.3' do
16 self.release = '5.3.4'21 self.release = '5.3.6'
17 self.base_url = 'https://www.lua.org/manual/5.3/'22 self.base_url = 'https://www.lua.org/manual/5.3/'
18 end23 end
19 24
lib/docs/scrapers/moment.rb+1−2
@@ -24,8 +24,7 @@ module Docs · 23 unmodified lines
24 HTML24 HTML
25 25
26 def get_latest_version(opts)26 def get_latest_version(opts)
27 doc = fetch_doc('http://momentjs.com/', opts)27 get_github_tags('moment', 'moment', opts)[0]['name']
28 doc.at_css('.hero-title > h1 > span').content
29 end28 end
30 end29 end
31end30end
lib/docs/scrapers/nginx.rb+4−3
@@ -2,7 +2,7 @@ module Docs · 1 unmodified line
2 class Nginx < UrlScraper2 class Nginx < UrlScraper
3 self.name = 'nginx'3 self.name = 'nginx'
4 self.type = 'nginx'4 self.type = 'nginx'
5 self.release = '1.17.2'5 self.release = '1.19.3'
6 self.base_url = 'https://nginx.org/en/docs/'6 self.base_url = 'https://nginx.org/en/docs/'
7 self.links = {7 self.links = {
8 home: 'https://nginx.org/',8 home: 'https://nginx.org/',
@@ -20,9 +20,10 @@ module Docs · 11 unmodified lines
20 20
21 options[:skip_patterns] = [/\/faq\//]21 options[:skip_patterns] = [/\/faq\//]
22 22
23 # http://nginx.org/LICENSE
23 options[:attribution] = <<-HTML24 options[:attribution] = <<-HTML
24 &copy; 2002-2019 Igor Sysoev<br>25 &copy; 2002-2020 Igor Sysoev<br>
25 &copy; 2011-2019 Nginx, Inc.<br>26 &copy; 2011-2020 Nginx, Inc.<br>
26 Licensed under the BSD License.27 Licensed under the BSD License.
27 HTML28 HTML
28 29
lib/docs/scrapers/node.rb+6−1
@@ -23,7 +23,12 @@ module Docs · 22 unmodified lines
23 HTML23 HTML
24 24
25 version do25 version do
26 self.release = '12.9.1'26 self.release = '14.15.0'
27 self.base_url = 'https://nodejs.org/dist/latest-v14.x/docs/api/'
28 end
29
30 version '12 LTS' do
31 self.release = '12.19.0'
27 self.base_url = 'https://nodejs.org/dist/latest-v12.x/docs/api/'32 self.base_url = 'https://nodejs.org/dist/latest-v12.x/docs/api/'
28 end33 end
29 34
lib/docs/scrapers/openjdk.rb+8−1
@@ -18,13 +18,20 @@ module Docs · 17 unmodified lines
18 /doc-files\//]18 /doc-files\//]
19 19
20 options[:attribution] = <<-HTML20 options[:attribution] = <<-HTML
21 &copy; 1993&ndash;2017, Oracle and/or its affiliates. All rights reserved.<br>21 &copy; 1993, 2020, Oracle and/or its affiliates. All rights reserved.<br>
22 Documentation extracted from Debian's OpenJDK Development Kit package.<br>22 Documentation extracted from Debian's OpenJDK Development Kit package.<br>
23 Licensed under the GNU General Public License, version 2, with the Classpath Exception.<br>23 Licensed under the GNU General Public License, version 2, with the Classpath Exception.<br>
24 Various third party code in OpenJDK is licensed under different licenses (see Debian package).<br>24 Various third party code in OpenJDK is licensed under different licenses (see Debian package).<br>
25 Java and OpenJDK are trademarks or registered trademarks of Oracle and/or its affiliates.25 Java and OpenJDK are trademarks or registered trademarks of Oracle and/or its affiliates.
26 HTML26 HTML
27 27
28 version '11' do
29 self.release = '11.0.9'
30 self.root_path = 'index.html'
31 self.base_url = 'https://docs.oracle.com/en/java/javase/11/docs/api/'
32 options[:only_patterns] = [/\Ajava\./]
33 end
34
28 version '8' do35 version '8' do
29 self.release = '8'36 self.release = '8'
30 37
lib/docs/scrapers/pandas.rb+1−3
@@ -62,9 +62,7 @@ module Docs · 61 unmodified lines
62 end62 end
63 63
64 def get_latest_version(opts)64 def get_latest_version(opts)
65 doc = fetch_doc('http://pandas.pydata.org/pandas-docs/stable/', opts)65 get_latest_github_release('pandas-dev', 'pandas', opts)
66 label = doc.at_css('.body > .section > p').content
67 label.scan(/Version: ([0-9.]+)/)[0][0]
68 end66 end
69 end67 end
70end68end
lib/docs/scrapers/perl.rb+1−2
@@ -46,8 +46,7 @@ module Docs · 45 unmodified lines
46 46
47 def get_latest_version(opts)47 def get_latest_version(opts)
48 doc = fetch_doc('https://perldoc.perl.org/', opts)48 doc = fetch_doc('https://perldoc.perl.org/', opts)
49 header = doc.at_css('h2.h1').content49 doc.at_css('#dropdownlink-stable').content
50 header.scan(/Perl ([0-9.]+)/)[0][0]
51 end50 end
52 end51 end
53end52end
lib/docs/scrapers/php.rb+2−3
@@ -68,9 +68,8 @@ module Docs · 67 unmodified lines
68 HTML68 HTML
69 69
70 def get_latest_version(opts)70 def get_latest_version(opts)
71 doc = fetch_doc('https://secure.php.net/manual/en/doc.changelog.php', opts)71 doc = fetch_doc('https://www.php.net/supported-versions.php', opts)
72 label = doc.at_css('tbody.gen-changelog > tr > td').content72 doc.at_css('table > tbody > .stable:last-of-type > td > a').content.strip
73 label.split(',').last.strip
74 end73 end
75 end74 end
76end75end
lib/docs/scrapers/postgresql.rb+6−1
@@ -51,10 +51,15 @@ module Docs · 50 unmodified lines
51 /\Aunsupported-features/ ]51 /\Aunsupported-features/ ]
52 52
53 options[:attribution] = <<-HTML53 options[:attribution] = <<-HTML
54 &copy; 1996&ndash;2019 The PostgreSQL Global Development Group<br>54 &copy; 1996&ndash;2020 The PostgreSQL Global Development Group<br>
55 Licensed under the PostgreSQL License.55 Licensed under the PostgreSQL License.
56 HTML56 HTML
57 57
58 version '13' do
59 self.release = '13.1'
60 self.base_url = "https://www.postgresql.org/docs/#{version}/"
61 end
62
58 version '12' do63 version '12' do
59 self.release = '12.1'64 self.release = '12.1'
60 self.base_url = "https://www.postgresql.org/docs/#{version}/"65 self.base_url = "https://www.postgresql.org/docs/#{version}/"
lib/docs/scrapers/pug.rb+5−1
@@ -3,7 +3,7 @@ module Docs · 2 unmodified lines
3 self.type = 'pug'3 self.type = 'pug'
4 self.base_url = 'https://pugjs.org/'4 self.base_url = 'https://pugjs.org/'
5 self.root_path = 'api/getting-started.html'5 self.root_path = 'api/getting-started.html'
6 self.release = '2.0.3'6 self.release = '3.0.0'
7 self.links = {7 self.links = {
8 home: 'https://pugjs.org/',8 home: 'https://pugjs.org/',
9 code: 'https://github.com/pugjs/pug'9 code: 'https://github.com/pugjs/pug'
@@ -18,6 +18,10 @@ module Docs · 8 unmodified lines
18 Licensed under the MIT license.18 Licensed under the MIT license.
19 HTML19 HTML
20 20
21 options[:skip_patterns] = [
22 /support/
23 ]
24
21 def get_latest_version(opts)25 def get_latest_version(opts)
22 get_npm_version('pug', opts)26 get_npm_version('pug', opts)
23 end27 end
lib/docs/scrapers/python.rb+12−4
@@ -23,22 +23,30 @@ module Docs · 22 unmodified lines
23 Licensed under the PSF License.23 Licensed under the PSF License.
24 HTML24 HTML
25 25
26 # mkdir -p docs/python~3.9 && cd docs/python~3.9 && curl -L https://docs.python.org/3.9/archives/python-3.9.0-docs-html.tar.bz2 | tar xj --strip-components=1
27 version '3.9' do # docs.python.org/3.9/download.html
28 self.release = '3.9.0'
29 self.base_url = 'https://docs.python.org/3.9/'
30
31 html_filters.push 'python/entries_v3', 'sphinx/clean_html', 'python/clean_html'
32 end
33
26 version '3.8' do # docs.python.org/3.8/download.html34 version '3.8' do # docs.python.org/3.8/download.html
27 self.release = '3.8.1'35 self.release = '3.8.6'
28 self.base_url = 'https://docs.python.org/3.8/'36 self.base_url = 'https://docs.python.org/3.8/'
29 37
30 html_filters.push 'python/entries_v3', 'sphinx/clean_html', 'python/clean_html'38 html_filters.push 'python/entries_v3', 'sphinx/clean_html', 'python/clean_html'
31 end39 end
32 40
33 version '3.7' do # docs.python.org/3.7/download.html41 version '3.7' do # docs.python.org/3.7/download.html
34 self.release = '3.7.6'42 self.release = '3.7.9'
35 self.base_url = 'https://docs.python.org/3.7/'43 self.base_url = 'https://docs.python.org/3.7/'
36 44
37 html_filters.push 'python/entries_v3', 'sphinx/clean_html', 'python/clean_html'45 html_filters.push 'python/entries_v3', 'sphinx/clean_html', 'python/clean_html'
38 end46 end
39 47
40 version '3.6' do # docs.python.org/3.6/download.html48 version '3.6' do # docs.python.org/3.6/download.html
41 self.release = '3.6.10'49 self.release = '3.6.12'
42 self.base_url = 'https://docs.python.org/3.6/'50 self.base_url = 'https://docs.python.org/3.6/'
43 51
44 html_filters.push 'python/entries_v3', 'sphinx/clean_html', 'python/clean_html'52 html_filters.push 'python/entries_v3', 'sphinx/clean_html', 'python/clean_html'
@@ -60,7 +68,7 @@ module Docs · 15 unmodified lines
60 68
61 def get_latest_version(opts)69 def get_latest_version(opts)
62 doc = fetch_doc('https://docs.python.org/', opts)70 doc = fetch_doc('https://docs.python.org/', opts)
63 doc.at_css('.version_switcher_placeholder').content71 doc.at_css('title').content.split(' ')[0]
64 end72 end
65 end73 end
66end74end
lib/docs/scrapers/rdoc/ruby.rb+4−0
@@ -69,6 +69,10 @@ module Docs · 68 unmodified lines
69 Licensed under their own licenses.69 Licensed under their own licenses.
70 HTML70 HTML
71 71
72 version '2.7' do
73 self.release = '2.7.2'
74 end
75
72 version '2.6' do76 version '2.6' do
73 self.release = '2.6.3'77 self.release = '2.6.3'
74 end78 end
lib/docs/scrapers/react.rb+1−1
@@ -2,7 +2,7 @@ module Docs · 1 unmodified line
2 class React < UrlScraper2 class React < UrlScraper
3 self.name = 'React'3 self.name = 'React'
4 self.type = 'simple'4 self.type = 'simple'
5 self.release = '16.12.0'5 self.release = '17.0.1'
6 self.base_url = 'https://reactjs.org/docs/'6 self.base_url = 'https://reactjs.org/docs/'
7 self.root_path = 'hello-world.html'7 self.root_path = 'hello-world.html'
8 self.links = {8 self.links = {
lib/docs/scrapers/react_native.rb+2−2
@@ -32,8 +32,8 @@ module Docs · 31 unmodified lines
32 HTML32 HTML
33 33
34 def get_latest_version(opts)34 def get_latest_version(opts)
35 doc = fetch_doc('https://facebook.github.io/react-native/docs/getting-started.html', opts)35 doc = fetch_doc('https://reactnative.dev/docs/getting-started', opts)
36 doc.at_css('header > a > h3').content36 doc.at_css('meta[name="docsearch:version"]')['content']
37 end37 end
38 end38 end
39end39end
lib/docs/scrapers/scikit_learn.rb+1−2
@@ -26,8 +26,7 @@ module Docs · 25 unmodified lines
26 HTML26 HTML
27 27
28 def get_latest_version(opts)28 def get_latest_version(opts)
29 doc = fetch_doc('https://scikit-learn.org/stable/documentation.html', opts)29 get_latest_github_release('scikit-learn', 'scikit-learn', opts)
30 doc.at_css('.body h1').content.scan(/([0-9.]+)/)[0][0]
31 end30 end
32 end31 end
33end32end
lib/docs/scrapers/sqlite.rb+1−1
@@ -2,7 +2,7 @@ module Docs · 1 unmodified line
2 class Sqlite < FileScraper2 class Sqlite < FileScraper
3 self.name = 'SQLite'3 self.name = 'SQLite'
4 self.type = 'sqlite'4 self.type = 'sqlite'
5 self.release = '3.30.1'5 self.release = '3.33.0'
6 self.base_url = 'https://sqlite.org/'6 self.base_url = 'https://sqlite.org/'
7 self.root_path = 'docs.html'7 self.root_path = 'docs.html'
8 self.initial_paths = %w(keyword_index.html)8 self.initial_paths = %w(keyword_index.html)
lib/docs/scrapers/statsmodels.rb+1−2
@@ -22,8 +22,7 @@ module Docs · 21 unmodified lines
22 HTML22 HTML
23 23
24 def get_latest_version(opts)24 def get_latest_version(opts)
25 doc = fetch_doc('http://www.statsmodels.org/stable/', opts)25 get_latest_github_release('statsmodels', 'statsmodels', opts)
26 doc.at_css('.sphinxsidebarwrapper h3 + p > b').content[1..-1]
27 end26 end
28 end27 end
29end28end
lib/docs/scrapers/vagrant.rb+1−2
@@ -20,8 +20,7 @@ module Docs · 19 unmodified lines
20 HTML20 HTML
21 21
22 def get_latest_version(opts)22 def get_latest_version(opts)
23 contents = get_github_file_contents('hashicorp', 'vagrant', 'website/config.rb', opts)23 get_github_tags('hashicorp', 'vagrant', opts)[0]['name'][1..-1]
24 contents.scan(/version\s+=\s+"([0-9.]+)"/)[0][0]
25 end24 end
26 end25 end
27end26end

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t