History

Finish get_latest_version for 81 scrapers and add uploading functionality

Jasper van Merle committed 7 years ago

Commit 3dc17a91 parent 2d1e8aa85 files · +739 −68Browse files

Unified · Split

85 files
  1. Gemfile +1 −0
  2. Gemfile.lock +3 −0
  3. lib/docs/core/scraper.rb +40 −16
  4. lib/docs/scrapers/angular.rb +2 −2
  5. lib/docs/scrapers/angularjs.rb +2 −2
  6. lib/docs/scrapers/ansible.rb +2 −2
  7. lib/docs/scrapers/apache.rb +2 −2
  8. lib/docs/scrapers/apache_pig.rb +2 −2
  9. lib/docs/scrapers/async.rb +2 −2
  10. lib/docs/scrapers/babel.rb +2 −2
  11. lib/docs/scrapers/backbone.rb +2 −2
  12. lib/docs/scrapers/bash.rb +2 −2
  13. lib/docs/scrapers/bluebird.rb +2 −2
  14. lib/docs/scrapers/bootstrap.rb +6 −0
  15. lib/docs/scrapers/bottle.rb +7 −0
  16. lib/docs/scrapers/bower.rb +4 −0
  17. lib/docs/scrapers/cakephp.rb +6 −0
  18. lib/docs/scrapers/chai.rb +4 −0
  19. lib/docs/scrapers/chef.rb +7 −0
  20. lib/docs/scrapers/clojure.rb +6 −0
  21. lib/docs/scrapers/cmake.rb +7 −0
  22. lib/docs/scrapers/codeception.rb +6 −0
  23. lib/docs/scrapers/codeceptjs.rb +4 −0
  24. lib/docs/scrapers/codeigniter.rb +7 −0
  25. lib/docs/scrapers/coffeescript.rb +4 −0
  26. lib/docs/scrapers/cordova.rb +9 −0
  27. lib/docs/scrapers/crystal.rb +6 −0
  28. lib/docs/scrapers/d.rb +6 −0
  29. lib/docs/scrapers/d3.rb +4 −0
  30. lib/docs/scrapers/dart.rb +7 −0
  31. lib/docs/scrapers/django.rb +6 −0
  32. lib/docs/scrapers/docker.rb +7 −0
  33. lib/docs/scrapers/dojo.rb +6 −0
  34. lib/docs/scrapers/drupal.rb +9 −0
  35. lib/docs/scrapers/electron.rb +6 −0
  36. lib/docs/scrapers/elixir.rb +6 −0
  37. lib/docs/scrapers/ember.rb +6 −0
  38. lib/docs/scrapers/erlang.rb +6 −0
  39. lib/docs/scrapers/eslint.rb +4 −0
  40. lib/docs/scrapers/express.rb +4 −0
  41. lib/docs/scrapers/falcon.rb +6 −0
  42. lib/docs/scrapers/fish.rb +6 −0
  43. lib/docs/scrapers/flow.rb +4 −0
  44. lib/docs/scrapers/git.rb +6 −0
  45. lib/docs/scrapers/gnu/gcc.rb +7 −0
  46. lib/docs/scrapers/gnu/gnu_fortran.rb +7 −0
  47. lib/docs/scrapers/go.rb +9 −0
  48. lib/docs/scrapers/godot.rb +6 −0
  49. lib/docs/scrapers/graphite.rb +6 −0
  50. lib/docs/scrapers/grunt.rb +4 −0
  51. lib/docs/scrapers/handlebars.rb +4 −0
  52. lib/docs/scrapers/haskell.rb +7 −0
  53. lib/docs/scrapers/haxe.rb +7 −0
  54. lib/docs/scrapers/homebrew.rb +6 −0
  55. lib/docs/scrapers/immutable.rb +4 −0
  56. lib/docs/scrapers/influxdata.rb +7 −0
  57. lib/docs/scrapers/jasmine.rb +6 −0
  58. lib/docs/scrapers/jekyll.rb +6 −0
  59. lib/docs/scrapers/jest.rb +6 −0
  60. lib/docs/scrapers/jquery/jquery_core.rb +4 −0
  61. lib/docs/scrapers/jquery/jquery_mobile.rb +7 −0
  62. lib/docs/scrapers/jquery/jquery_ui.rb +4 −0
  63. lib/docs/scrapers/jsdoc.rb +6 −0
  64. lib/docs/scrapers/julia.rb +6 −0
  65. lib/docs/scrapers/knockout.rb +6 −0
  66. lib/docs/scrapers/koa.rb +4 −0
  67. lib/docs/scrapers/kotlin.rb +6 −0
  68. lib/docs/scrapers/laravel.rb +6 −0
  69. lib/docs/scrapers/leaflet.rb +6 −0
  70. lib/docs/scrapers/less.rb +7 −0
  71. lib/docs/scrapers/liquid.rb +6 −0
  72. lib/docs/scrapers/lodash.rb +6 −0
  73. lib/docs/scrapers/love.rb +6 −0
  74. lib/docs/scrapers/lua.rb +6 −0
  75. lib/docs/scrapers/marionette.rb +4 −0
  76. lib/docs/scrapers/matplotlib.rb +6 −0
  77. lib/docs/scrapers/meteor.rb +6 −0
  78. lib/docs/scrapers/mocha.rb +4 −0
  79. lib/docs/scrapers/modernizr.rb +4 −0
  80. lib/docs/scrapers/moment.rb +6 −0
  81. lib/docs/scrapers/mongoose.rb +7 −0
  82. lib/docs/scrapers/rdoc/minitest.rb +6 −0
  83. lib/docs/scrapers/rdoc/rails.rb +6 −0
  84. lib/docs/scrapers/rdoc/ruby.rb +12 −0
  85. lib/tasks/updates.thor +254 −32
Gemfile+1−0
@@ -40,6 +40,7 @@ group :docs do · 39 unmodified lines
40 gem 'unix_utils', require: false40 gem 'unix_utils', require: false
41 gem 'tty-pager', require: false41 gem 'tty-pager', require: false
42 gem 'net-sftp', '>= 2.1.3.rc2', require: false42 gem 'net-sftp', '>= 2.1.3.rc2', require: false
43 gem 'terminal-table', require: false
43end44end
44 45
45group :test do46group :test do
Gemfile.lock+3−0
@@ -101,6 +101,8 @@ GEM · 100 unmodified lines
101 unicode-display_width (~> 1.4.0)101 unicode-display_width (~> 1.4.0)
102 unicode_utils (~> 1.4.0)102 unicode_utils (~> 1.4.0)
103 strings-ansi (0.1.0)103 strings-ansi (0.1.0)
104 terminal-table (1.8.0)
105 unicode-display_width (~> 1.1, >= 1.1.1)
104 thin (1.7.2)106 thin (1.7.2)
105 daemons (~> 1.0, >= 1.0.9)107 daemons (~> 1.0, >= 1.0.9)
106 eventmachine (~> 1.0, >= 1.0.4)108 eventmachine (~> 1.0, >= 1.0.4)
@@ -153,6 +155,7 @@ DEPENDENCIES · 46 unmodified lines
153 sinatra-contrib155 sinatra-contrib
154 sprockets156 sprockets
155 sprockets-helpers157 sprockets-helpers
158 terminal-table
156 thin159 thin
157 thor160 thor
158 tty-pager161 tty-pager
lib/docs/core/scraper.rb+40−16
@@ -132,7 +132,7 @@ module Docs · 131 unmodified lines
132 end132 end
133 end133 end
134 134
135 def get_latest_version(&block)135 def get_latest_version(options, &block)
136 raise NotImplementedError136 raise NotImplementedError
137 end137 end
138 138
@@ -147,15 +147,15 @@ module Docs · 8 unmodified lines
147 # 1 -> 2 = outdated147 # 1 -> 2 = outdated
148 # 1.1 -> 1.2 = outdated148 # 1.1 -> 1.2 = outdated
149 # 1.1.1 -> 1.1.2 = not outdated149 # 1.1.1 -> 1.1.2 = not outdated
150 def is_outdated(current_version, latest_version)150 def is_outdated(scraper_version, latest_version)
151 current_parts = current_version.split(/\./).map(&:to_i)151 scraper_parts = scraper_version.split(/\./).map(&:to_i)
152 latest_parts = latest_version.split(/\./).map(&:to_i)152 latest_parts = latest_version.split(/\./).map(&:to_i)
153 153
154 # Only check the first two parts, the third part is for patch updates154 # Only check the first two parts, the third part is for patch updates
155 [0, 1].each do |i|155 [0, 1].each do |i|
156 break if i >= current_parts.length or i >= latest_parts.length156 break if i >= scraper_parts.length or i >= latest_parts.length
157 return true if latest_parts[i] > current_parts[i]157 return true if latest_parts[i] > scraper_parts[i]
158 return false if latest_parts[i] < current_parts[i]158 return false if latest_parts[i] < scraper_parts[i]
159 end159 end
160 160
161 false161 false
@@ -231,38 +231,62 @@ module Docs · 69 unmodified lines
231 {}231 {}
232 end232 end
233 233
234 #
234 # Utility methods for get_latest_version235 # Utility methods for get_latest_version
236 #
235 237
236 def fetch(url, &block)238 def fetch(url, options, &block)
237 Request.run(url) do |response|239 headers = {}
240
241 if options.key?(:github_token) and url.start_with?('https://api.github.com/')
242 headers['Authorization'] = "token #{options[:github_token]}"
243 end
244
245 options[:logger].debug("Fetching #{url}")
246
247 Request.run(url, { headers: headers }) do |response|
238 if response.success?248 if response.success?
239 block.call response.body249 block.call response.body
240 else250 else
251 options[:logger].error("Couldn't fetch #{url} (response code #{response.code})")
241 block.call nil252 block.call nil
242 end253 end
243 end254 end
244 end255 end
245 256
246 def fetch_doc(url, &block)257 def fetch_doc(url, options, &block)
247 fetch(url) do |body|258 fetch(url, options) do |body|
248 parser = Parser.new(body)259 block.call Nokogiri::HTML.parse body, nil, 'UTF-8'
249 block.call parser.html
250 end260 end
251 end261 end
252 262
253 def fetch_json(url, &block)263 def fetch_json(url, options, &block)
254 fetch(url) do |body|264 fetch(url, options) do |body|
255 json = JSON.parse(body)265 json = JSON.parse(body)
256 block.call json266 block.call json
257 end267 end
258 end268 end
259 269
260 def get_npm_version(package, &block)270 def get_npm_version(package, options, &block)
261 fetch_json("https://registry.npmjs.com/#{package}") do |json|271 fetch_json("https://registry.npmjs.com/#{package}", options) do |json|
262 block.call json['dist-tags']['latest']272 block.call json['dist-tags']['latest']
263 end273 end
264 end274 end
265 275
276 def get_latest_github_release(owner, repo, options, &block)
277 fetch_json("https://api.github.com/repos/#{owner}/#{repo}/releases/latest", options, &block)
278 end
279
280 def get_github_tags(owner, repo, options, &block)
281 fetch_json("https://api.github.com/repos/#{owner}/#{repo}/tags", options, &block)
282 end
283
284 def get_github_file_contents(owner, repo, path, options, &block)
285 fetch_json("https://api.github.com/repos/#{owner}/#{repo}/contents/#{path}", options) do |json|
286 block.call(Base64.decode64(json['content']))
287 end
288 end
289
266 module FixInternalUrlsBehavior290 module FixInternalUrlsBehavior
267 def self.included(base)291 def self.included(base)
268 base.extend ClassMethods292 base.extend ClassMethods
lib/docs/scrapers/angular.rb+2−2
@@ -155,8 +155,8 @@ module Docs · 154 unmodified lines
155 end155 end
156 end156 end
157 157
158 def get_latest_version(&block)158 def get_latest_version(options, &block)
159 get_npm_version('@angular/core', &block)159 get_npm_version('@angular/core', options, &block)
160 end160 end
161 161
162 private162 private
lib/docs/scrapers/angularjs.rb+2−2
@@ -70,8 +70,8 @@ module Docs · 69 unmodified lines
70 self.base_url = "https://code.angularjs.org/#{release}/docs/partials/"70 self.base_url = "https://code.angularjs.org/#{release}/docs/partials/"
71 end71 end
72 72
73 def get_latest_version(&block)73 def get_latest_version(options, &block)
74 get_npm_version('angular', &block)74 get_npm_version('angular', options, &block)
75 end75 end
76 end76 end
77end77end
lib/docs/scrapers/ansible.rb+2−2
@@ -88,8 +88,8 @@ module Docs · 87 unmodified lines
88 list_of_all_modules.html)88 list_of_all_modules.html)
89 end89 end
90 90
91 def get_latest_version(&block)91 def get_latest_version(options, &block)
92 fetch_doc('https://docs.ansible.com/ansible/latest/index.html') do |doc|92 fetch_doc('https://docs.ansible.com/ansible/latest/index.html', options) do |doc|
93 block.call doc.at_css('.DocSiteProduct-CurrentVersion').content.strip93 block.call doc.at_css('.DocSiteProduct-CurrentVersion').content.strip
94 end94 end
95 end95 end
lib/docs/scrapers/apache.rb+2−2
@@ -34,8 +34,8 @@ module Docs · 33 unmodified lines
34 Licensed under the Apache License, Version 2.0.34 Licensed under the Apache License, Version 2.0.
35 HTML35 HTML
36 36
37 def get_latest_version(&block)37 def get_latest_version(options, &block)
38 fetch_doc('http://httpd.apache.org/docs/') do |doc|38 fetch_doc('http://httpd.apache.org/docs/', options) do |doc|
39 block.call doc.at_css('#apcontents > ul a')['href'][0...-1]39 block.call doc.at_css('#apcontents > ul a')['href'][0...-1]
40 end40 end
41 end41 end
lib/docs/scrapers/apache_pig.rb+2−2
@@ -43,8 +43,8 @@ module Docs · 42 unmodified lines
43 self.base_url = "https://pig.apache.org/docs/r#{release}/"43 self.base_url = "https://pig.apache.org/docs/r#{release}/"
44 end44 end
45 45
46 def get_latest_version(&block)46 def get_latest_version(options, &block)
47 fetch_doc('https://pig.apache.org/') do |doc|47 fetch_doc('https://pig.apache.org/', options) do |doc|
48 item = doc.at_css('div[id="menu_1.2"] > .menuitem:last-child')48 item = doc.at_css('div[id="menu_1.2"] > .menuitem:last-child')
49 block.call item.content.strip.sub(/Release /, '')49 block.call item.content.strip.sub(/Release /, '')
50 end50 end
lib/docs/scrapers/async.rb+2−2
@@ -18,8 +18,8 @@ module Docs · 17 unmodified lines
18 Licensed under the MIT License.18 Licensed under the MIT License.
19 HTML19 HTML
20 20
21 def get_latest_version(&block)21 def get_latest_version(options, &block)
22 fetch_doc('https://caolan.github.io/async/') do |doc|22 fetch_doc('https://caolan.github.io/async/', options) do |doc|
23 version = doc.at_css('#version-dropdown > a').content.strip[1..-1]23 version = doc.at_css('#version-dropdown > a').content.strip[1..-1]
24 block.call version24 block.call version
25 end25 end
lib/docs/scrapers/babel.rb+2−2
@@ -23,8 +23,8 @@ module Docs · 22 unmodified lines
23 '<div></div>'23 '<div></div>'
24 end24 end
25 25
26 def get_latest_version(&block)26 def get_latest_version(options, &block)
27 fetch_doc('https://babeljs.io/docs/en/') do |doc|27 fetch_doc('https://babeljs.io/docs/en/', options) do |doc|
28 block.call doc.at_css('a[href="/versions"] > h3').content28 block.call doc.at_css('a[href="/versions"] > h3').content
29 end29 end
30 end30 end
lib/docs/scrapers/backbone.rb+2−2
@@ -21,8 +21,8 @@ module Docs · 20 unmodified lines
21 Licensed under the MIT License.21 Licensed under the MIT License.
22 HTML22 HTML
23 23
24 def get_latest_version(&block)24 def get_latest_version(options, &block)
25 fetch_doc('https://backbonejs.org/') do |doc|25 fetch_doc('https://backbonejs.org/', options) do |doc|
26 version = doc.at_css('.version').content26 version = doc.at_css('.version').content
27 block.call version[1...-1]27 block.call version[1...-1]
28 end28 end
lib/docs/scrapers/bash.rb+2−2
@@ -18,8 +18,8 @@ module Docs · 17 unmodified lines
18 Licensed under the GNU Free Documentation License.18 Licensed under the GNU Free Documentation License.
19 HTML19 HTML
20 20
21 def get_latest_version(&block)21 def get_latest_version(options, &block)
22 fetch('https://www.gnu.org/software/bash/manual/html_node/index.html') do |body|22 fetch('https://www.gnu.org/software/bash/manual/html_node/index.html', options) do |body|
23 version = body.scan(/, Version ([0-9.]+)/)[0][0]23 version = body.scan(/, Version ([0-9.]+)/)[0][0]
24 block.call version[0...-1]24 block.call version[0...-1]
25 end25 end
lib/docs/scrapers/bluebird.rb+2−2
@@ -19,8 +19,8 @@ module Docs · 18 unmodified lines
19 Licensed under the MIT License.19 Licensed under the MIT License.
20 HTML20 HTML
21 21
22 def get_latest_version(&block)22 def get_latest_version(options, &block)
23 get_npm_version('bluebird', &block)23 get_npm_version('bluebird', options, &block)
24 end24 end
25 end25 end
26end26end
lib/docs/scrapers/bootstrap.rb+6−0
@@ -34,5 +34,11 @@ module Docs · 33 unmodified lines
34 34
35 options[:only] = %w(getting-started/ css/ components/ javascript/)35 options[:only] = %w(getting-started/ css/ components/ javascript/)
36 end36 end
37
38 def get_latest_version(options, &block)
39 fetch_doc('https://getbootstrap.com/', options) do |doc|
40 block.call doc.at_css('#bd-versions').content.strip[1..-1]
41 end
42 end
37 end43 end
38end44end
lib/docs/scrapers/bottle.rb+7−0
@@ -27,5 +27,12 @@ module Docs · 26 unmodified lines
27 self.release = '0.11.7'27 self.release = '0.11.7'
28 self.base_url = "https://bottlepy.org/docs/#{self.version}/"28 self.base_url = "https://bottlepy.org/docs/#{self.version}/"
29 end29 end
30
31 def get_latest_version(options, &block)
32 fetch_doc('https://bottlepy.org/docs/stable/', options) do |doc|
33 label = doc.at_css('.sphinxsidebarwrapper > ul > li > b')
34 block.call label.content.sub(/Bottle /, '')
35 end
36 end
30 end37 end
31end38end
lib/docs/scrapers/bower.rb+4−0
@@ -19,5 +19,9 @@ module Docs · 18 unmodified lines
19 &copy; 2018 Bower contributors<br>19 &copy; 2018 Bower contributors<br>
20 Licensed under the MIT License.20 Licensed under the MIT License.
21 HTML21 HTML
22
23 def get_latest_version(options, &block)
24 get_npm_version('bower', options, &block)
25 end
22 end26 end
23end27end
lib/docs/scrapers/cakephp.rb+6−0
@@ -71,6 +71,12 @@ module Docs · 70 unmodified lines
71 self.base_url = 'https://api.cakephp.org/2.7/'71 self.base_url = 'https://api.cakephp.org/2.7/'
72 end72 end
73 73
74 def get_latest_version(options, &block)
75 fetch_doc('https://api.cakephp.org/3.7/', options) do |doc|
76 block.call doc.at_css('.version-picker .dropdown-toggle').content.strip
77 end
78 end
79
74 private80 private
75 81
76 def parse(response)82 def parse(response)
lib/docs/scrapers/chai.rb+4−0
@@ -23,5 +23,9 @@ module Docs · 22 unmodified lines
23 &copy; 2016 Chai.js Assertion Library<br>23 &copy; 2016 Chai.js Assertion Library<br>
24 Licensed under the MIT License.24 Licensed under the MIT License.
25 HTML25 HTML
26
27 def get_latest_version(options, &block)
28 get_npm_version('chai', options, &block)
29 end
26 end30 end
27end31end
lib/docs/scrapers/chef.rb+7−0
@@ -47,5 +47,12 @@ module Docs · 46 unmodified lines
47 47
48 options[:only_patterns] = [/\A#{client_path}\//, /\A#{server_path}\//]48 options[:only_patterns] = [/\A#{client_path}\//, /\A#{server_path}\//]
49 end49 end
50
51 def get_latest_version(options, &block)
52 fetch_doc('https://docs-archive.chef.io/', options) do |doc|
53 cell = doc.at_css('.main-archives > tr:nth-child(2) > td:nth-child(2)')
54 block.call cell.content.sub(/Chef Client /, '')
55 end
56 end
50 end57 end
51end58end
lib/docs/scrapers/clojure.rb+6−0
@@ -27,5 +27,11 @@ module Docs · 26 unmodified lines
27 self.release = '1.7'27 self.release = '1.7'
28 self.base_url = 'https://clojure.github.io/clojure/branch-clojure-1.7.0/'28 self.base_url = 'https://clojure.github.io/clojure/branch-clojure-1.7.0/'
29 end29 end
30
31 def get_latest_version(options, &block)
32 fetch_doc('http://clojure.github.io/clojure/index.html', options) do |doc|
33 block.call doc.at_css('#header-version').content[1..-1]
34 end
35 end
30 end36 end
31end37end
lib/docs/scrapers/cmake.rb+7−0
@@ -59,5 +59,12 @@ module Docs · 58 unmodified lines
59 self.release = '3.5.2'59 self.release = '3.5.2'
60 self.base_url = 'https://cmake.org/cmake/help/v3.5/'60 self.base_url = 'https://cmake.org/cmake/help/v3.5/'
61 end61 end
62
63 def get_latest_version(options, &block)
64 fetch_doc('https://cmake.org/documentation/', options) do |doc|
65 link = doc.at_css('.entry-content ul > li > strong > a > big')
66 block.call link.content.scan(/([0-9.]+)/)[0][0]
67 end
68 end
62 end69 end
63end70end
lib/docs/scrapers/codeception.rb+6−0
@@ -18,5 +18,11 @@ module Docs · 17 unmodified lines
18 &copy; 2011 Michael Bodnarchuk and contributors<br>18 &copy; 2011 Michael Bodnarchuk and contributors<br>
19 Licensed under the MIT License.19 Licensed under the MIT License.
20 HTML20 HTML
21
22 def get_latest_version(options, &block)
23 fetch_doc('https://codeception.com/changelog', options) do |doc|
24 block.call doc.at_css('#page > h4').content
25 end
26 end
21 end27 end
22end28end
lib/docs/scrapers/codeceptjs.rb+4−0
@@ -21,5 +21,9 @@ module Docs · 20 unmodified lines
21 &copy; 2015 DavertMik &lt;davert@codegyre.com&gt; (http://codegyre.com)<br>21 &copy; 2015 DavertMik &lt;davert@codegyre.com&gt; (http://codegyre.com)<br>
22 Licensed under the MIT License.22 Licensed under the MIT License.
23 HTML23 HTML
24
25 def get_latest_version(options, &block)
26 get_npm_version('codeceptjs', options, &block)
27 end
24 end28 end
25end29end
lib/docs/scrapers/codeigniter.rb+7−0
@@ -38,5 +38,12 @@ module Docs · 37 unmodified lines
38 version '3' do38 version '3' do
39 self.release = '3.1.8'39 self.release = '3.1.8'
40 end40 end
41
42 def get_latest_version(options, &block)
43 fetch_doc('https://codeigniter.com/user_guide/changelog.html', options) do |doc|
44 header = doc.at_css('#change-log h2')
45 block.call header.content.scan(/([0-9.]+)/)[0][0]
46 end
47 end
41 end48 end
42end49end
lib/docs/scrapers/coffeescript.rb+4−0
@@ -30,5 +30,9 @@ module Docs · 29 unmodified lines
30 30
31 options[:container] = '.container'31 options[:container] = '.container'
32 end32 end
33
34 def get_latest_version(options, &block)
35 get_npm_version('coffeescript', options, &block)
36 end
33 end37 end
34end38end
lib/docs/scrapers/cordova.rb+9−0
@@ -42,5 +42,14 @@ module Docs · 41 unmodified lines
42 self.release = '6.5.0'42 self.release = '6.5.0'
43 self.base_url = 'https://cordova.apache.org/docs/en/6.x/'43 self.base_url = 'https://cordova.apache.org/docs/en/6.x/'
44 end44 end
45
46 def get_latest_version(options, &block)
47 fetch_doc('https://cordova.apache.org/docs/en/latest/', options) do |doc|
48 label = doc.at_css('#versionDropdown').content.strip
49 version = label.scan(/([0-9.]+)/)[0][0]
50 version = version[0...-1] if version.end_with?('.')
51 block.call version
52 end
53 end
45 end54 end
46end55end
lib/docs/scrapers/crystal.rb+6−0
@@ -34,5 +34,11 @@ module Docs · 33 unmodified lines
34 HTML34 HTML
35 end35 end
36 }36 }
37
38 def get_latest_version(options, &block)
39 fetch('https://crystal-lang.org/api', options) do |body|
40 block.call body.scan(/Crystal Docs ([0-9.]+)/)[0][0]
41 end
42 end
37 end43 end
38end44end
lib/docs/scrapers/d.rb+6−0
@@ -26,5 +26,11 @@ module Docs · 25 unmodified lines
26 def initial_urls26 def initial_urls
27 %w(https://dlang.org/phobos/index.html https://dlang.org/spec/intro.html)27 %w(https://dlang.org/phobos/index.html https://dlang.org/spec/intro.html)
28 end28 end
29
30 def get_latest_version(options, &block)
31 fetch_doc('https://dlang.org/changelog/', options) do |doc|
32 block.call doc.at_css('#content > ul > li:nth-child(2) > a')['id']
33 end
34 end
29 end35 end
30end36end
lib/docs/scrapers/d3.rb+4−0
@@ -58,5 +58,9 @@ module Docs · 57 unmodified lines
58 options[:root_title] = 'D3.js'58 options[:root_title] = 'D3.js'
59 options[:only_patterns] = [/\.md\z/]59 options[:only_patterns] = [/\.md\z/]
60 end60 end
61
62 def get_latest_version(options, &block)
63 get_npm_version('d3', options, &block)
64 end
61 end65 end
62end66end
lib/docs/scrapers/dart.rb+7−0
@@ -31,5 +31,12 @@ module Docs · 30 unmodified lines
31 self.release = '1.24.3'31 self.release = '1.24.3'
32 self.base_url = "https://api.dartlang.org/stable/#{release}/"32 self.base_url = "https://api.dartlang.org/stable/#{release}/"
33 end33 end
34
35 def get_latest_version(options, &block)
36 fetch_doc('https://api.dartlang.org/', options) do |doc|
37 label = doc.at_css('footer > span').content.strip
38 block.call label.sub(/Dart /, '')
39 end
40 end
34 end41 end
35end42end
lib/docs/scrapers/django.rb+6−0
@@ -63,5 +63,11 @@ module Docs · 62 unmodified lines
63 self.release = '1.8.18'63 self.release = '1.8.18'
64 self.base_url = 'https://docs.djangoproject.com/en/1.8/'64 self.base_url = 'https://docs.djangoproject.com/en/1.8/'
65 end65 end
66
67 def get_latest_version(options, &block)
68 fetch_doc('https://docs.djangoproject.com/', options) do |doc|
69 block.call doc.at_css('#doc-versions > li.current > span > strong').content
70 end
71 end
66 end72 end
67end73end
lib/docs/scrapers/docker.rb+7−0
@@ -137,5 +137,12 @@ module Docs · 136 unmodified lines
137 options[:container] = '#docs'137 options[:container] = '#docs'
138 options[:only_patterns] << /\Aswarm\//138 options[:only_patterns] << /\Aswarm\//
139 end139 end
140
141 def get_latest_version(options, &block)
142 fetch_doc('https://docs.docker.com/', options) do |doc|
143 label = doc.at_css('.nav-container button.dropdown-toggle').content.strip
144 block.call label.scan(/([0-9.]+)/)[0][0]
145 end
146 end
140 end147 end
141end148end
lib/docs/scrapers/dojo.rb+6−0
@@ -36,6 +36,12 @@ module Docs · 35 unmodified lines
36 urls.map { |url| "<a href='#{url}'>#{url}</a>" }.join36 urls.map { |url| "<a href='#{url}'>#{url}</a>" }.join
37 end37 end
38 38
39 def get_latest_version(options, &block)
40 fetch_doc('https://dojotoolkit.org/api/', options) do |doc|
41 block.call doc.at_css('#versionSelector > option[selected]').content
42 end
43 end
44
39 private45 private
40 46
41 def get_url_list(json, set = Set.new)47 def get_url_list(json, set = Set.new)
lib/docs/scrapers/drupal.rb+9−0
@@ -98,5 +98,14 @@ module Docs · 97 unmodified lines
98 /\A[\w\-\.]+\.php\/7\.x\z/98 /\A[\w\-\.]+\.php\/7\.x\z/
99 ]99 ]
100 end100 end
101
102 def get_latest_version(options, &block)
103 fetch_doc('http://cgit.drupalcode.org/drupal', options) do |doc|
104 version = doc.at_css('td.form > form > select > option[selected]').content
105 version = version.scan(/([0-9.]+)/)[0][0]
106 version = version[0...-1] if version.end_with?('.')
107 block.call version
108 end
109 end
101 end110 end
102end111end
lib/docs/scrapers/electron.rb+6−0
@@ -22,5 +22,11 @@ module Docs · 21 unmodified lines
22 &copy; 2013&ndash;2018 GitHub Inc.<br>22 &copy; 2013&ndash;2018 GitHub Inc.<br>
23 Licensed under the MIT license.23 Licensed under the MIT license.
24 HTML24 HTML
25
26 def get_latest_version(options, &block)
27 fetch_doc('https://electronjs.org/docs', options) do |doc|
28 block.call doc.at_css('.docs-version').content
29 end
30 end
25 end31 end
26end32end
lib/docs/scrapers/elixir.rb+6−0
@@ -97,5 +97,11 @@ module Docs · 96 unmodified lines
97 'https://elixir-lang.org/getting-started/'97 'https://elixir-lang.org/getting-started/'
98 ]98 ]
99 end99 end
100
101 def get_latest_version(options, &block)
102 fetch_doc('https://hexdocs.pm/elixir/api-reference.html', options) do |doc|
103 block.call doc.at_css('h2.sidebar-projectVersion').content.strip[1..-1]
104 end
105 end
100 end106 end
101end107end
lib/docs/scrapers/ember.rb+6−0
@@ -56,5 +56,11 @@ module Docs · 55 unmodified lines
56 https://emberjs.com/api/ember-data/2.14/classes/DS56 https://emberjs.com/api/ember-data/2.14/classes/DS
57 )57 )
58 end58 end
59
60 def get_latest_version(options, &block)
61 fetch_doc('https://emberjs.com/api/ember/release', options) do |doc|
62 block.call doc.at_css('.sidebar > .select-container .ember-power-select-selected-item').content.strip
63 end
64 end
59 end65 end
60end66end
lib/docs/scrapers/erlang.rb+6−0
@@ -55,5 +55,11 @@ module Docs · 54 unmodified lines
55 version '18' do55 version '18' do
56 self.release = '18.3'56 self.release = '18.3'
57 end57 end
58
59 def get_latest_version(options, &block)
60 fetch_doc('https://www.erlang.org/downloads', options) do |doc|
61 block.call doc.at_css('.col-lg-3 > ul > li').content.strip
62 end
63 end
58 end64 end
59end65end
lib/docs/scrapers/eslint.rb+4−0
@@ -20,5 +20,9 @@ module Docs · 19 unmodified lines
20 &copy; JS Foundation and other contributors<br>20 &copy; JS Foundation and other contributors<br>
21 Licensed under the MIT License.21 Licensed under the MIT License.
22 HTML22 HTML
23
24 def get_latest_version(options, &block)
25 get_npm_version('eslint', options, &block)
26 end
23 end27 end
24end28end
lib/docs/scrapers/express.rb+4−0
@@ -28,5 +28,9 @@ module Docs · 27 unmodified lines
28 &copy; 2017 StrongLoop, IBM, and other expressjs.com contributors.<br>28 &copy; 2017 StrongLoop, IBM, and other expressjs.com contributors.<br>
29 Licensed under the Creative Commons Attribution-ShareAlike License v3.0.29 Licensed under the Creative Commons Attribution-ShareAlike License v3.0.
30 HTML30 HTML
31
32 def get_latest_version(options, &block)
33 get_npm_version('express', options, &block)
34 end
31 end35 end
32end36end
lib/docs/scrapers/falcon.rb+6−0
@@ -33,5 +33,11 @@ module Docs · 32 unmodified lines
33 self.release = '1.2.0'33 self.release = '1.2.0'
34 self.base_url = "https://falcon.readthedocs.io/en/#{self.release}/"34 self.base_url = "https://falcon.readthedocs.io/en/#{self.release}/"
35 end35 end
36
37 def get_latest_version(options, &block)
38 fetch_doc('https://falcon.readthedocs.io/en/stable/changes/index.html', options) do |doc|
39 block.call doc.at_css('#changelogs ul > li > a').content
40 end
41 end
36 end42 end
37end43end
lib/docs/scrapers/fish.rb+6−0
@@ -46,5 +46,11 @@ module Docs · 45 unmodified lines
46 self.release = '2.2.0'46 self.release = '2.2.0'
47 self.base_url = "https://fishshell.com/docs/#{version}/"47 self.base_url = "https://fishshell.com/docs/#{version}/"
48 end48 end
49
50 def get_latest_version(options, &block)
51 fetch_doc('http://fishshell.com/docs/current/index.html', options) do |doc|
52 block.call doc.at_css('#toc-index').content.scan(/([0-9.]+)/)[0][0]
53 end
54 end
49 end55 end
50end56end
lib/docs/scrapers/flow.rb+4−0
@@ -18,5 +18,9 @@ module Docs · 17 unmodified lines
18 &copy; 2013&ndash;present Facebook Inc.<br>18 &copy; 2013&ndash;present Facebook Inc.<br>
19 Licensed under the MIT License.19 Licensed under the MIT License.
20 HTML20 HTML
21
22 def get_latest_version(options, &block)
23 get_npm_version('flow-bin', options, &block)
24 end
21 end25 end
22end26end
lib/docs/scrapers/git.rb+6−0
@@ -19,5 +19,11 @@ module Docs · 18 unmodified lines
19 &copy; 2005&ndash;2018 Linus Torvalds and others<br>19 &copy; 2005&ndash;2018 Linus Torvalds and others<br>
20 Licensed under the GNU General Public License version 2.20 Licensed under the GNU General Public License version 2.
21 HTML21 HTML
22
23 def get_latest_version(options, &block)
24 fetch_doc('https://git-scm.com/', options) do |doc|
25 block.call doc.at_css('.version').content.strip
26 end
27 end
22 end28 end
23end29end
lib/docs/scrapers/gnu/gcc.rb+7−0
@@ -99,5 +99,12 @@ module Docs · 98 unmodified lines
99 99
100 options[:replace_paths] = CPP_PATHS100 options[:replace_paths] = CPP_PATHS
101 end101 end
102
103 def get_latest_version(options, &block)
104 fetch_doc('https://gcc.gnu.org/onlinedocs/', options) do |doc|
105 label = doc.at_css('ul > li > ul > li > a').content.strip
106 block.call label.scan(/([0-9.]+)/)[0][0]
107 end
108 end
102 end109 end
103end110end
lib/docs/scrapers/gnu/gnu_fortran.rb+7−0
@@ -25,5 +25,12 @@ module Docs · 24 unmodified lines
25 self.release = '4.9.3'25 self.release = '4.9.3'
26 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"26 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
27 end27 end
28
29 def get_latest_version(options, &block)
30 fetch_doc('https://gcc.gnu.org/onlinedocs/', options) do |doc|
31 label = doc.at_css('ul > li > ul > li > a').content.strip
32 block.call label.scan(/([0-9.]+)/)[0][0]
33 end
34 end
28 end35 end
29end36end
lib/docs/scrapers/go.rb+9−0
@@ -24,6 +24,15 @@ module Docs · 23 unmodified lines
24 Licensed under the Creative Commons Attribution License 3.0.24 Licensed under the Creative Commons Attribution License 3.0.
25 HTML25 HTML
26 26
27 def get_latest_version(options, &block)
28 fetch_doc('https://golang.org/pkg/', options) do |doc|
29 footer = doc.at_css('#footer').content
30 version = footer.scan(/go([0-9.]+)/)[0][0]
31 version = version[0...-1] if version.end_with?('.')
32 block.call version
33 end
34 end
35
27 private36 private
28 37
29 def parse(response) # Hook here because Nokogori removes whitespace from textareas38 def parse(response) # Hook here because Nokogori removes whitespace from textareas
lib/docs/scrapers/godot.rb+6−0
@@ -37,5 +37,11 @@ module Docs · 36 unmodified lines
37 self.release = '2.1'37 self.release = '2.1'
38 self.base_url = "http://docs.godotengine.org/en/#{self.version}/"38 self.base_url = "http://docs.godotengine.org/en/#{self.version}/"
39 end39 end
40
41 def get_latest_version(options, &block)
42 fetch_doc('https://docs.godotengine.org/', options) do |doc|
43 block.call doc.at_css('.version').content.strip
44 end
45 end
40 end46 end
41end47end
lib/docs/scrapers/graphite.rb+6−0
@@ -17,5 +17,11 @@ module Docs · 16 unmodified lines
17 &copy; 2011&ndash;2016 The Graphite Project<br>17 &copy; 2011&ndash;2016 The Graphite Project<br>
18 Licensed under the Apache License, Version 2.0.18 Licensed under the Apache License, Version 2.0.
19 HTML19 HTML
20
21 def get_latest_version(options, &block)
22 fetch_doc('https://graphite.readthedocs.io/en/latest/releases.html', options) do |doc|
23 block.call doc.at_css('#release-notes li > a').content
24 end
25 end
20 end26 end
21end27end
lib/docs/scrapers/grunt.rb+4−0
@@ -26,5 +26,9 @@ module Docs · 25 unmodified lines
26 &copy; GruntJS Team<br>26 &copy; GruntJS Team<br>
27 Licensed under the MIT License.27 Licensed under the MIT License.
28 HTML28 HTML
29
30 def get_latest_version(options, &block)
31 get_npm_version('grunt-cli', options, &block)
32 end
29 end33 end
30end34end
lib/docs/scrapers/handlebars.rb+4−0
@@ -19,5 +19,9 @@ module Docs · 18 unmodified lines
19 &copy; 2011&ndash;2017 by Yehuda Katz<br>19 &copy; 2011&ndash;2017 by Yehuda Katz<br>
20 Licensed under the MIT License.20 Licensed under the MIT License.
21 HTML21 HTML
22
23 def get_latest_version(options, &block)
24 get_npm_version('handlebars', options, &block)
25 end
22 end26 end
23end27end
lib/docs/scrapers/haskell.rb+7−0
@@ -68,5 +68,12 @@ module Docs · 67 unmodified lines
68 68
69 options[:only_patterns] = [/\Alibraries\//]69 options[:only_patterns] = [/\Alibraries\//]
70 end70 end
71
72 def get_latest_version(options, &block)
73 fetch_doc('https://downloads.haskell.org/~ghc/latest/docs/html/users_guide/', options) do |doc|
74 label = doc.at_css('.related > ul > li:last-child').content
75 block.call label.scan(/([0-9.]+)/)[0][0]
76 end
77 end
71 end78 end
72end79end
lib/docs/scrapers/haxe.rb+7−0
@@ -66,5 +66,12 @@ module Docs · 65 unmodified lines
66 version 'Python' do66 version 'Python' do
67 self.base_url = 'https://api.haxe.org/python/'67 self.base_url = 'https://api.haxe.org/python/'
68 end68 end
69
70 def get_latest_version(options, &block)
71 fetch_doc('https://api.haxe.org/', options) do |doc|
72 label = doc.at_css('.container.main-content h1 > small').content
73 block.call label.sub(/version /, '')
74 end
75 end
69 end76 end
70end77end
lib/docs/scrapers/homebrew.rb+6−0
@@ -19,5 +19,11 @@ module Docs · 18 unmodified lines
19 &copy; 2009&ndash;present Homebrew contributors<br>19 &copy; 2009&ndash;present Homebrew contributors<br>
20 Licensed under the BSD 2-Clause License.20 Licensed under the BSD 2-Clause License.
21 HTML21 HTML
22
23 def get_latest_version(options, &block)
24 get_latest_github_release('Homebrew', 'brew', options) do |release|
25 block.call release['name']
26 end
27 end
22 end28 end
23end29end
lib/docs/scrapers/immutable.rb+4−0
@@ -54,5 +54,9 @@ module Docs · 53 unmodified lines
54 JS54 JS
55 capybara.html55 capybara.html
56 end56 end
57
58 def get_latest_version(options, &block)
59 get_npm_version('immutable', options, &block)
60 end
57 end61 end
58end62end
lib/docs/scrapers/influxdata.rb+7−0
@@ -46,5 +46,12 @@ module Docs · 45 unmodified lines
46 &copy; 2015 InfluxData, Inc.<br>46 &copy; 2015 InfluxData, Inc.<br>
47 Licensed under the MIT license.47 Licensed under the MIT license.
48 HTML48 HTML
49
50 def get_latest_version(options, &block)
51 fetch_doc('https://docs.influxdata.com/influxdb/', options) do |doc|
52 label = doc.at_css('.navbar--current-product').content.strip
53 block.call label.scan(/([0-9.]+)/)[0][0]
54 end
55 end
49 end56 end
50end57end
lib/docs/scrapers/jasmine.rb+6−0
@@ -17,5 +17,11 @@ module Docs · 16 unmodified lines
17 &copy; 2008&ndash;2017 Pivotal Labs<br>17 &copy; 2008&ndash;2017 Pivotal Labs<br>
18 Licensed under the MIT License.18 Licensed under the MIT License.
19 HTML19 HTML
20
21 def get_latest_version(options, &block)
22 get_latest_github_release('jasmine', 'jasmine', options) do |release|
23 block.call release['name']
24 end
25 end
20 end26 end
21end27end
lib/docs/scrapers/jekyll.rb+6−0
@@ -28,5 +28,11 @@ module Docs · 27 unmodified lines
28 &copy; 2008&ndash;2018 Tom Preston-Werner and Jekyll contributors<br>28 &copy; 2008&ndash;2018 Tom Preston-Werner and Jekyll contributors<br>
29 Licensed under the MIT license.29 Licensed under the MIT license.
30 HTML30 HTML
31
32 def get_latest_version(options, &block)
33 fetch_doc('https://jekyllrb.com/docs/', options) do |doc|
34 block.call doc.at_css('.meta a').content[1..-1]
35 end
36 end
31 end37 end
32end38end
lib/docs/scrapers/jest.rb+6−0
@@ -17,5 +17,11 @@ module Docs · 16 unmodified lines
17 &copy; 2014&ndash;present Facebook Inc.<br>17 &copy; 2014&ndash;present Facebook Inc.<br>
18 Licensed under the BSD License.18 Licensed under the BSD License.
19 HTML19 HTML
20
21 def get_latest_version(options, &block)
22 fetch_doc('https://jestjs.io/docs/en/getting-started', options) do |doc|
23 block.call doc.at_css('header > a > h3').content
24 end
25 end
20 end26 end
21end27end
lib/docs/scrapers/jquery/jquery_core.rb+4−0
@@ -22,5 +22,9 @@ module Docs · 21 unmodified lines
22 /Selectors\/odd/i,22 /Selectors\/odd/i,
23 /index/i23 /index/i
24 ]24 ]
25
26 def get_latest_version(options, &block)
27 get_npm_version('jquery', options, &block)
28 end
25 end29 end
26end30end
lib/docs/scrapers/jquery/jquery_mobile.rb+7−0
@@ -16,5 +16,12 @@ module Docs · 15 unmodified lines
16 options[:fix_urls] = ->(url) do16 options[:fix_urls] = ->(url) do
17 url.sub! 'http://api.jquerymobile.com/', 'https://api.jquerymobile.com/'17 url.sub! 'http://api.jquerymobile.com/', 'https://api.jquerymobile.com/'
18 end18 end
19
20 def get_latest_version(options, &block)
21 fetch_doc('https://jquerymobile.com/', options) do |doc|
22 label = doc.at_css('.download-box > .download-option:last-child > span').content
23 block.call label.sub(/Version /, '')
24 end
25 end
19 end26 end
20end27end
lib/docs/scrapers/jquery/jquery_ui.rb+4−0
@@ -15,5 +15,9 @@ module Docs · 14 unmodified lines
15 options[:fix_urls] = ->(url) do15 options[:fix_urls] = ->(url) do
16 url.sub! 'http://api.jqueryui.com/', 'https://api.jqueryui.com/'16 url.sub! 'http://api.jqueryui.com/', 'https://api.jqueryui.com/'
17 end17 end
18
19 def get_latest_version(options, &block)
20 get_npm_version('jquery-ui', options, &block)
21 end
18 end22 end
19end23end
lib/docs/scrapers/jsdoc.rb+6−0
@@ -21,5 +21,11 @@ module Docs · 20 unmodified lines
21 &copy; 2011&ndash;2017 the contributors to the JSDoc 3 documentation project<br>21 &copy; 2011&ndash;2017 the contributors to the JSDoc 3 documentation project<br>
22 Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0.22 Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0.
23 HTML23 HTML
24
25 def get_latest_version(options, &block)
26 get_latest_github_release('jsdoc3', 'jsdoc', options) do |release|
27 block.call release['tag_name']
28 end
29 end
24 end30 end
25end31end
lib/docs/scrapers/julia.rb+6−0
@@ -49,5 +49,11 @@ module Docs · 48 unmodified lines
49 49
50 html_filters.push 'julia/entries_sphinx', 'julia/clean_html_sphinx', 'sphinx/clean_html'50 html_filters.push 'julia/entries_sphinx', 'julia/clean_html_sphinx', 'sphinx/clean_html'
51 end51 end
52
53 def get_latest_version(options, &block)
54 get_latest_github_release('JuliaLang', 'julia', options) do |release|
55 block.call release['tag_name'][1..-1]
56 end
57 end
52 end58 end
53end59end
lib/docs/scrapers/knockout.rb+6−0
@@ -33,5 +33,11 @@ module Docs · 32 unmodified lines
33 &copy; Steven Sanderson, the Knockout.js team, and other contributors<br>33 &copy; Steven Sanderson, the Knockout.js team, and other contributors<br>
34 Licensed under the MIT License.34 Licensed under the MIT License.
35 HTML35 HTML
36
37 def get_latest_version(options, &block)
38 get_latest_github_release('knockout', 'knockout', options) do |release|
39 block.call release['tag_name'][1..-1]
40 end
41 end
36 end42 end
37end43end
lib/docs/scrapers/koa.rb+4−0
@@ -34,5 +34,9 @@ module Docs · 33 unmodified lines
34 &copy; 2018 Koa contributors<br>34 &copy; 2018 Koa contributors<br>
35 Licensed under the MIT License.35 Licensed under the MIT License.
36 HTML36 HTML
37
38 def get_latest_version(options, &block)
39 get_npm_version('koa', options, &block)
40 end
37 end41 end
38end42end
lib/docs/scrapers/kotlin.rb+6−0
@@ -28,5 +28,11 @@ module Docs · 27 unmodified lines
28 &copy; 2010&ndash;2018 JetBrains s.r.o.<br>28 &copy; 2010&ndash;2018 JetBrains s.r.o.<br>
29 Licensed under the Apache License, Version 2.0.29 Licensed under the Apache License, Version 2.0.
30 HTML30 HTML
31
32 def get_latest_version(options, &block)
33 get_latest_github_release('JetBrains', 'kotlin', options) do |release|
34 block.call release['tag_name'][1..-1]
35 end
36 end
31 end37 end
32end38end
lib/docs/scrapers/laravel.rb+6−0
@@ -133,5 +133,11 @@ module Docs · 132 unmodified lines
133 url133 url
134 end134 end
135 end135 end
136
137 def get_latest_version(options, &block)
138 get_latest_github_release('laravel', 'laravel', options) do |release|
139 block.call release['tag_name'][1..-1]
140 end
141 end
136 end142 end
137end143end
lib/docs/scrapers/leaflet.rb+6−0
@@ -39,5 +39,11 @@ module Docs · 38 unmodified lines
39 self.base_url = "https://leafletjs.com/reference-#{release}.html"39 self.base_url = "https://leafletjs.com/reference-#{release}.html"
40 end40 end
41 41
42 def get_latest_version(options, &block)
43 fetch_doc('https://leafletjs.com/index.html', options) do |doc|
44 link = doc.css('ul > li > a').to_a.select {|node| node.content == 'Docs'}.first
45 block.call link['href'].scan(/reference-([0-9.]+)\.html/)[0][0]
46 end
47 end
42 end48 end
43end49end
lib/docs/scrapers/less.rb+7−0
@@ -21,5 +21,12 @@ module Docs · 20 unmodified lines
21 &copy; 2009&ndash;2016 The Core Less Team<br>21 &copy; 2009&ndash;2016 The Core Less Team<br>
22 Licensed under the Creative Commons Attribution License 3.0.22 Licensed under the Creative Commons Attribution License 3.0.
23 HTML23 HTML
24
25 def get_latest_version(options, &block)
26 fetch_doc('http://lesscss.org/features/', options) do |doc|
27 label = doc.at_css('.footer-links > li').content
28 block.call label.scan(/([0-9.]+)/)[0][0]
29 end
30 end
24 end31 end
25end32end
lib/docs/scrapers/liquid.rb+6−0
@@ -19,5 +19,11 @@ module Docs · 18 unmodified lines
19 &copy; 2005, 2006 Tobias Luetke<br>19 &copy; 2005, 2006 Tobias Luetke<br>
20 Licensed under the MIT License.20 Licensed under the MIT License.
21 HTML21 HTML
22
23 def get_latest_version(options, &block)
24 get_github_tags('Shopify', 'liquid', options) do |tags|
25 block.call tags[0]['name'][1..-1]
26 end
27 end
22 end28 end
23end29end
lib/docs/scrapers/lodash.rb+6−0
@@ -32,5 +32,11 @@ module Docs · 31 unmodified lines
32 self.release = '2.4.2'32 self.release = '2.4.2'
33 self.base_url = "https://lodash.com/docs/#{release}"33 self.base_url = "https://lodash.com/docs/#{release}"
34 end34 end
35
36 def get_latest_version(options, &block)
37 fetch_doc('https://lodash.com/docs/', options) do |doc|
38 block.call doc.at_css('#version > option[selected]').content
39 end
40 end
35 end41 end
36end42end
lib/docs/scrapers/love.rb+6−0
@@ -39,5 +39,11 @@ module Docs · 38 unmodified lines
39 &copy; 2006&ndash;2016 L&Ouml;VE Development Team<br>39 &copy; 2006&ndash;2016 L&Ouml;VE Development Team<br>
40 Licensed under the GNU Free Documentation License, Version 1.3.40 Licensed under the GNU Free Documentation License, Version 1.3.
41 HTML41 HTML
42
43 def get_latest_version(options, &block)
44 fetch_doc('https://love2d.org/wiki/Version_History', options) do |doc|
45 block.call doc.at_css('#mw-content-text table a').content
46 end
47 end
42 end48 end
43end49end
lib/docs/scrapers/lua.rb+6−0
@@ -26,5 +26,11 @@ module Docs · 25 unmodified lines
26 self.release = '5.1.5'26 self.release = '5.1.5'
27 self.base_url = 'https://www.lua.org/manual/5.1/'27 self.base_url = 'https://www.lua.org/manual/5.1/'
28 end28 end
29
30 def get_latest_version(options, &block)
31 fetch_doc('https://www.lua.org/manual/', options) do |doc|
32 block.call doc.at_css('p.menubar > a').content
33 end
34 end
29 end35 end
30end36end
lib/docs/scrapers/marionette.rb+4−0
@@ -38,5 +38,9 @@ module Docs · 37 unmodified lines
38 38
39 html_filters.push 'marionette/entries_v2'39 html_filters.push 'marionette/entries_v2'
40 end40 end
41
42 def get_latest_version(options, &block)
43 get_npm_version('backbone.marionette', options, &block)
44 end
41 end45 end
42end46end
lib/docs/scrapers/matplotlib.rb+6−0
@@ -64,5 +64,11 @@ module Docs · 63 unmodified lines
64 "https://matplotlib.org/#{release}/mpl_toolkits/axes_grid/api/"64 "https://matplotlib.org/#{release}/mpl_toolkits/axes_grid/api/"
65 ]65 ]
66 end66 end
67
68 def get_latest_version(options, &block)
69 get_latest_github_release('matplotlib', 'matplotlib', options) do |release|
70 block.call release['tag_name'][1..-1]
71 end
72 end
67 end73 end
68end74end
lib/docs/scrapers/meteor.rb+6−0
@@ -45,5 +45,11 @@ module Docs · 44 unmodified lines
45 self.base_urls = ['https://guide.meteor.com/v1.3/', "https://docs.meteor.com/v#{self.release}/"]45 self.base_urls = ['https://guide.meteor.com/v1.3/', "https://docs.meteor.com/v#{self.release}/"]
46 options[:fix_urls] = nil46 options[:fix_urls] = nil
47 end47 end
48
49 def get_latest_version(options, &block)
50 fetch_doc('https://docs.meteor.com/#/full/', options) do |doc|
51 block.call doc.at_css('select.version-select > option').content
52 end
53 end
48 end54 end
49end55end
lib/docs/scrapers/mocha.rb+4−0
@@ -18,5 +18,9 @@ module Docs · 17 unmodified lines
18 &copy; 2011&ndash;2018 JS Foundation and contributors<br>18 &copy; 2011&ndash;2018 JS Foundation and contributors<br>
19 Licensed under the Creative Commons Attribution 4.0 International License.19 Licensed under the Creative Commons Attribution 4.0 International License.
20 HTML20 HTML
21
22 def get_latest_version(options, &block)
23 get_npm_version('mocha', options, &block)
24 end
21 end25 end
22end26end
lib/docs/scrapers/modernizr.rb+4−0
@@ -15,5 +15,9 @@ module Docs · 14 unmodified lines
15 &copy; 2009&ndash;2017 The Modernizr team<br>15 &copy; 2009&ndash;2017 The Modernizr team<br>
16 Licensed under the MIT License.16 Licensed under the MIT License.
17 HTML17 HTML
18
19 def get_latest_version(options, &block)
20 get_npm_version('modernizr', options, &block)
21 end
18 end22 end
19end23end
lib/docs/scrapers/moment.rb+6−0
@@ -22,5 +22,11 @@ module Docs · 21 unmodified lines
22 &copy; JS Foundation and other contributors<br>22 &copy; JS Foundation and other contributors<br>
23 Licensed under the MIT License.23 Licensed under the MIT License.
24 HTML24 HTML
25
26 def get_latest_version(options, &block)
27 fetch_doc('http://momentjs.com/', options) do |doc|
28 block.call doc.at_css('.hero-title > h1 > span').content
29 end
30 end
25 end31 end
26end32end
lib/docs/scrapers/mongoose.rb+7−0
@@ -26,5 +26,12 @@ module Docs · 25 unmodified lines
26 &copy; 2010 LearnBoost<br>26 &copy; 2010 LearnBoost<br>
27 Licensed under the MIT License.27 Licensed under the MIT License.
28 HTML28 HTML
29
30 def get_latest_version(options, &block)
31 fetch_doc('https://mongoosejs.com/docs/', options) do |doc|
32 label = doc.at_css('.pure-menu-link').content.strip
33 block.call label.sub(/Version /, '')
34 end
35 end
29 end36 end
30end37end
lib/docs/scrapers/rdoc/minitest.rb+6−0
@@ -21,5 +21,11 @@ module Docs · 20 unmodified lines
21 &copy; Ryan Davis, seattle.rb<br>21 &copy; Ryan Davis, seattle.rb<br>
22 Licensed under the MIT License.22 Licensed under the MIT License.
23 HTML23 HTML
24
25 def get_latest_version(options, &block)
26 get_github_file_contents('seattlerb', 'minitest', 'History.rdoc', options) do |contents|
27 block.call contents.scan(/([0-9.]+)/)[0][0]
28 end
29 end
24 end30 end
25end31end
lib/docs/scrapers/rdoc/rails.rb+6−0
@@ -93,5 +93,11 @@ module Docs · 92 unmodified lines
93 version '4.1' do93 version '4.1' do
94 self.release = '4.1.16'94 self.release = '4.1.16'
95 end95 end
96
97 def get_latest_version(options, &block)
98 get_latest_github_release('rails', 'rails', options) do |release|
99 block.call release['name']
100 end
101 end
96 end102 end
97end103end
lib/docs/scrapers/rdoc/ruby.rb+12−0
@@ -84,5 +84,17 @@ module Docs · 83 unmodified lines
84 version '2.2' do84 version '2.2' do
85 self.release = '2.2.10'85 self.release = '2.2.10'
86 end86 end
87
88 def get_latest_version(options, &block)
89 get_github_tags('ruby', 'ruby', options) do |tags|
90 tags.each do |tag|
91 version = tag['name'].gsub(/_/, '.')[1..-1]
92 if !/^([0-9.]+)$/.match(version).nil? && version.count('.') == 2
93 block.call version
94 break
95 end
96 end
97 end
98 end
87 end99 end
88end100end
lib/tasks/updates.thor+254−32
@@ -1,4 +1,12 @@
1class UpdatesCLI < Thor1class UpdatesCLI < Thor
2 # The GitHub user that is allowed to upload reports
3 # TODO: Update this before creating a PR
4 UPLOAD_USER = 'jmerle'
5
6 # The repository to create an issue in when uploading the results
7 # TODO: Update this before creating a PR
8 UPLOAD_REPO = 'jmerle/devdocs'
9
2 def self.to_s10 def self.to_s
3 'Updates'11 'Updates'
4 end12 end
@@ -6,10 +14,14 @@ class UpdatesCLI < Thor · 1 unmodified line
6 def initialize(*args)14 def initialize(*args)
7 require 'docs'15 require 'docs'
8 require 'progress_bar'16 require 'progress_bar'
17 require 'terminal-table'
18 require 'date'
9 super19 super
10 end20 end
11 21
12 desc 'check [--verbose] [doc]...', 'Check for outdated documentations'22 desc 'check [--github-token] [--upload] [--verbose] [doc]...', 'Check for outdated documentations'
23 option :github_token, :type => :string
24 option :upload, :type => :boolean
13 option :verbose, :type => :boolean25 option :verbose, :type => :boolean
14 def check(*names)26 def check(*names)
15 # Convert names to a list of Scraper instances27 # Convert names to a list of Scraper instances
@@ -19,23 +31,26 @@ class UpdatesCLI < Thor · 3 unmodified lines
19 # Check all documentations for updates when no arguments are given31 # Check all documentations for updates when no arguments are given
20 docs = Docs.all if docs.empty?32 docs = Docs.all if docs.empty?
21 33
22 progress_bar = ::ProgressBar.new docs.length34 opts = {
23 progress_bar.write35 logger: logger
36 }
24 37
25 results = docs.map do |doc|38 if options.key?(:github_token)
26 result = check_doc(doc)39 opts[:github_token] = options[:github_token]
27 progress_bar.increment!
28 result
29 end40 end
30 41
31 valid_results = results.select {|result| result.is_a?(Hash)}42 with_progress_bar do |bar|
43 bar.max = docs.length
44 bar.write
45 end
32 46
33 up_to_date_results = valid_results.select {|result| !result[:is_outdated]}47 results = docs.map do |doc|
34 outdated_results = valid_results.select {|result| result[:is_outdated]}48 result = check_doc(doc, opts)
49 with_progress_bar(&:increment!)
50 result
51 end
35 52
36 log_results('Up-to-date', up_to_date_results) if options[:verbose] and !up_to_date_results.empty?53 process_results(results)
37 logger.info("") if options[:verbose] and !up_to_date_results.empty? and !outdated_results.empty?
38 log_results('Outdated', outdated_results) unless outdated_results.empty?
39 rescue Docs::DocNotFound => error54 rescue Docs::DocNotFound => error
40 logger.error(error)55 logger.error(error)
41 logger.info('Run "thor docs:list" to see the list of docs.')56 logger.info('Run "thor docs:list" to see the list of docs.')
@@ -43,53 +58,260 @@ class UpdatesCLI < Thor · 1 unmodified line
43 58
44 private59 private
45 60
46 def check_doc(doc)61 def check_doc(doc, options)
47 # Newer scraper versions always come before older scraper versions62 # Newer scraper versions always come before older scraper versions
48 # Therefore, the first item's release value is the latest current scraper version63 # Therefore, the first item's release value is the latest scraper version
49 #64 #
50 # For example, a scraper could scrape 3 versions: 10, 11 and 1265 # For example, a scraper could scrape 3 versions: 10, 11 and 12
51 # doc.versions.first would be the scraper for version 1266 # doc.versions.first would be the scraper for version 12
52 instance = doc.versions.first.new67 instance = doc.versions.first.new
53 68
54 return nil unless instance.class.method_defined?(:options)69 scraper_version = instance.class.method_defined?(:options) ? instance.options[:release] : nil
55 70 return error_result(doc, '`options[:release]` does not exist') if scraper_version.nil?
56 current_version = instance.options[:release]
57 return nil if current_version.nil?
58 71
59 logger.debug("Checking #{doc.name}")72 logger.debug("Checking #{doc.name}")
60 73
61 instance.get_latest_version do |latest_version|74 instance.get_latest_version(options) do |latest_version|
62 return {75 return {
63 name: doc.name,76 name: doc.name,
64 current_version: current_version,77 scraper_version: scraper_version,
65 latest_version: latest_version,78 latest_version: latest_version,
66 is_outdated: instance.is_outdated(current_version, latest_version)79 is_outdated: instance.is_outdated(scraper_version, latest_version)
67 }80 }
68 end81 end
69
70 return nil
71 rescue NotImplementedError82 rescue NotImplementedError
72 logger.warn("Couldn't check #{doc.name}, get_latest_version is not implemented")83 logger.warn("Couldn't check #{doc.name}, get_latest_version is not implemented")
84 error_result(doc, '`get_latest_version` is not implemented')
73 rescue85 rescue
74 logger.error("Error while checking #{doc.name}")86 logger.error("Error while checking #{doc.name}")
75 raise87 raise
76 end88 end
77 89
78 def log_results(label, results)90 def error_result(doc, reason)
79 logger.info("#{label} documentations (#{results.length}):")91 {
92 name: doc.name,
93 error: reason
94 }
95 end
96
97 def process_results(results)
98 successful_results = results.select {|result| result.key?(:is_outdated)}
99 failed_results = results.select {|result| result.key?(:error)}
100
101 up_to_date_results = successful_results.select {|result| !result[:is_outdated]}
102 outdated_results = successful_results.select {|result| result[:is_outdated]}
103
104 log_results(outdated_results, up_to_date_results, failed_results)
105 upload_results(outdated_results, up_to_date_results, failed_results) if options[:upload]
106 end
107
108 #
109 # Result logging methods
110 #
111
112 def log_results(outdated_results, up_to_date_results, failed_results)
113 log_failed_results(failed_results) unless failed_results.empty?
114 log_successful_results('Up-to-date', up_to_date_results) unless up_to_date_results.empty?
115 log_successful_results('Outdated', outdated_results) unless outdated_results.empty?
116 end
117
118 def log_successful_results(label, results)
119 title = "#{label} documentations (#{results.length})"
120 headings = ['Documentation', 'Scraper version', 'Latest version']
121 rows = results.map {|result| [result[:name], result[:scraper_version], result[:latest_version]]}
122
123 table = Terminal::Table.new :title => title, :headings => headings, :rows => rows
124 puts table
125 end
126
127 def log_failed_results(results)
128 title = "Documentations that could not be checked (#{results.length})"
129 headings = %w(Documentation Reason)
130 rows = results.map {|result| [result[:name], result[:error]]}
131
132 table = Terminal::Table.new :title => title, :headings => headings, :rows => rows
133 puts table
134 end
135
136 #
137 # Upload methods
138 #
139
140 def upload_results(outdated_results, up_to_date_results, failed_results)
141 # We can't create issues without a GitHub token
142 unless options.key?(:github_token)
143 logger.error('Please specify a GitHub token with the public_repo permission for devdocs-bot with the --github-token parameter')
144 return
145 end
146
147 logger.info('Uploading the results to a new GitHub issue')
148
149 logger.info('Checking if the GitHub token belongs to the correct user')
150 github_get('/user') do |user|
151 # Only allow the DevDocs bot to upload reports
152 if user['login'] == UPLOAD_USER
153 issue = results_to_issue(outdated_results, up_to_date_results, failed_results)
154
155 logger.info('Creating a new GitHub issue')
156 github_post("/repos/#{UPLOAD_REPO}/issues", issue) do |created_issue|
157 search_params = {
158 q: "Documentation versions report in:title author:#{UPLOAD_USER} is:issue repo:#{UPLOAD_REPO}",
159 sort: 'created',
160 order: 'desc'
161 }
162
163 logger.info('Checking if the previous issue is still open')
164 github_get('/search/issues', search_params) do |matching_issues|
165 previous_issue = matching_issues['items'].find {|item| item['number'] != created_issue['number']}
166
167 if previous_issue.nil?
168 logger.info('No previous issue found')
169 log_upload_success(created_issue)
170 else
171 comment = "This report was superseded by ##{created_issue['number']}."
172
173 logger.info('Commenting on the previous issue')
174 github_post("/repos/#{UPLOAD_REPO}/issues/#{previous_issue['number']}/comments", {body: comment}) do |_|
175 if previous_issue['closed_at'].nil?
176 logger.info('Closing the previous issue')
177 github_patch("/repos/#{UPLOAD_REPO}/issues/#{previous_issue['number']}", {state: 'closed'}) do |_|
178 log_upload_success(created_issue)
179 end
180 else
181 logger.info('The previous issue has already been closed')
182 log_upload_success(created_issue)
183 end
184 end
185 end
186 end
187 end
188 else
189 logger.error("Only #{UPLOAD_USER} is supposed to upload the results to a new issue. The specified github token is not for #{UPLOAD_USER}.")
190 end
191 end
192 end
193
194 def results_to_issue(outdated_results, up_to_date_results, failed_results)
195 results = [
196 successful_results_to_markdown('Outdated', outdated_results),
197 successful_results_to_markdown('Up-to-date', up_to_date_results),
198 failed_results_to_markdown(failed_results)
199 ]
200
201 results_str = results.select {|result| !result.nil?}.join("\n\n")
202
203 title = "Documentation versions report for #{Date.today.strftime('%B')} 2019"
204 body = <<-MARKDOWN
205## What is this?
206
207This is an automatically created issue which contains information about the version status of the documentations available on DevDocs. The results of this report can be used by maintainers when updating outdated documentations.
208
209Maintainers can close this issue when all documentations are up-to-date. This issue is automatically closed when the next report is created.
210
211## Results
212
213The #{outdated_results.length + up_to_date_results.length + failed_results.length} documentations are divided as follows:
214- #{outdated_results.length} that #{outdated_results.length == 1 ? 'is' : 'are'} outdated
215- #{up_to_date_results.length} that #{up_to_date_results.length == 1 ? 'is' : 'are'} up-to-date (patch updates are ignored)
216- #{failed_results.length} that could not be checked
217 MARKDOWN
218
219 {
220 title: title,
221 body: body.strip + "\n\n" + results_str
222 }
223 end
224
225 def successful_results_to_markdown(label, results)
226 return nil if results.empty?
227
228 title = "#{label} documentations (#{results.length})"
229 headings = ['Documentation', 'Scraper version', 'Latest version']
230 rows = results.map {|result| [result[:name], result[:scraper_version], result[:latest_version]]}
231
232 results_to_markdown(title, headings, rows)
233 end
234
235 def failed_results_to_markdown(results)
236 return nil if results.empty?
237
238 title = "Documentations that could not be checked (#{results.length})"
239 headings = %w(Documentation Reason)
240 rows = results.map {|result| [result[:name], result[:error]]}
241
242 results_to_markdown(title, headings, rows)
243 end
244
245 def results_to_markdown(title, headings, rows)
246 "<details>\n<summary>#{title}</summary>\n\n#{create_markdown_table(headings, rows)}\n</details>"
247 end
248
249 def create_markdown_table(headings, rows)
250 header = headings.join(' | ')
251 separator = '-|' * headings.length
252 body = rows.map {|row| row.join(' | ')}
253
254 header + "\n" + separator[0...-1] + "\n" + body.join("\n")
255 end
256
257 def log_upload_success(created_issue)
258 logger.info("Successfully uploaded the results to #{created_issue['html_url']}")
259 end
260
261 #
262 # HTTP utilities
263 #
264
265 def github_get(endpoint, params = {}, &block)
266 github_request(endpoint, {method: :get, params: params}, &block)
267 end
268
269 def github_post(endpoint, params, &block)
270 github_request(endpoint, {method: :post, body: params.to_json}, &block)
271 end
272
273 def github_patch(endpoint, params, &block)
274 github_request(endpoint, {method: :patch, body: params.to_json}, &block)
275 end
80 276
81 results.each do |result|277 def github_request(endpoint, opts, &block)
82 logger.info("#{result[:name]}: #{result[:current_version]} -> #{result[:latest_version]}")278 url = "https://api.github.com#{endpoint}"
279
280 # GitHub token authentication
281 opts[:headers] = {
282 Authorization: "token #{options[:github_token]}"
283 }
284
285 # GitHub requires the Content-Type to be application/json when a body is passed
286 if opts.key?(:body)
287 opts[:headers]['Content-Type'] = 'application/json'
83 end288 end
289
290 logger.debug("Making a #{opts[:method]} request to #{url}")
291
292 Docs::Request.run(url, opts) do |response|
293 # response.success? is false if the response code is 201
294 # GitHub returns 201 Created after an issue is created
295 if response.success? || response.code == 201
296 block.call JSON.parse(response.body)
297 else
298 logger.error("Couldn't make a #{opts[:method]} request to #{url} (response code #{response.code})")
299 block.call nil
300 end
301 end
302 end
303
304 # A utility method which ensures no progress bar is shown when stdout is not a tty
305 def with_progress_bar(&block)
306 return unless $stdout.tty?
307 @progress_bar ||= ::ProgressBar.new
308 block.call @progress_bar
84 end309 end
85 310
86 def logger311 def logger
87 @logger ||= Logger.new($stdout).tap do |logger|312 @logger ||= Logger.new($stdout).tap do |logger|
88 logger.level = options[:verbose] ? Logger::DEBUG : Logger::INFO313 logger.level = options[:verbose] ? Logger::DEBUG : Logger::INFO
89 logger.formatter = proc do |severity, datetime, progname, msg|314 logger.formatter = proc {|severity, datetime, progname, msg| "[#{severity}] #{msg}\n"}
90 prefix = severity != "INFO" ? "[#{severity}] " : ""
91 "#{prefix}#{msg}\n"
92 end
93 end315 end
94 end316 end
95end317end