History

Finish get_latest_version for 81 scrapers and add uploading functionality

Jasper van Merle committed 7 years ago

Commit 3dc17a91 parent 2d1e8aa85 files · +739 −68Browse files

Unified · Split

85 files
  1. Gemfile +1 −0
  2. Gemfile.lock +3 −0
  3. lib/docs/core/scraper.rb +40 −16
  4. lib/docs/scrapers/angular.rb +2 −2
  5. lib/docs/scrapers/angularjs.rb +2 −2
  6. lib/docs/scrapers/ansible.rb +2 −2
  7. lib/docs/scrapers/apache.rb +2 −2
  8. lib/docs/scrapers/apache_pig.rb +2 −2
  9. lib/docs/scrapers/async.rb +2 −2
  10. lib/docs/scrapers/babel.rb +2 −2
  11. lib/docs/scrapers/backbone.rb +2 −2
  12. lib/docs/scrapers/bash.rb +2 −2
  13. lib/docs/scrapers/bluebird.rb +2 −2
  14. lib/docs/scrapers/bootstrap.rb +6 −0
  15. lib/docs/scrapers/bottle.rb +7 −0
  16. lib/docs/scrapers/bower.rb +4 −0
  17. lib/docs/scrapers/cakephp.rb +6 −0
  18. lib/docs/scrapers/chai.rb +4 −0
  19. lib/docs/scrapers/chef.rb +7 −0
  20. lib/docs/scrapers/clojure.rb +6 −0
  21. lib/docs/scrapers/cmake.rb +7 −0
  22. lib/docs/scrapers/codeception.rb +6 −0
  23. lib/docs/scrapers/codeceptjs.rb +4 −0
  24. lib/docs/scrapers/codeigniter.rb +7 −0
  25. lib/docs/scrapers/coffeescript.rb +4 −0
  26. lib/docs/scrapers/cordova.rb +9 −0
  27. lib/docs/scrapers/crystal.rb +6 −0
  28. lib/docs/scrapers/d.rb +6 −0
  29. lib/docs/scrapers/d3.rb +4 −0
  30. lib/docs/scrapers/dart.rb +7 −0
  31. lib/docs/scrapers/django.rb +6 −0
  32. lib/docs/scrapers/docker.rb +7 −0
  33. lib/docs/scrapers/dojo.rb +6 −0
  34. lib/docs/scrapers/drupal.rb +9 −0
  35. lib/docs/scrapers/electron.rb +6 −0
  36. lib/docs/scrapers/elixir.rb +6 −0
  37. lib/docs/scrapers/ember.rb +6 −0
  38. lib/docs/scrapers/erlang.rb +6 −0
  39. lib/docs/scrapers/eslint.rb +4 −0
  40. lib/docs/scrapers/express.rb +4 −0
  41. lib/docs/scrapers/falcon.rb +6 −0
  42. lib/docs/scrapers/fish.rb +6 −0
  43. lib/docs/scrapers/flow.rb +4 −0
  44. lib/docs/scrapers/git.rb +6 −0
  45. lib/docs/scrapers/gnu/gcc.rb +7 −0
  46. lib/docs/scrapers/gnu/gnu_fortran.rb +7 −0
  47. lib/docs/scrapers/go.rb +9 −0
  48. lib/docs/scrapers/godot.rb +6 −0
  49. lib/docs/scrapers/graphite.rb +6 −0
  50. lib/docs/scrapers/grunt.rb +4 −0
  51. lib/docs/scrapers/handlebars.rb +4 −0
  52. lib/docs/scrapers/haskell.rb +7 −0
  53. lib/docs/scrapers/haxe.rb +7 −0
  54. lib/docs/scrapers/homebrew.rb +6 −0
  55. lib/docs/scrapers/immutable.rb +4 −0
  56. lib/docs/scrapers/influxdata.rb +7 −0
  57. lib/docs/scrapers/jasmine.rb +6 −0
  58. lib/docs/scrapers/jekyll.rb +6 −0
  59. lib/docs/scrapers/jest.rb +6 −0
  60. lib/docs/scrapers/jquery/jquery_core.rb +4 −0
  61. lib/docs/scrapers/jquery/jquery_mobile.rb +7 −0
  62. lib/docs/scrapers/jquery/jquery_ui.rb +4 −0
  63. lib/docs/scrapers/jsdoc.rb +6 −0
  64. lib/docs/scrapers/julia.rb +6 −0
  65. lib/docs/scrapers/knockout.rb +6 −0
  66. lib/docs/scrapers/koa.rb +4 −0
  67. lib/docs/scrapers/kotlin.rb +6 −0
  68. lib/docs/scrapers/laravel.rb +6 −0
  69. lib/docs/scrapers/leaflet.rb +6 −0
  70. lib/docs/scrapers/less.rb +7 −0
  71. lib/docs/scrapers/liquid.rb +6 −0
  72. lib/docs/scrapers/lodash.rb +6 −0
  73. lib/docs/scrapers/love.rb +6 −0
  74. lib/docs/scrapers/lua.rb +6 −0
  75. lib/docs/scrapers/marionette.rb +4 −0
  76. lib/docs/scrapers/matplotlib.rb +6 −0
  77. lib/docs/scrapers/meteor.rb +6 −0
  78. lib/docs/scrapers/mocha.rb +4 −0
  79. lib/docs/scrapers/modernizr.rb +4 −0
  80. lib/docs/scrapers/moment.rb +6 −0
  81. lib/docs/scrapers/mongoose.rb +7 −0
  82. lib/docs/scrapers/rdoc/minitest.rb +6 −0
  83. lib/docs/scrapers/rdoc/rails.rb +6 −0
  84. lib/docs/scrapers/rdoc/ruby.rb +12 −0
  85. lib/tasks/updates.thor +254 −32
Gemfile+1−0
@@ -40,6 +40,7 @@ group :docs do · 39 unmodified lines
4040 gem 'unix_utils', require: false
4141 gem 'tty-pager', require: false
4242 gem 'net-sftp', '>= 2.1.3.rc2', require: false
43+ gem 'terminal-table', require: false
4344 end
4445
4546 group :test do
Gemfile.lock+3−0
@@ -101,6 +101,8 @@ GEM · 100 unmodified lines
101101 unicode-display_width (~> 1.4.0)
102102 unicode_utils (~> 1.4.0)
103103 strings-ansi (0.1.0)
104+ terminal-table (1.8.0)
105+ unicode-display_width (~> 1.1, >= 1.1.1)
104106 thin (1.7.2)
105107 daemons (~> 1.0, >= 1.0.9)
106108 eventmachine (~> 1.0, >= 1.0.4)
@@ -153,6 +155,7 @@ DEPENDENCIES · 46 unmodified lines
153155 sinatra-contrib
154156 sprockets
155157 sprockets-helpers
158+ terminal-table
156159 thin
157160 thor
158161 tty-pager
lib/docs/core/scraper.rb+40−16
@@ -132,7 +132,7 @@ module Docs · 131 unmodified lines
132132 end
133133 end
134134
135− def get_latest_version(&block)
135+ def get_latest_version(options, &block)
136136 raise NotImplementedError
137137 end
138138
@@ -147,15 +147,15 @@ module Docs · 8 unmodified lines
147147 # 1 -> 2 = outdated
148148 # 1.1 -> 1.2 = outdated
149149 # 1.1.1 -> 1.1.2 = not outdated
150− def is_outdated(current_version, latest_version)
151− current_parts = current_version.split(/\./).map(&:to_i)
150+ def is_outdated(scraper_version, latest_version)
151+ scraper_parts = scraper_version.split(/\./).map(&:to_i)
152152 latest_parts = latest_version.split(/\./).map(&:to_i)
153153
154154 # Only check the first two parts, the third part is for patch updates
155155 [0, 1].each do |i|
156− break if i >= current_parts.length or i >= latest_parts.length
157− return true if latest_parts[i] > current_parts[i]
158− return false if latest_parts[i] < current_parts[i]
156+ break if i >= scraper_parts.length or i >= latest_parts.length
157+ return true if latest_parts[i] > scraper_parts[i]
158+ return false if latest_parts[i] < scraper_parts[i]
159159 end
160160
161161 false
@@ -231,38 +231,62 @@ module Docs · 69 unmodified lines
231231 {}
232232 end
233233
234+ #
234235 # Utility methods for get_latest_version
236+ #
235237
236− def fetch(url, &block)
237− Request.run(url) do |response|
238+ def fetch(url, options, &block)
239+ headers = {}
240+
241+ if options.key?(:github_token) and url.start_with?('https://api.github.com/')
242+ headers['Authorization'] = "token #{options[:github_token]}"
243+ end
244+
245+ options[:logger].debug("Fetching #{url}")
246+
247+ Request.run(url, { headers: headers }) do |response|
238248 if response.success?
239249 block.call response.body
240250 else
251+ options[:logger].error("Couldn't fetch #{url} (response code #{response.code})")
241252 block.call nil
242253 end
243254 end
244255 end
245256
246− def fetch_doc(url, &block)
247− fetch(url) do |body|
248− parser = Parser.new(body)
249− block.call parser.html
257+ def fetch_doc(url, options, &block)
258+ fetch(url, options) do |body|
259+ block.call Nokogiri::HTML.parse body, nil, 'UTF-8'
250260 end
251261 end
252262
253− def fetch_json(url, &block)
254− fetch(url) do |body|
263+ def fetch_json(url, options, &block)
264+ fetch(url, options) do |body|
255265 json = JSON.parse(body)
256266 block.call json
257267 end
258268 end
259269
260− def get_npm_version(package, &block)
261− fetch_json("https://registry.npmjs.com/#{package}") do |json|
270+ def get_npm_version(package, options, &block)
271+ fetch_json("https://registry.npmjs.com/#{package}", options) do |json|
262272 block.call json['dist-tags']['latest']
263273 end
264274 end
265275
276+ def get_latest_github_release(owner, repo, options, &block)
277+ fetch_json("https://api.github.com/repos/#{owner}/#{repo}/releases/latest", options, &block)
278+ end
279+
280+ def get_github_tags(owner, repo, options, &block)
281+ fetch_json("https://api.github.com/repos/#{owner}/#{repo}/tags", options, &block)
282+ end
283+
284+ def get_github_file_contents(owner, repo, path, options, &block)
285+ fetch_json("https://api.github.com/repos/#{owner}/#{repo}/contents/#{path}", options) do |json|
286+ block.call(Base64.decode64(json['content']))
287+ end
288+ end
289+
266290 module FixInternalUrlsBehavior
267291 def self.included(base)
268292 base.extend ClassMethods
lib/docs/scrapers/angular.rb+2−2
@@ -155,8 +155,8 @@ module Docs · 154 unmodified lines
155155 end
156156 end
157157
158− def get_latest_version(&block)
159− get_npm_version('@angular/core', &block)
158+ def get_latest_version(options, &block)
159+ get_npm_version('@angular/core', options, &block)
160160 end
161161
162162 private
lib/docs/scrapers/angularjs.rb+2−2
@@ -70,8 +70,8 @@ module Docs · 69 unmodified lines
7070 self.base_url = "https://code.angularjs.org/#{release}/docs/partials/"
7171 end
7272
73− def get_latest_version(&block)
74− get_npm_version('angular', &block)
73+ def get_latest_version(options, &block)
74+ get_npm_version('angular', options, &block)
7575 end
7676 end
7777 end
lib/docs/scrapers/ansible.rb+2−2
@@ -88,8 +88,8 @@ module Docs · 87 unmodified lines
8888 list_of_all_modules.html)
8989 end
9090
91− def get_latest_version(&block)
92− fetch_doc('https://docs.ansible.com/ansible/latest/index.html') do |doc|
91+ def get_latest_version(options, &block)
92+ fetch_doc('https://docs.ansible.com/ansible/latest/index.html', options) do |doc|
9393 block.call doc.at_css('.DocSiteProduct-CurrentVersion').content.strip
9494 end
9595 end
lib/docs/scrapers/apache.rb+2−2
@@ -34,8 +34,8 @@ module Docs · 33 unmodified lines
3434 Licensed under the Apache License, Version 2.0.
3535 HTML
3636
37− def get_latest_version(&block)
38− fetch_doc('http://httpd.apache.org/docs/') do |doc|
37+ def get_latest_version(options, &block)
38+ fetch_doc('http://httpd.apache.org/docs/', options) do |doc|
3939 block.call doc.at_css('#apcontents > ul a')['href'][0...-1]
4040 end
4141 end
lib/docs/scrapers/apache_pig.rb+2−2
@@ -43,8 +43,8 @@ module Docs · 42 unmodified lines
4343 self.base_url = "https://pig.apache.org/docs/r#{release}/"
4444 end
4545
46− def get_latest_version(&block)
47− fetch_doc('https://pig.apache.org/') do |doc|
46+ def get_latest_version(options, &block)
47+ fetch_doc('https://pig.apache.org/', options) do |doc|
4848 item = doc.at_css('div[id="menu_1.2"] > .menuitem:last-child')
4949 block.call item.content.strip.sub(/Release /, '')
5050 end
lib/docs/scrapers/async.rb+2−2
@@ -18,8 +18,8 @@ module Docs · 17 unmodified lines
1818 Licensed under the MIT License.
1919 HTML
2020
21− def get_latest_version(&block)
22− fetch_doc('https://caolan.github.io/async/') do |doc|
21+ def get_latest_version(options, &block)
22+ fetch_doc('https://caolan.github.io/async/', options) do |doc|
2323 version = doc.at_css('#version-dropdown > a').content.strip[1..-1]
2424 block.call version
2525 end
lib/docs/scrapers/babel.rb+2−2
@@ -23,8 +23,8 @@ module Docs · 22 unmodified lines
2323 '<div></div>'
2424 end
2525
26− def get_latest_version(&block)
27− fetch_doc('https://babeljs.io/docs/en/') do |doc|
26+ def get_latest_version(options, &block)
27+ fetch_doc('https://babeljs.io/docs/en/', options) do |doc|
2828 block.call doc.at_css('a[href="/versions"] > h3').content
2929 end
3030 end
lib/docs/scrapers/backbone.rb+2−2
@@ -21,8 +21,8 @@ module Docs · 20 unmodified lines
2121 Licensed under the MIT License.
2222 HTML
2323
24− def get_latest_version(&block)
25− fetch_doc('https://backbonejs.org/') do |doc|
24+ def get_latest_version(options, &block)
25+ fetch_doc('https://backbonejs.org/', options) do |doc|
2626 version = doc.at_css('.version').content
2727 block.call version[1...-1]
2828 end
lib/docs/scrapers/bash.rb+2−2
@@ -18,8 +18,8 @@ module Docs · 17 unmodified lines
1818 Licensed under the GNU Free Documentation License.
1919 HTML
2020
21− def get_latest_version(&block)
22− fetch('https://www.gnu.org/software/bash/manual/html_node/index.html') do |body|
21+ def get_latest_version(options, &block)
22+ fetch('https://www.gnu.org/software/bash/manual/html_node/index.html', options) do |body|
2323 version = body.scan(/, Version ([0-9.]+)/)[0][0]
2424 block.call version[0...-1]
2525 end
lib/docs/scrapers/bluebird.rb+2−2
@@ -19,8 +19,8 @@ module Docs · 18 unmodified lines
1919 Licensed under the MIT License.
2020 HTML
2121
22− def get_latest_version(&block)
23− get_npm_version('bluebird', &block)
22+ def get_latest_version(options, &block)
23+ get_npm_version('bluebird', options, &block)
2424 end
2525 end
2626 end
lib/docs/scrapers/bootstrap.rb+6−0
@@ -34,5 +34,11 @@ module Docs · 33 unmodified lines
3434
3535 options[:only] = %w(getting-started/ css/ components/ javascript/)
3636 end
37+
38+ def get_latest_version(options, &block)
39+ fetch_doc('https://getbootstrap.com/', options) do |doc|
40+ block.call doc.at_css('#bd-versions').content.strip[1..-1]
41+ end
42+ end
3743 end
3844 end
lib/docs/scrapers/bottle.rb+7−0
@@ -27,5 +27,12 @@ module Docs · 26 unmodified lines
2727 self.release = '0.11.7'
2828 self.base_url = "https://bottlepy.org/docs/#{self.version}/"
2929 end
30+
31+ def get_latest_version(options, &block)
32+ fetch_doc('https://bottlepy.org/docs/stable/', options) do |doc|
33+ label = doc.at_css('.sphinxsidebarwrapper > ul > li > b')
34+ block.call label.content.sub(/Bottle /, '')
35+ end
36+ end
3037 end
3138 end
lib/docs/scrapers/bower.rb+4−0
@@ -19,5 +19,9 @@ module Docs · 18 unmodified lines
1919 &copy; 2018 Bower contributors<br>
2020 Licensed under the MIT License.
2121 HTML
22+
23+ def get_latest_version(options, &block)
24+ get_npm_version('bower', options, &block)
25+ end
2226 end
2327 end
lib/docs/scrapers/cakephp.rb+6−0
@@ -71,6 +71,12 @@ module Docs · 70 unmodified lines
7171 self.base_url = 'https://api.cakephp.org/2.7/'
7272 end
7373
74+ def get_latest_version(options, &block)
75+ fetch_doc('https://api.cakephp.org/3.7/', options) do |doc|
76+ block.call doc.at_css('.version-picker .dropdown-toggle').content.strip
77+ end
78+ end
79+
7480 private
7581
7682 def parse(response)
lib/docs/scrapers/chai.rb+4−0
@@ -23,5 +23,9 @@ module Docs · 22 unmodified lines
2323 &copy; 2016 Chai.js Assertion Library<br>
2424 Licensed under the MIT License.
2525 HTML
26+
27+ def get_latest_version(options, &block)
28+ get_npm_version('chai', options, &block)
29+ end
2630 end
2731 end
lib/docs/scrapers/chef.rb+7−0
@@ -47,5 +47,12 @@ module Docs · 46 unmodified lines
4747
4848 options[:only_patterns] = [/\A#{client_path}\//, /\A#{server_path}\//]
4949 end
50+
51+ def get_latest_version(options, &block)
52+ fetch_doc('https://docs-archive.chef.io/', options) do |doc|
53+ cell = doc.at_css('.main-archives > tr:nth-child(2) > td:nth-child(2)')
54+ block.call cell.content.sub(/Chef Client /, '')
55+ end
56+ end
5057 end
5158 end
lib/docs/scrapers/clojure.rb+6−0
@@ -27,5 +27,11 @@ module Docs · 26 unmodified lines
2727 self.release = '1.7'
2828 self.base_url = 'https://clojure.github.io/clojure/branch-clojure-1.7.0/'
2929 end
30+
31+ def get_latest_version(options, &block)
32+ fetch_doc('http://clojure.github.io/clojure/index.html', options) do |doc|
33+ block.call doc.at_css('#header-version').content[1..-1]
34+ end
35+ end
3036 end
3137 end
lib/docs/scrapers/cmake.rb+7−0
@@ -59,5 +59,12 @@ module Docs · 58 unmodified lines
5959 self.release = '3.5.2'
6060 self.base_url = 'https://cmake.org/cmake/help/v3.5/'
6161 end
62+
63+ def get_latest_version(options, &block)
64+ fetch_doc('https://cmake.org/documentation/', options) do |doc|
65+ link = doc.at_css('.entry-content ul > li > strong > a > big')
66+ block.call link.content.scan(/([0-9.]+)/)[0][0]
67+ end
68+ end
6269 end
6370 end
lib/docs/scrapers/codeception.rb+6−0
@@ -18,5 +18,11 @@ module Docs · 17 unmodified lines
1818 &copy; 2011 Michael Bodnarchuk and contributors<br>
1919 Licensed under the MIT License.
2020 HTML
21+
22+ def get_latest_version(options, &block)
23+ fetch_doc('https://codeception.com/changelog', options) do |doc|
24+ block.call doc.at_css('#page > h4').content
25+ end
26+ end
2127 end
2228 end
lib/docs/scrapers/codeceptjs.rb+4−0
@@ -21,5 +21,9 @@ module Docs · 20 unmodified lines
2121 &copy; 2015 DavertMik &lt;davert@codegyre.com&gt; (http://codegyre.com)<br>
2222 Licensed under the MIT License.
2323 HTML
24+
25+ def get_latest_version(options, &block)
26+ get_npm_version('codeceptjs', options, &block)
27+ end
2428 end
2529 end
lib/docs/scrapers/codeigniter.rb+7−0
@@ -38,5 +38,12 @@ module Docs · 37 unmodified lines
3838 version '3' do
3939 self.release = '3.1.8'
4040 end
41+
42+ def get_latest_version(options, &block)
43+ fetch_doc('https://codeigniter.com/user_guide/changelog.html', options) do |doc|
44+ header = doc.at_css('#change-log h2')
45+ block.call header.content.scan(/([0-9.]+)/)[0][0]
46+ end
47+ end
4148 end
4249 end
lib/docs/scrapers/coffeescript.rb+4−0
@@ -30,5 +30,9 @@ module Docs · 29 unmodified lines
3030
3131 options[:container] = '.container'
3232 end
33+
34+ def get_latest_version(options, &block)
35+ get_npm_version('coffeescript', options, &block)
36+ end
3337 end
3438 end
lib/docs/scrapers/cordova.rb+9−0
@@ -42,5 +42,14 @@ module Docs · 41 unmodified lines
4242 self.release = '6.5.0'
4343 self.base_url = 'https://cordova.apache.org/docs/en/6.x/'
4444 end
45+
46+ def get_latest_version(options, &block)
47+ fetch_doc('https://cordova.apache.org/docs/en/latest/', options) do |doc|
48+ label = doc.at_css('#versionDropdown').content.strip
49+ version = label.scan(/([0-9.]+)/)[0][0]
50+ version = version[0...-1] if version.end_with?('.')
51+ block.call version
52+ end
53+ end
4554 end
4655 end
lib/docs/scrapers/crystal.rb+6−0
@@ -34,5 +34,11 @@ module Docs · 33 unmodified lines
3434 HTML
3535 end
3636 }
37+
38+ def get_latest_version(options, &block)
39+ fetch('https://crystal-lang.org/api', options) do |body|
40+ block.call body.scan(/Crystal Docs ([0-9.]+)/)[0][0]
41+ end
42+ end
3743 end
3844 end
lib/docs/scrapers/d.rb+6−0
@@ -26,5 +26,11 @@ module Docs · 25 unmodified lines
2626 def initial_urls
2727 %w(https://dlang.org/phobos/index.html https://dlang.org/spec/intro.html)
2828 end
29+
30+ def get_latest_version(options, &block)
31+ fetch_doc('https://dlang.org/changelog/', options) do |doc|
32+ block.call doc.at_css('#content > ul > li:nth-child(2) > a')['id']
33+ end
34+ end
2935 end
3036 end
lib/docs/scrapers/d3.rb+4−0
@@ -58,5 +58,9 @@ module Docs · 57 unmodified lines
5858 options[:root_title] = 'D3.js'
5959 options[:only_patterns] = [/\.md\z/]
6060 end
61+
62+ def get_latest_version(options, &block)
63+ get_npm_version('d3', options, &block)
64+ end
6165 end
6266 end
lib/docs/scrapers/dart.rb+7−0
@@ -31,5 +31,12 @@ module Docs · 30 unmodified lines
3131 self.release = '1.24.3'
3232 self.base_url = "https://api.dartlang.org/stable/#{release}/"
3333 end
34+
35+ def get_latest_version(options, &block)
36+ fetch_doc('https://api.dartlang.org/', options) do |doc|
37+ label = doc.at_css('footer > span').content.strip
38+ block.call label.sub(/Dart /, '')
39+ end
40+ end
3441 end
3542 end
lib/docs/scrapers/django.rb+6−0
@@ -63,5 +63,11 @@ module Docs · 62 unmodified lines
6363 self.release = '1.8.18'
6464 self.base_url = 'https://docs.djangoproject.com/en/1.8/'
6565 end
66+
67+ def get_latest_version(options, &block)
68+ fetch_doc('https://docs.djangoproject.com/', options) do |doc|
69+ block.call doc.at_css('#doc-versions > li.current > span > strong').content
70+ end
71+ end
6672 end
6773 end
lib/docs/scrapers/docker.rb+7−0
@@ -137,5 +137,12 @@ module Docs · 136 unmodified lines
137137 options[:container] = '#docs'
138138 options[:only_patterns] << /\Aswarm\//
139139 end
140+
141+ def get_latest_version(options, &block)
142+ fetch_doc('https://docs.docker.com/', options) do |doc|
143+ label = doc.at_css('.nav-container button.dropdown-toggle').content.strip
144+ block.call label.scan(/([0-9.]+)/)[0][0]
145+ end
146+ end
140147 end
141148 end
lib/docs/scrapers/dojo.rb+6−0
@@ -36,6 +36,12 @@ module Docs · 35 unmodified lines
3636 urls.map { |url| "<a href='#{url}'>#{url}</a>" }.join
3737 end
3838
39+ def get_latest_version(options, &block)
40+ fetch_doc('https://dojotoolkit.org/api/', options) do |doc|
41+ block.call doc.at_css('#versionSelector > option[selected]').content
42+ end
43+ end
44+
3945 private
4046
4147 def get_url_list(json, set = Set.new)
lib/docs/scrapers/drupal.rb+9−0
@@ -98,5 +98,14 @@ module Docs · 97 unmodified lines
9898 /\A[\w\-\.]+\.php\/7\.x\z/
9999 ]
100100 end
101+
102+ def get_latest_version(options, &block)
103+ fetch_doc('http://cgit.drupalcode.org/drupal', options) do |doc|
104+ version = doc.at_css('td.form > form > select > option[selected]').content
105+ version = version.scan(/([0-9.]+)/)[0][0]
106+ version = version[0...-1] if version.end_with?('.')
107+ block.call version
108+ end
109+ end
101110 end
102111 end
lib/docs/scrapers/electron.rb+6−0
@@ -22,5 +22,11 @@ module Docs · 21 unmodified lines
2222 &copy; 2013&ndash;2018 GitHub Inc.<br>
2323 Licensed under the MIT license.
2424 HTML
25+
26+ def get_latest_version(options, &block)
27+ fetch_doc('https://electronjs.org/docs', options) do |doc|
28+ block.call doc.at_css('.docs-version').content
29+ end
30+ end
2531 end
2632 end
lib/docs/scrapers/elixir.rb+6−0
@@ -97,5 +97,11 @@ module Docs · 96 unmodified lines
9797 'https://elixir-lang.org/getting-started/'
9898 ]
9999 end
100+
101+ def get_latest_version(options, &block)
102+ fetch_doc('https://hexdocs.pm/elixir/api-reference.html', options) do |doc|
103+ block.call doc.at_css('h2.sidebar-projectVersion').content.strip[1..-1]
104+ end
105+ end
100106 end
101107 end
lib/docs/scrapers/ember.rb+6−0
@@ -56,5 +56,11 @@ module Docs · 55 unmodified lines
5656 https://emberjs.com/api/ember-data/2.14/classes/DS
5757 )
5858 end
59+
60+ def get_latest_version(options, &block)
61+ fetch_doc('https://emberjs.com/api/ember/release', options) do |doc|
62+ block.call doc.at_css('.sidebar > .select-container .ember-power-select-selected-item').content.strip
63+ end
64+ end
5965 end
6066 end
lib/docs/scrapers/erlang.rb+6−0
@@ -55,5 +55,11 @@ module Docs · 54 unmodified lines
5555 version '18' do
5656 self.release = '18.3'
5757 end
58+
59+ def get_latest_version(options, &block)
60+ fetch_doc('https://www.erlang.org/downloads', options) do |doc|
61+ block.call doc.at_css('.col-lg-3 > ul > li').content.strip
62+ end
63+ end
5864 end
5965 end
lib/docs/scrapers/eslint.rb+4−0
@@ -20,5 +20,9 @@ module Docs · 19 unmodified lines
2020 &copy; JS Foundation and other contributors<br>
2121 Licensed under the MIT License.
2222 HTML
23+
24+ def get_latest_version(options, &block)
25+ get_npm_version('eslint', options, &block)
26+ end
2327 end
2428 end
lib/docs/scrapers/express.rb+4−0
@@ -28,5 +28,9 @@ module Docs · 27 unmodified lines
2828 &copy; 2017 StrongLoop, IBM, and other expressjs.com contributors.<br>
2929 Licensed under the Creative Commons Attribution-ShareAlike License v3.0.
3030 HTML
31+
32+ def get_latest_version(options, &block)
33+ get_npm_version('express', options, &block)
34+ end
3135 end
3236 end
lib/docs/scrapers/falcon.rb+6−0
@@ -33,5 +33,11 @@ module Docs · 32 unmodified lines
3333 self.release = '1.2.0'
3434 self.base_url = "https://falcon.readthedocs.io/en/#{self.release}/"
3535 end
36+
37+ def get_latest_version(options, &block)
38+ fetch_doc('https://falcon.readthedocs.io/en/stable/changes/index.html', options) do |doc|
39+ block.call doc.at_css('#changelogs ul > li > a').content
40+ end
41+ end
3642 end
3743 end
lib/docs/scrapers/fish.rb+6−0
@@ -46,5 +46,11 @@ module Docs · 45 unmodified lines
4646 self.release = '2.2.0'
4747 self.base_url = "https://fishshell.com/docs/#{version}/"
4848 end
49+
50+ def get_latest_version(options, &block)
51+ fetch_doc('http://fishshell.com/docs/current/index.html', options) do |doc|
52+ block.call doc.at_css('#toc-index').content.scan(/([0-9.]+)/)[0][0]
53+ end
54+ end
4955 end
5056 end
lib/docs/scrapers/flow.rb+4−0
@@ -18,5 +18,9 @@ module Docs · 17 unmodified lines
1818 &copy; 2013&ndash;present Facebook Inc.<br>
1919 Licensed under the MIT License.
2020 HTML
21+
22+ def get_latest_version(options, &block)
23+ get_npm_version('flow-bin', options, &block)
24+ end
2125 end
2226 end
lib/docs/scrapers/git.rb+6−0
@@ -19,5 +19,11 @@ module Docs · 18 unmodified lines
1919 &copy; 2005&ndash;2018 Linus Torvalds and others<br>
2020 Licensed under the GNU General Public License version 2.
2121 HTML
22+
23+ def get_latest_version(options, &block)
24+ fetch_doc('https://git-scm.com/', options) do |doc|
25+ block.call doc.at_css('.version').content.strip
26+ end
27+ end
2228 end
2329 end
lib/docs/scrapers/gnu/gcc.rb+7−0
@@ -99,5 +99,12 @@ module Docs · 98 unmodified lines
9999
100100 options[:replace_paths] = CPP_PATHS
101101 end
102+
103+ def get_latest_version(options, &block)
104+ fetch_doc('https://gcc.gnu.org/onlinedocs/', options) do |doc|
105+ label = doc.at_css('ul > li > ul > li > a').content.strip
106+ block.call label.scan(/([0-9.]+)/)[0][0]
107+ end
108+ end
102109 end
103110 end
lib/docs/scrapers/gnu/gnu_fortran.rb+7−0
@@ -25,5 +25,12 @@ module Docs · 24 unmodified lines
2525 self.release = '4.9.3'
2626 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
2727 end
28+
29+ def get_latest_version(options, &block)
30+ fetch_doc('https://gcc.gnu.org/onlinedocs/', options) do |doc|
31+ label = doc.at_css('ul > li > ul > li > a').content.strip
32+ block.call label.scan(/([0-9.]+)/)[0][0]
33+ end
34+ end
2835 end
2936 end
lib/docs/scrapers/go.rb+9−0
@@ -24,6 +24,15 @@ module Docs · 23 unmodified lines
2424 Licensed under the Creative Commons Attribution License 3.0.
2525 HTML
2626
27+ def get_latest_version(options, &block)
28+ fetch_doc('https://golang.org/pkg/', options) do |doc|
29+ footer = doc.at_css('#footer').content
30+ version = footer.scan(/go([0-9.]+)/)[0][0]
31+ version = version[0...-1] if version.end_with?('.')
32+ block.call version
33+ end
34+ end
35+
2736 private
2837
2938 def parse(response) # Hook here because Nokogori removes whitespace from textareas
lib/docs/scrapers/godot.rb+6−0
@@ -37,5 +37,11 @@ module Docs · 36 unmodified lines
3737 self.release = '2.1'
3838 self.base_url = "http://docs.godotengine.org/en/#{self.version}/"
3939 end
40+
41+ def get_latest_version(options, &block)
42+ fetch_doc('https://docs.godotengine.org/', options) do |doc|
43+ block.call doc.at_css('.version').content.strip
44+ end
45+ end
4046 end
4147 end
lib/docs/scrapers/graphite.rb+6−0
@@ -17,5 +17,11 @@ module Docs · 16 unmodified lines
1717 &copy; 2011&ndash;2016 The Graphite Project<br>
1818 Licensed under the Apache License, Version 2.0.
1919 HTML
20+
21+ def get_latest_version(options, &block)
22+ fetch_doc('https://graphite.readthedocs.io/en/latest/releases.html', options) do |doc|
23+ block.call doc.at_css('#release-notes li > a').content
24+ end
25+ end
2026 end
2127 end
lib/docs/scrapers/grunt.rb+4−0
@@ -26,5 +26,9 @@ module Docs · 25 unmodified lines
2626 &copy; GruntJS Team<br>
2727 Licensed under the MIT License.
2828 HTML
29+
30+ def get_latest_version(options, &block)
31+ get_npm_version('grunt-cli', options, &block)
32+ end
2933 end
3034 end
lib/docs/scrapers/handlebars.rb+4−0
@@ -19,5 +19,9 @@ module Docs · 18 unmodified lines
1919 &copy; 2011&ndash;2017 by Yehuda Katz<br>
2020 Licensed under the MIT License.
2121 HTML
22+
23+ def get_latest_version(options, &block)
24+ get_npm_version('handlebars', options, &block)
25+ end
2226 end
2327 end
lib/docs/scrapers/haskell.rb+7−0
@@ -68,5 +68,12 @@ module Docs · 67 unmodified lines
6868
6969 options[:only_patterns] = [/\Alibraries\//]
7070 end
71+
72+ def get_latest_version(options, &block)
73+ fetch_doc('https://downloads.haskell.org/~ghc/latest/docs/html/users_guide/', options) do |doc|
74+ label = doc.at_css('.related > ul > li:last-child').content
75+ block.call label.scan(/([0-9.]+)/)[0][0]
76+ end
77+ end
7178 end
7279 end
lib/docs/scrapers/haxe.rb+7−0
@@ -66,5 +66,12 @@ module Docs · 65 unmodified lines
6666 version 'Python' do
6767 self.base_url = 'https://api.haxe.org/python/'
6868 end
69+
70+ def get_latest_version(options, &block)
71+ fetch_doc('https://api.haxe.org/', options) do |doc|
72+ label = doc.at_css('.container.main-content h1 > small').content
73+ block.call label.sub(/version /, '')
74+ end
75+ end
6976 end
7077 end
lib/docs/scrapers/homebrew.rb+6−0
@@ -19,5 +19,11 @@ module Docs · 18 unmodified lines
1919 &copy; 2009&ndash;present Homebrew contributors<br>
2020 Licensed under the BSD 2-Clause License.
2121 HTML
22+
23+ def get_latest_version(options, &block)
24+ get_latest_github_release('Homebrew', 'brew', options) do |release|
25+ block.call release['name']
26+ end
27+ end
2228 end
2329 end
lib/docs/scrapers/immutable.rb+4−0
@@ -54,5 +54,9 @@ module Docs · 53 unmodified lines
5454 JS
5555 capybara.html
5656 end
57+
58+ def get_latest_version(options, &block)
59+ get_npm_version('immutable', options, &block)
60+ end
5761 end
5862 end
lib/docs/scrapers/influxdata.rb+7−0
@@ -46,5 +46,12 @@ module Docs · 45 unmodified lines
4646 &copy; 2015 InfluxData, Inc.<br>
4747 Licensed under the MIT license.
4848 HTML
49+
50+ def get_latest_version(options, &block)
51+ fetch_doc('https://docs.influxdata.com/influxdb/', options) do |doc|
52+ label = doc.at_css('.navbar--current-product').content.strip
53+ block.call label.scan(/([0-9.]+)/)[0][0]
54+ end
55+ end
4956 end
5057 end
lib/docs/scrapers/jasmine.rb+6−0
@@ -17,5 +17,11 @@ module Docs · 16 unmodified lines
1717 &copy; 2008&ndash;2017 Pivotal Labs<br>
1818 Licensed under the MIT License.
1919 HTML
20+
21+ def get_latest_version(options, &block)
22+ get_latest_github_release('jasmine', 'jasmine', options) do |release|
23+ block.call release['name']
24+ end
25+ end
2026 end
2127 end
lib/docs/scrapers/jekyll.rb+6−0
@@ -28,5 +28,11 @@ module Docs · 27 unmodified lines
2828 &copy; 2008&ndash;2018 Tom Preston-Werner and Jekyll contributors<br>
2929 Licensed under the MIT license.
3030 HTML
31+
32+ def get_latest_version(options, &block)
33+ fetch_doc('https://jekyllrb.com/docs/', options) do |doc|
34+ block.call doc.at_css('.meta a').content[1..-1]
35+ end
36+ end
3137 end
3238 end
lib/docs/scrapers/jest.rb+6−0
@@ -17,5 +17,11 @@ module Docs · 16 unmodified lines
1717 &copy; 2014&ndash;present Facebook Inc.<br>
1818 Licensed under the BSD License.
1919 HTML
20+
21+ def get_latest_version(options, &block)
22+ fetch_doc('https://jestjs.io/docs/en/getting-started', options) do |doc|
23+ block.call doc.at_css('header > a > h3').content
24+ end
25+ end
2026 end
2127 end
lib/docs/scrapers/jquery/jquery_core.rb+4−0
@@ -22,5 +22,9 @@ module Docs · 21 unmodified lines
2222 /Selectors\/odd/i,
2323 /index/i
2424 ]
25+
26+ def get_latest_version(options, &block)
27+ get_npm_version('jquery', options, &block)
28+ end
2529 end
2630 end
lib/docs/scrapers/jquery/jquery_mobile.rb+7−0
@@ -16,5 +16,12 @@ module Docs · 15 unmodified lines
1616 options[:fix_urls] = ->(url) do
1717 url.sub! 'http://api.jquerymobile.com/', 'https://api.jquerymobile.com/'
1818 end
19+
20+ def get_latest_version(options, &block)
21+ fetch_doc('https://jquerymobile.com/', options) do |doc|
22+ label = doc.at_css('.download-box > .download-option:last-child > span').content
23+ block.call label.sub(/Version /, '')
24+ end
25+ end
1926 end
2027 end
lib/docs/scrapers/jquery/jquery_ui.rb+4−0
@@ -15,5 +15,9 @@ module Docs · 14 unmodified lines
1515 options[:fix_urls] = ->(url) do
1616 url.sub! 'http://api.jqueryui.com/', 'https://api.jqueryui.com/'
1717 end
18+
19+ def get_latest_version(options, &block)
20+ get_npm_version('jquery-ui', options, &block)
21+ end
1822 end
1923 end
lib/docs/scrapers/jsdoc.rb+6−0
@@ -21,5 +21,11 @@ module Docs · 20 unmodified lines
2121 &copy; 2011&ndash;2017 the contributors to the JSDoc 3 documentation project<br>
2222 Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0.
2323 HTML
24+
25+ def get_latest_version(options, &block)
26+ get_latest_github_release('jsdoc3', 'jsdoc', options) do |release|
27+ block.call release['tag_name']
28+ end
29+ end
2430 end
2531 end
lib/docs/scrapers/julia.rb+6−0
@@ -49,5 +49,11 @@ module Docs · 48 unmodified lines
4949
5050 html_filters.push 'julia/entries_sphinx', 'julia/clean_html_sphinx', 'sphinx/clean_html'
5151 end
52+
53+ def get_latest_version(options, &block)
54+ get_latest_github_release('JuliaLang', 'julia', options) do |release|
55+ block.call release['tag_name'][1..-1]
56+ end
57+ end
5258 end
5359 end
lib/docs/scrapers/knockout.rb+6−0
@@ -33,5 +33,11 @@ module Docs · 32 unmodified lines
3333 &copy; Steven Sanderson, the Knockout.js team, and other contributors<br>
3434 Licensed under the MIT License.
3535 HTML
36+
37+ def get_latest_version(options, &block)
38+ get_latest_github_release('knockout', 'knockout', options) do |release|
39+ block.call release['tag_name'][1..-1]
40+ end
41+ end
3642 end
3743 end
lib/docs/scrapers/koa.rb+4−0
@@ -34,5 +34,9 @@ module Docs · 33 unmodified lines
3434 &copy; 2018 Koa contributors<br>
3535 Licensed under the MIT License.
3636 HTML
37+
38+ def get_latest_version(options, &block)
39+ get_npm_version('koa', options, &block)
40+ end
3741 end
3842 end
lib/docs/scrapers/kotlin.rb+6−0
@@ -28,5 +28,11 @@ module Docs · 27 unmodified lines
2828 &copy; 2010&ndash;2018 JetBrains s.r.o.<br>
2929 Licensed under the Apache License, Version 2.0.
3030 HTML
31+
32+ def get_latest_version(options, &block)
33+ get_latest_github_release('JetBrains', 'kotlin', options) do |release|
34+ block.call release['tag_name'][1..-1]
35+ end
36+ end
3137 end
3238 end
lib/docs/scrapers/laravel.rb+6−0
@@ -133,5 +133,11 @@ module Docs · 132 unmodified lines
133133 url
134134 end
135135 end
136+
137+ def get_latest_version(options, &block)
138+ get_latest_github_release('laravel', 'laravel', options) do |release|
139+ block.call release['tag_name'][1..-1]
140+ end
141+ end
136142 end
137143 end
lib/docs/scrapers/leaflet.rb+6−0
@@ -39,5 +39,11 @@ module Docs · 38 unmodified lines
3939 self.base_url = "https://leafletjs.com/reference-#{release}.html"
4040 end
4141
42+ def get_latest_version(options, &block)
43+ fetch_doc('https://leafletjs.com/index.html', options) do |doc|
44+ link = doc.css('ul > li > a').to_a.select {|node| node.content == 'Docs'}.first
45+ block.call link['href'].scan(/reference-([0-9.]+)\.html/)[0][0]
46+ end
47+ end
4248 end
4349 end
lib/docs/scrapers/less.rb+7−0
@@ -21,5 +21,12 @@ module Docs · 20 unmodified lines
2121 &copy; 2009&ndash;2016 The Core Less Team<br>
2222 Licensed under the Creative Commons Attribution License 3.0.
2323 HTML
24+
25+ def get_latest_version(options, &block)
26+ fetch_doc('http://lesscss.org/features/', options) do |doc|
27+ label = doc.at_css('.footer-links > li').content
28+ block.call label.scan(/([0-9.]+)/)[0][0]
29+ end
30+ end
2431 end
2532 end
lib/docs/scrapers/liquid.rb+6−0
@@ -19,5 +19,11 @@ module Docs · 18 unmodified lines
1919 &copy; 2005, 2006 Tobias Luetke<br>
2020 Licensed under the MIT License.
2121 HTML
22+
23+ def get_latest_version(options, &block)
24+ get_github_tags('Shopify', 'liquid', options) do |tags|
25+ block.call tags[0]['name'][1..-1]
26+ end
27+ end
2228 end
2329 end
lib/docs/scrapers/lodash.rb+6−0
@@ -32,5 +32,11 @@ module Docs · 31 unmodified lines
3232 self.release = '2.4.2'
3333 self.base_url = "https://lodash.com/docs/#{release}"
3434 end
35+
36+ def get_latest_version(options, &block)
37+ fetch_doc('https://lodash.com/docs/', options) do |doc|
38+ block.call doc.at_css('#version > option[selected]').content
39+ end
40+ end
3541 end
3642 end
lib/docs/scrapers/love.rb+6−0
@@ -39,5 +39,11 @@ module Docs · 38 unmodified lines
3939 &copy; 2006&ndash;2016 L&Ouml;VE Development Team<br>
4040 Licensed under the GNU Free Documentation License, Version 1.3.
4141 HTML
42+
43+ def get_latest_version(options, &block)
44+ fetch_doc('https://love2d.org/wiki/Version_History', options) do |doc|
45+ block.call doc.at_css('#mw-content-text table a').content
46+ end
47+ end
4248 end
4349 end
lib/docs/scrapers/lua.rb+6−0
@@ -26,5 +26,11 @@ module Docs · 25 unmodified lines
2626 self.release = '5.1.5'
2727 self.base_url = 'https://www.lua.org/manual/5.1/'
2828 end
29+
30+ def get_latest_version(options, &block)
31+ fetch_doc('https://www.lua.org/manual/', options) do |doc|
32+ block.call doc.at_css('p.menubar > a').content
33+ end
34+ end
2935 end
3036 end
lib/docs/scrapers/marionette.rb+4−0
@@ -38,5 +38,9 @@ module Docs · 37 unmodified lines
3838
3939 html_filters.push 'marionette/entries_v2'
4040 end
41+
42+ def get_latest_version(options, &block)
43+ get_npm_version('backbone.marionette', options, &block)
44+ end
4145 end
4246 end
lib/docs/scrapers/matplotlib.rb+6−0
@@ -64,5 +64,11 @@ module Docs · 63 unmodified lines
6464 "https://matplotlib.org/#{release}/mpl_toolkits/axes_grid/api/"
6565 ]
6666 end
67+
68+ def get_latest_version(options, &block)
69+ get_latest_github_release('matplotlib', 'matplotlib', options) do |release|
70+ block.call release['tag_name'][1..-1]
71+ end
72+ end
6773 end
6874 end
lib/docs/scrapers/meteor.rb+6−0
@@ -45,5 +45,11 @@ module Docs · 44 unmodified lines
4545 self.base_urls = ['https://guide.meteor.com/v1.3/', "https://docs.meteor.com/v#{self.release}/"]
4646 options[:fix_urls] = nil
4747 end
48+
49+ def get_latest_version(options, &block)
50+ fetch_doc('https://docs.meteor.com/#/full/', options) do |doc|
51+ block.call doc.at_css('select.version-select > option').content
52+ end
53+ end
4854 end
4955 end
lib/docs/scrapers/mocha.rb+4−0
@@ -18,5 +18,9 @@ module Docs · 17 unmodified lines
1818 &copy; 2011&ndash;2018 JS Foundation and contributors<br>
1919 Licensed under the Creative Commons Attribution 4.0 International License.
2020 HTML
21+
22+ def get_latest_version(options, &block)
23+ get_npm_version('mocha', options, &block)
24+ end
2125 end
2226 end
lib/docs/scrapers/modernizr.rb+4−0
@@ -15,5 +15,9 @@ module Docs · 14 unmodified lines
1515 &copy; 2009&ndash;2017 The Modernizr team<br>
1616 Licensed under the MIT License.
1717 HTML
18+
19+ def get_latest_version(options, &block)
20+ get_npm_version('modernizr', options, &block)
21+ end
1822 end
1923 end
lib/docs/scrapers/moment.rb+6−0
@@ -22,5 +22,11 @@ module Docs · 21 unmodified lines
2222 &copy; JS Foundation and other contributors<br>
2323 Licensed under the MIT License.
2424 HTML
25+
26+ def get_latest_version(options, &block)
27+ fetch_doc('http://momentjs.com/', options) do |doc|
28+ block.call doc.at_css('.hero-title > h1 > span').content
29+ end
30+ end
2531 end
2632 end
lib/docs/scrapers/mongoose.rb+7−0
@@ -26,5 +26,12 @@ module Docs · 25 unmodified lines
2626 &copy; 2010 LearnBoost<br>
2727 Licensed under the MIT License.
2828 HTML
29+
30+ def get_latest_version(options, &block)
31+ fetch_doc('https://mongoosejs.com/docs/', options) do |doc|
32+ label = doc.at_css('.pure-menu-link').content.strip
33+ block.call label.sub(/Version /, '')
34+ end
35+ end
2936 end
3037 end
lib/docs/scrapers/rdoc/minitest.rb+6−0
@@ -21,5 +21,11 @@ module Docs · 20 unmodified lines
2121 &copy; Ryan Davis, seattle.rb<br>
2222 Licensed under the MIT License.
2323 HTML
24+
25+ def get_latest_version(options, &block)
26+ get_github_file_contents('seattlerb', 'minitest', 'History.rdoc', options) do |contents|
27+ block.call contents.scan(/([0-9.]+)/)[0][0]
28+ end
29+ end
2430 end
2531 end
lib/docs/scrapers/rdoc/rails.rb+6−0
@@ -93,5 +93,11 @@ module Docs · 92 unmodified lines
9393 version '4.1' do
9494 self.release = '4.1.16'
9595 end
96+
97+ def get_latest_version(options, &block)
98+ get_latest_github_release('rails', 'rails', options) do |release|
99+ block.call release['name']
100+ end
101+ end
96102 end
97103 end
lib/docs/scrapers/rdoc/ruby.rb+12−0
@@ -84,5 +84,17 @@ module Docs · 83 unmodified lines
8484 version '2.2' do
8585 self.release = '2.2.10'
8686 end
87+
88+ def get_latest_version(options, &block)
89+ get_github_tags('ruby', 'ruby', options) do |tags|
90+ tags.each do |tag|
91+ version = tag['name'].gsub(/_/, '.')[1..-1]
92+ if !/^([0-9.]+)$/.match(version).nil? && version.count('.') == 2
93+ block.call version
94+ break
95+ end
96+ end
97+ end
98+ end
8799 end
88100 end
lib/tasks/updates.thor+254−32
@@ -1,4 +1,12 @@
11 class UpdatesCLI < Thor
2+ # The GitHub user that is allowed to upload reports
3+ # TODO: Update this before creating a PR
4+ UPLOAD_USER = 'jmerle'
5+
6+ # The repository to create an issue in when uploading the results
7+ # TODO: Update this before creating a PR
8+ UPLOAD_REPO = 'jmerle/devdocs'
9+
210 def self.to_s
311 'Updates'
412 end
@@ -6,10 +14,14 @@ class UpdatesCLI < Thor · 1 unmodified line
614 def initialize(*args)
715 require 'docs'
816 require 'progress_bar'
17+ require 'terminal-table'
18+ require 'date'
919 super
1020 end
1121
12− desc 'check [--verbose] [doc]...', 'Check for outdated documentations'
22+ desc 'check [--github-token] [--upload] [--verbose] [doc]...', 'Check for outdated documentations'
23+ option :github_token, :type => :string
24+ option :upload, :type => :boolean
1325 option :verbose, :type => :boolean
1426 def check(*names)
1527 # Convert names to a list of Scraper instances
@@ -19,23 +31,26 @@ class UpdatesCLI < Thor · 3 unmodified lines
1931 # Check all documentations for updates when no arguments are given
2032 docs = Docs.all if docs.empty?
2133
22− progress_bar = ::ProgressBar.new docs.length
23− progress_bar.write
34+ opts = {
35+ logger: logger
36+ }
2437
25− results = docs.map do |doc|
26− result = check_doc(doc)
27− progress_bar.increment!
28− result
38+ if options.key?(:github_token)
39+ opts[:github_token] = options[:github_token]
2940 end
3041
31− valid_results = results.select {|result| result.is_a?(Hash)}
42+ with_progress_bar do |bar|
43+ bar.max = docs.length
44+ bar.write
45+ end
3246
33− up_to_date_results = valid_results.select {|result| !result[:is_outdated]}
34− outdated_results = valid_results.select {|result| result[:is_outdated]}
47+ results = docs.map do |doc|
48+ result = check_doc(doc, opts)
49+ with_progress_bar(&:increment!)
50+ result
51+ end
3552
36− log_results('Up-to-date', up_to_date_results) if options[:verbose] and !up_to_date_results.empty?
37− logger.info("") if options[:verbose] and !up_to_date_results.empty? and !outdated_results.empty?
38− log_results('Outdated', outdated_results) unless outdated_results.empty?
53+ process_results(results)
3954 rescue Docs::DocNotFound => error
4055 logger.error(error)
4156 logger.info('Run "thor docs:list" to see the list of docs.')
@@ -43,53 +58,260 @@ class UpdatesCLI < Thor · 1 unmodified line
4358
4459 private
4560
46− def check_doc(doc)
61+ def check_doc(doc, options)
4762 # Newer scraper versions always come before older scraper versions
48− # Therefore, the first item's release value is the latest current scraper version
63+ # Therefore, the first item's release value is the latest scraper version
4964 #
5065 # For example, a scraper could scrape 3 versions: 10, 11 and 12
5166 # doc.versions.first would be the scraper for version 12
5267 instance = doc.versions.first.new
5368
54− return nil unless instance.class.method_defined?(:options)
55−
56− current_version = instance.options[:release]
57− return nil if current_version.nil?
69+ scraper_version = instance.class.method_defined?(:options) ? instance.options[:release] : nil
70+ return error_result(doc, '`options[:release]` does not exist') if scraper_version.nil?
5871
5972 logger.debug("Checking #{doc.name}")
6073
61− instance.get_latest_version do |latest_version|
74+ instance.get_latest_version(options) do |latest_version|
6275 return {
6376 name: doc.name,
64− current_version: current_version,
77+ scraper_version: scraper_version,
6578 latest_version: latest_version,
66− is_outdated: instance.is_outdated(current_version, latest_version)
79+ is_outdated: instance.is_outdated(scraper_version, latest_version)
6780 }
6881 end
69−
70− return nil
7182 rescue NotImplementedError
7283 logger.warn("Couldn't check #{doc.name}, get_latest_version is not implemented")
84+ error_result(doc, '`get_latest_version` is not implemented')
7385 rescue
7486 logger.error("Error while checking #{doc.name}")
7587 raise
7688 end
7789
78− def log_results(label, results)
79− logger.info("#{label} documentations (#{results.length}):")
90+ def error_result(doc, reason)
91+ {
92+ name: doc.name,
93+ error: reason
94+ }
95+ end
96+
97+ def process_results(results)
98+ successful_results = results.select {|result| result.key?(:is_outdated)}
99+ failed_results = results.select {|result| result.key?(:error)}
100+
101+ up_to_date_results = successful_results.select {|result| !result[:is_outdated]}
102+ outdated_results = successful_results.select {|result| result[:is_outdated]}
103+
104+ log_results(outdated_results, up_to_date_results, failed_results)
105+ upload_results(outdated_results, up_to_date_results, failed_results) if options[:upload]
106+ end
107+
108+ #
109+ # Result logging methods
110+ #
111+
112+ def log_results(outdated_results, up_to_date_results, failed_results)
113+ log_failed_results(failed_results) unless failed_results.empty?
114+ log_successful_results('Up-to-date', up_to_date_results) unless up_to_date_results.empty?
115+ log_successful_results('Outdated', outdated_results) unless outdated_results.empty?
116+ end
117+
118+ def log_successful_results(label, results)
119+ title = "#{label} documentations (#{results.length})"
120+ headings = ['Documentation', 'Scraper version', 'Latest version']
121+ rows = results.map {|result| [result[:name], result[:scraper_version], result[:latest_version]]}
122+
123+ table = Terminal::Table.new :title => title, :headings => headings, :rows => rows
124+ puts table
125+ end
126+
127+ def log_failed_results(results)
128+ title = "Documentations that could not be checked (#{results.length})"
129+ headings = %w(Documentation Reason)
130+ rows = results.map {|result| [result[:name], result[:error]]}
131+
132+ table = Terminal::Table.new :title => title, :headings => headings, :rows => rows
133+ puts table
134+ end
135+
136+ #
137+ # Upload methods
138+ #
139+
140+ def upload_results(outdated_results, up_to_date_results, failed_results)
141+ # We can't create issues without a GitHub token
142+ unless options.key?(:github_token)
143+ logger.error('Please specify a GitHub token with the public_repo permission for devdocs-bot with the --github-token parameter')
144+ return
145+ end
146+
147+ logger.info('Uploading the results to a new GitHub issue')
148+
149+ logger.info('Checking if the GitHub token belongs to the correct user')
150+ github_get('/user') do |user|
151+ # Only allow the DevDocs bot to upload reports
152+ if user['login'] == UPLOAD_USER
153+ issue = results_to_issue(outdated_results, up_to_date_results, failed_results)
154+
155+ logger.info('Creating a new GitHub issue')
156+ github_post("/repos/#{UPLOAD_REPO}/issues", issue) do |created_issue|
157+ search_params = {
158+ q: "Documentation versions report in:title author:#{UPLOAD_USER} is:issue repo:#{UPLOAD_REPO}",
159+ sort: 'created',
160+ order: 'desc'
161+ }
162+
163+ logger.info('Checking if the previous issue is still open')
164+ github_get('/search/issues', search_params) do |matching_issues|
165+ previous_issue = matching_issues['items'].find {|item| item['number'] != created_issue['number']}
166+
167+ if previous_issue.nil?
168+ logger.info('No previous issue found')
169+ log_upload_success(created_issue)
170+ else
171+ comment = "This report was superseded by ##{created_issue['number']}."
172+
173+ logger.info('Commenting on the previous issue')
174+ github_post("/repos/#{UPLOAD_REPO}/issues/#{previous_issue['number']}/comments", {body: comment}) do |_|
175+ if previous_issue['closed_at'].nil?
176+ logger.info('Closing the previous issue')
177+ github_patch("/repos/#{UPLOAD_REPO}/issues/#{previous_issue['number']}", {state: 'closed'}) do |_|
178+ log_upload_success(created_issue)
179+ end
180+ else
181+ logger.info('The previous issue has already been closed')
182+ log_upload_success(created_issue)
183+ end
184+ end
185+ end
186+ end
187+ end
188+ else
189+ logger.error("Only #{UPLOAD_USER} is supposed to upload the results to a new issue. The specified github token is not for #{UPLOAD_USER}.")
190+ end
191+ end
192+ end
193+
194+ def results_to_issue(outdated_results, up_to_date_results, failed_results)
195+ results = [
196+ successful_results_to_markdown('Outdated', outdated_results),
197+ successful_results_to_markdown('Up-to-date', up_to_date_results),
198+ failed_results_to_markdown(failed_results)
199+ ]
200+
201+ results_str = results.select {|result| !result.nil?}.join("\n\n")
202+
203+ title = "Documentation versions report for #{Date.today.strftime('%B')} 2019"
204+ body = <<-MARKDOWN
205+## What is this?
206+
207+This is an automatically created issue which contains information about the version status of the documentations available on DevDocs. The results of this report can be used by maintainers when updating outdated documentations.
208+
209+Maintainers can close this issue when all documentations are up-to-date. This issue is automatically closed when the next report is created.
210+
211+## Results
212+
213+The #{outdated_results.length + up_to_date_results.length + failed_results.length} documentations are divided as follows:
214+- #{outdated_results.length} that #{outdated_results.length == 1 ? 'is' : 'are'} outdated
215+- #{up_to_date_results.length} that #{up_to_date_results.length == 1 ? 'is' : 'are'} up-to-date (patch updates are ignored)
216+- #{failed_results.length} that could not be checked
217+ MARKDOWN
218+
219+ {
220+ title: title,
221+ body: body.strip + "\n\n" + results_str
222+ }
223+ end
224+
225+ def successful_results_to_markdown(label, results)
226+ return nil if results.empty?
227+
228+ title = "#{label} documentations (#{results.length})"
229+ headings = ['Documentation', 'Scraper version', 'Latest version']
230+ rows = results.map {|result| [result[:name], result[:scraper_version], result[:latest_version]]}
231+
232+ results_to_markdown(title, headings, rows)
233+ end
234+
235+ def failed_results_to_markdown(results)
236+ return nil if results.empty?
237+
238+ title = "Documentations that could not be checked (#{results.length})"
239+ headings = %w(Documentation Reason)
240+ rows = results.map {|result| [result[:name], result[:error]]}
241+
242+ results_to_markdown(title, headings, rows)
243+ end
244+
245+ def results_to_markdown(title, headings, rows)
246+ "<details>\n<summary>#{title}</summary>\n\n#{create_markdown_table(headings, rows)}\n</details>"
247+ end
248+
249+ def create_markdown_table(headings, rows)
250+ header = headings.join(' | ')
251+ separator = '-|' * headings.length
252+ body = rows.map {|row| row.join(' | ')}
253+
254+ header + "\n" + separator[0...-1] + "\n" + body.join("\n")
255+ end
256+
257+ def log_upload_success(created_issue)
258+ logger.info("Successfully uploaded the results to #{created_issue['html_url']}")
259+ end
260+
261+ #
262+ # HTTP utilities
263+ #
264+
265+ def github_get(endpoint, params = {}, &block)
266+ github_request(endpoint, {method: :get, params: params}, &block)
267+ end
268+
269+ def github_post(endpoint, params, &block)
270+ github_request(endpoint, {method: :post, body: params.to_json}, &block)
271+ end
272+
273+ def github_patch(endpoint, params, &block)
274+ github_request(endpoint, {method: :patch, body: params.to_json}, &block)
275+ end
80276
81− results.each do |result|
82− logger.info("#{result[:name]}: #{result[:current_version]} -> #{result[:latest_version]}")
277+ def github_request(endpoint, opts, &block)
278+ url = "https://api.github.com#{endpoint}"
279+
280+ # GitHub token authentication
281+ opts[:headers] = {
282+ Authorization: "token #{options[:github_token]}"
283+ }
284+
285+ # GitHub requires the Content-Type to be application/json when a body is passed
286+ if opts.key?(:body)
287+ opts[:headers]['Content-Type'] = 'application/json'
83288 end
289+
290+ logger.debug("Making a #{opts[:method]} request to #{url}")
291+
292+ Docs::Request.run(url, opts) do |response|
293+ # response.success? is false if the response code is 201
294+ # GitHub returns 201 Created after an issue is created
295+ if response.success? || response.code == 201
296+ block.call JSON.parse(response.body)
297+ else
298+ logger.error("Couldn't make a #{opts[:method]} request to #{url} (response code #{response.code})")
299+ block.call nil
300+ end
301+ end
302+ end
303+
304+ # A utility method which ensures no progress bar is shown when stdout is not a tty
305+ def with_progress_bar(&block)
306+ return unless $stdout.tty?
307+ @progress_bar ||= ::ProgressBar.new
308+ block.call @progress_bar
84309 end
85310
86311 def logger
87312 @logger ||= Logger.new($stdout).tap do |logger|
88313 logger.level = options[:verbose] ? Logger::DEBUG : Logger::INFO
89− logger.formatter = proc do |severity, datetime, progname, msg|
90− prefix = severity != "INFO" ? "[#{severity}] " : ""
91− "#{prefix}#{msg}\n"
92− end
314+ logger.formatter = proc {|severity, datetime, progname, msg| "[#{severity}] #{msg}\n"}
93315 end
94316 end
95317 end