History

Merge branch 'master' into docker-1803

Jasper van Merle committed 7 years ago

Commit 53425622 parents 845f612 5f5ac5b231 files · +2044 −533Browse files

Showing the changes to lib (158 of 231 files). Show all

Unified · Split

158 files
  1. lib/app.rb +42 −24
  2. lib/docs/core/doc.rb +99 −1
  3. lib/docs/core/scrapers/file_scraper.rb +2 −2
  4. lib/docs/filters/elixir/clean_html.rb +5 −0
  5. lib/docs/filters/homebrew/clean_html.rb +5 −0
  6. lib/docs/filters/homebrew/entries.rb +3 −1
  7. lib/docs/filters/qt/clean_html.rb +1 −0
  8. lib/docs/filters/qt/entries.rb +1 −0
  9. lib/docs/scrapers/angular.rb +4 −0
  10. lib/docs/scrapers/angularjs.rb +4 −0
  11. lib/docs/scrapers/ansible.rb +5 −0
  12. lib/docs/scrapers/apache.rb +5 −0
  13. lib/docs/scrapers/apache_pig.rb +5 −0
  14. lib/docs/scrapers/async.rb +5 −0
  15. lib/docs/scrapers/babel.rb +5 −0
  16. lib/docs/scrapers/backbone.rb +5 −0
  17. lib/docs/scrapers/bash.rb +6 −1
  18. lib/docs/scrapers/bluebird.rb +4 −0
  19. lib/docs/scrapers/bootstrap.rb +5 −0
  20. lib/docs/scrapers/bottle.rb +6 −0
  21. lib/docs/scrapers/bower.rb +4 −0
  22. lib/docs/scrapers/c.rb +7 −0
  23. lib/docs/scrapers/cakephp.rb +5 −0
  24. lib/docs/scrapers/chai.rb +4 −0
  25. lib/docs/scrapers/chef.rb +5 −0
  26. lib/docs/scrapers/clojure.rb +14 −4
  27. lib/docs/scrapers/cmake.rb +6 −0
  28. lib/docs/scrapers/codeception.rb +5 −0
  29. lib/docs/scrapers/codeceptjs.rb +4 −0
  30. lib/docs/scrapers/codeigniter.rb +6 −0
  31. lib/docs/scrapers/coffeescript.rb +4 −0
  32. lib/docs/scrapers/cordova.rb +10 −0
  33. lib/docs/scrapers/cpp.rb +8 −0
  34. lib/docs/scrapers/crystal.rb +5 −0
  35. lib/docs/scrapers/d.rb +5 −0
  36. lib/docs/scrapers/d3.rb +4 −0
  37. lib/docs/scrapers/dart.rb +6 −0
  38. lib/docs/scrapers/django.rb +5 −0
  39. lib/docs/scrapers/docker.rb +6 −0
  40. lib/docs/scrapers/dojo.rb +5 −0
  41. lib/docs/scrapers/drupal.rb +5 −0
  42. lib/docs/scrapers/electron.rb +5 −0
  43. lib/docs/scrapers/elixir.rb +33 −2
  44. lib/docs/scrapers/ember.rb +5 −0
  45. lib/docs/scrapers/erlang.rb +5 −0
  46. lib/docs/scrapers/eslint.rb +4 −0
  47. lib/docs/scrapers/express.rb +4 −0
  48. lib/docs/scrapers/falcon.rb +5 −0
  49. lib/docs/scrapers/fish.rb +5 −0
  50. lib/docs/scrapers/flow.rb +4 −0
  51. lib/docs/scrapers/git.rb +5 −0
  52. lib/docs/scrapers/gnu/gcc.rb +6 −0
  53. lib/docs/scrapers/gnu/gnu_fortran.rb +6 −0
  54. lib/docs/scrapers/go.rb +5 −0
  55. lib/docs/scrapers/godot.rb +5 −0
  56. lib/docs/scrapers/graphite.rb +5 −0
  57. lib/docs/scrapers/grunt.rb +4 −0
  58. lib/docs/scrapers/handlebars.rb +4 −0
  59. lib/docs/scrapers/haskell.rb +8 −1
  60. lib/docs/scrapers/haxe.rb +6 −0
  61. lib/docs/scrapers/homebrew.rb +5 −1
  62. lib/docs/scrapers/http.rb +2 −0
  63. lib/docs/scrapers/immutable.rb +4 −0
  64. lib/docs/scrapers/influxdata.rb +6 −0
  65. lib/docs/scrapers/jasmine.rb +4 −0
  66. lib/docs/scrapers/jekyll.rb +5 −0
  67. lib/docs/scrapers/jest.rb +5 −0
  68. lib/docs/scrapers/jquery/jquery_core.rb +4 −0
  69. lib/docs/scrapers/jquery/jquery_mobile.rb +5 −0
  70. lib/docs/scrapers/jquery/jquery_ui.rb +4 −0
  71. lib/docs/scrapers/jsdoc.rb +4 −0
  72. lib/docs/scrapers/julia.rb +4 −0
  73. lib/docs/scrapers/knockout.rb +4 −0
  74. lib/docs/scrapers/koa.rb +4 −0
  75. lib/docs/scrapers/kotlin.rb +4 −0
  76. lib/docs/scrapers/laravel.rb +4 −0
  77. lib/docs/scrapers/leaflet.rb +5 −0
  78. lib/docs/scrapers/less.rb +6 −0
  79. lib/docs/scrapers/liquid.rb +5 −0
  80. lib/docs/scrapers/lodash.rb +5 −0
  81. lib/docs/scrapers/love.rb +5 −0
  82. lib/docs/scrapers/lua.rb +5 −0
  83. lib/docs/scrapers/marionette.rb +4 −0
  84. lib/docs/scrapers/markdown.rb +4 −0
  85. lib/docs/scrapers/matplotlib.rb +4 −0
  86. lib/docs/scrapers/mdn/css.rb +2 −0
  87. lib/docs/scrapers/mdn/dom.rb +2 −0
  88. lib/docs/scrapers/mdn/dom_events.rb +2 −0
  89. lib/docs/scrapers/mdn/html.rb +2 −0
  90. lib/docs/scrapers/mdn/javascript.rb +2 −0
  91. lib/docs/scrapers/mdn/mdn.rb +5 −0
  92. lib/docs/scrapers/mdn/svg.rb +2 −0
  93. lib/docs/scrapers/mdn/xslt_xpath.rb +2 −0
  94. lib/docs/scrapers/meteor.rb +5 −0
  95. lib/docs/scrapers/mocha.rb +4 −0
  96. lib/docs/scrapers/modernizr.rb +4 −0
  97. lib/docs/scrapers/moment.rb +5 −0
  98. lib/docs/scrapers/mongoose.rb +6 −0
  99. lib/docs/scrapers/nginx.rb +6 −0
  100. lib/docs/scrapers/nginx_lua_module.rb +9 −0
  101. lib/docs/scrapers/nim.rb +5 −0
  102. lib/docs/scrapers/node.rb +5 −0
  103. lib/docs/scrapers/nokogiri2.rb +4 −0
  104. lib/docs/scrapers/npm.rb +4 −0
  105. lib/docs/scrapers/numpy.rb +4 −0
  106. lib/docs/scrapers/openjdk.rb +20 −0
  107. lib/docs/scrapers/opentsdb.rb +4 −0
  108. lib/docs/scrapers/padrino.rb +4 −0
  109. lib/docs/scrapers/pandas.rb +6 −0
  110. lib/docs/scrapers/perl.rb +6 −0
  111. lib/docs/scrapers/phalcon.rb +5 −0
  112. lib/docs/scrapers/phaser.rb +4 −0
  113. lib/docs/scrapers/phoenix.rb +5 −0
  114. lib/docs/scrapers/php.rb +6 −0
  115. lib/docs/scrapers/phpunit.rb +6 −0
  116. lib/docs/scrapers/postgresql.rb +6 −0
  117. lib/docs/scrapers/pug.rb +4 −0
  118. lib/docs/scrapers/puppeteer.rb +5 −0
  119. lib/docs/scrapers/pygame.rb +5 −0
  120. lib/docs/scrapers/python.rb +5 −0
  121. lib/docs/scrapers/q.rb +4 −0
  122. lib/docs/scrapers/qt.rb +17 −2
  123. lib/docs/scrapers/ramda.rb +5 −0
  124. lib/docs/scrapers/rdoc/minitest.rb +5 −0
  125. lib/docs/scrapers/rdoc/rails.rb +4 −0
  126. lib/docs/scrapers/rdoc/ruby.rb +11 −0
  127. lib/docs/scrapers/react.rb +5 −0
  128. lib/docs/scrapers/react_native.rb +5 −0
  129. lib/docs/scrapers/redis.rb +6 −0
  130. lib/docs/scrapers/redux.rb +4 −0
  131. lib/docs/scrapers/relay.rb +5 −0
  132. lib/docs/scrapers/requirejs.rb +4 −0
  133. lib/docs/scrapers/rethinkdb.rb +4 −0
  134. lib/docs/scrapers/rust.rb +6 −0
  135. lib/docs/scrapers/sass.rb +4 −0
  136. lib/docs/scrapers/scikit_image.rb +5 −0
  137. lib/docs/scrapers/scikit_learn.rb +4 −0
  138. lib/docs/scrapers/sinon.rb +5 −0
  139. lib/docs/scrapers/socketio.rb +4 −0
  140. lib/docs/scrapers/sqlite.rb +5 −0
  141. lib/docs/scrapers/statsmodels.rb +4 −0
  142. lib/docs/scrapers/support_tables.rb +6 −0
  143. lib/docs/scrapers/symfony.rb +4 −0
  144. lib/docs/scrapers/tcl_tk.rb +5 −0
  145. lib/docs/scrapers/tensorflow.rb +4 −0
  146. lib/docs/scrapers/terraform.rb +5 −0
  147. lib/docs/scrapers/twig.rb +5 −0
  148. lib/docs/scrapers/typescript.rb +4 −0
  149. lib/docs/scrapers/underscore.rb +5 −0
  150. lib/docs/scrapers/vagrant.rb +5 −0
  151. lib/docs/scrapers/vue.rb +4 −0
  152. lib/docs/scrapers/vulkan.rb +5 −0
  153. lib/docs/scrapers/webpack.rb +4 −0
  154. lib/docs/scrapers/yarn.rb +4 −0
  155. lib/docs/scrapers/yii.rb +4 −0
  156. lib/tasks/assets.thor +3 −0
  157. lib/tasks/sprites.thor +213 −0
  158. lib/tasks/updates.thor +315 −0
lib/app.rb+42−24
@@ -37,6 +37,9 @@ class App < Sinatra::Application · 36 unmodified lines
3737
3838 set :csp, false
3939
40+ require 'docs'
41+ Docs.generate_manifest
42+
4043 Dir[docs_path, root.join(assets_prefix, '*/')].each do |path|
4144 sprockets.append_path(path)
4245 end
@@ -50,6 +53,11 @@ class App < Sinatra::Application · 7 unmodified lines
5053 end
5154
5255 configure :test, :development do
56+ require 'thor'
57+ load 'tasks/sprites.thor'
58+
59+ SpritesCLI.new.invoke(:generate)
60+
5361 require 'active_support/per_thread_registry'
5462 require 'active_support/cache'
5563 sprockets.cache = ActiveSupport::Cache.lookup_store :file_store, root.join('tmp', 'cache', 'assets', environment.to_s)
@@ -192,35 +200,45 @@ class App < Sinatra::Application · 136 unmodified lines
192200 request.query_string.empty? ? nil : "?#{request.query_string}"
193201 end
194202
195− def manifest_asset_urls
196− @@manifest_asset_urls ||= [
203+ def service_worker_asset_urls
204+ @@service_worker_asset_urls ||= [
197205 javascript_path('application', asset_host: false),
198206 stylesheet_path('application'),
199− image_path('docs-1.png'),
200− image_path('docs-1@2x.png'),
201− image_path('docs-2.png'),
202− image_path('docs-2@2x.png'),
203− asset_path('docs.js')
204− ]
207+ image_path('sprites/docs.png'),
208+ image_path('sprites/docs@2x.png'),
209+ asset_path('docs.js'),
210+ App.production? ? nil : javascript_path('debug'),
211+ ].compact
205212 end
206213
207− def app_size
208− @app_size ||= memoized_cookies['size'].nil? ? '20rem' : "#{memoized_cookies['size']}px"
209− end
214+ # Returns a cache name for the service worker to use which changes if any of the assets changes
215+ # When a manifest exist, this name is only created once based on the asset manifest because it never changes without a server restart
216+ # If a manifest does not exist, it is created every time this method is called because the assets can change while the server is running
217+ def service_worker_cache_name
218+ if File.exist?(App.assets_manifest_path)
219+ if defined?(@@service_worker_cache_name)
220+ return @@service_worker_cache_name
221+ end
210222
211− def app_layout
212− memoized_cookies['layout']
213− end
223+ digest = Sprockets::Manifest
224+ .new(nil, App.assets_manifest_path)
225+ .files
226+ .values
227+ .map {|file| file["digest"]}
228+ .join
214229
215− def app_theme
216− @app_theme ||= memoized_cookies['dark'].nil? ? 'default' : 'dark'
217− end
230+ return @@service_worker_cache_name ||= Digest::MD5.hexdigest(digest)
231+ else
232+ paths = App.sprockets
233+ .each_file
234+ .to_a
235+ .reject {|file| file.start_with?(App.docs_path)}
218236
219− def dark_theme?
220− app_theme == 'dark'
237+ return App.sprockets.pack_hexdigest(App.sprockets.files_digest(paths))
238+ end
221239 end
222240
223− def redirect_via_js(path) # courtesy of HTML5 App Cache
241+ def redirect_via_js(path)
224242 response.set_cookie :initial_path, value: path, expires: Time.now + 15, path: '/'
225243 redirect '/', 302
226244 end
@@ -243,15 +261,15 @@ class App < Sinatra::Application · 16 unmodified lines
243261 end
244262 end
245263
246− get '/manifest.appcache' do
247− content_type 'text/cache-manifest'
264+ get '/service-worker.js' do
265+ content_type 'application/javascript'
248266 expires 0, :'no-cache'
249− erb :manifest
267+ erb :'service-worker.js'
250268 end
251269
252270 get '/' do
253271 return redirect "/#q=#{params[:q]}" if params[:q]
254− return redirect '/' unless request.query_string.empty? # courtesy of HTML5 App Cache
272+ return redirect '/' unless request.query_string.empty?
255273 response.headers['Content-Security-Policy'] = settings.csp if settings.csp
256274 erb :index
257275 end
lib/docs/core/doc.rb+99−1
@@ -152,7 +152,6 @@ module Docs · 151 unmodified lines
152152 end
153153 end
154154
155−
156155 def initialize
157156 raise NotImplementedError, "#{self.class} is an abstract class and cannot be instantiated." if self.class.abstract
158157 end
@@ -164,5 +163,104 @@ module Docs · 5 unmodified lines
164163 def build_pages(&block)
165164 raise NotImplementedError
166165 end
166+
167+ def get_scraper_version(opts)
168+ if self.class.method_defined?(:options) and !options[:release].nil?
169+ options[:release]
170+ else
171+ # If options[:release] does not exist, we return the Epoch timestamp of when the doc was last modified in DevDocs production
172+ json = fetch_json('https://devdocs.io/docs.json', opts)
173+ items = json.select {|item| item['name'] == self.class.name}
174+ items = items.map {|item| item['mtime']}
175+ items.max
176+ end
177+ end
178+
179+ # Should return the latest version of this documentation
180+ # If options[:release] is defined, it should be in the same format
181+ # If options[:release] is not defined, it should return the Epoch timestamp of when the documentation was last updated
182+ # If the docs will never change, simply return '1.0.0'
183+ def get_latest_version(opts)
184+ raise NotImplementedError
185+ end
186+
187+ # Returns whether or not this scraper is outdated.
188+ #
189+ # The default implementation assumes the documentation uses a semver(-like) approach when it comes to versions.
190+ # Patch updates are ignored because there are usually little to no documentation changes in bug-fix-only releases.
191+ #
192+ # Scrapers of documentations that do not use this versioning approach should override this method.
193+ #
194+ # Examples of the default implementation:
195+ # 1 -> 2 = outdated
196+ # 1.1 -> 1.2 = outdated
197+ # 1.1.1 -> 1.1.2 = not outdated
198+ def is_outdated(scraper_version, latest_version)
199+ scraper_parts = scraper_version.to_s.split(/\./).map(&:to_i)
200+ latest_parts = latest_version.to_s.split(/\./).map(&:to_i)
201+
202+ # Only check the first two parts, the third part is for patch updates
203+ [0, 1].each do |i|
204+ break if i >= scraper_parts.length or i >= latest_parts.length
205+ return true if latest_parts[i] > scraper_parts[i]
206+ return false if latest_parts[i] < scraper_parts[i]
207+ end
208+
209+ false
210+ end
211+
212+ private
213+
214+ #
215+ # Utility methods for get_latest_version
216+ #
217+
218+ def fetch(url, opts)
219+ headers = {}
220+
221+ if opts.key?(:github_token) and url.start_with?('https://api.github.com/')
222+ headers['Authorization'] = "token #{opts[:github_token]}"
223+ end
224+
225+ opts[:logger].debug("Fetching #{url}")
226+ response = Request.run(url, { connecttimeout: 15, headers: headers })
227+
228+ if response.success?
229+ response.body
230+ else
231+ reason = response.timed_out? ? "Timed out while connecting to #{url}" : "Couldn't fetch #{url} (response code #{response.code})"
232+ opts[:logger].error(reason)
233+ raise reason
234+ end
235+ end
236+
237+ def fetch_doc(url, opts)
238+ body = fetch(url, opts)
239+ Nokogiri::HTML.parse(body, nil, 'UTF-8')
240+ end
241+
242+ def fetch_json(url, opts)
243+ JSON.parse fetch(url, opts)
244+ end
245+
246+ def get_npm_version(package, opts)
247+ json = fetch_json("https://registry.npmjs.com/#{package}", opts)
248+ json['dist-tags']['latest']
249+ end
250+
251+ def get_latest_github_release(owner, repo, opts)
252+ release = fetch_json("https://api.github.com/repos/#{owner}/#{repo}/releases/latest", opts)
253+ tag_name = release['tag_name']
254+ tag_name.start_with?('v') ? tag_name[1..-1] : tag_name
255+ end
256+
257+ def get_github_tags(owner, repo, opts)
258+ fetch_json("https://api.github.com/repos/#{owner}/#{repo}/tags", opts)
259+ end
260+
261+ def get_github_file_contents(owner, repo, path, opts)
262+ json = fetch_json("https://api.github.com/repos/#{owner}/#{repo}/contents/#{path}", opts)
263+ Base64.decode64(json['content'])
264+ end
167265 end
168266 end
lib/docs/core/scrapers/file_scraper.rb+2−2
@@ -29,7 +29,7 @@ module Docs · 28 unmodified lines
2929
3030 def request_one(url)
3131 assert_source_directory_exists
32− Response.new read_file(url_to_path(url)), URL.parse(url)
32+ Response.new read_file(File.join(source_directory, url_to_path(url))), URL.parse(url)
3333 end
3434
3535 def request_all(urls)
@@ -50,7 +50,7 @@ module Docs · 14 unmodified lines
5050 end
5151
5252 def read_file(path)
53− File.read(File.join(source_directory, path))
53+ File.read(path)
5454 rescue
5555 instrument 'warn.doc', msg: "Failed to open file: #{path}"
5656 nil
lib/docs/filters/elixir/clean_html.rb+5−0
@@ -57,6 +57,11 @@ module Docs · 56 unmodified lines
5757 node.parent.after(node)
5858 end
5959
60+ css('.signature').each do |node|
61+ non_text_children = node.xpath('node()[not(self::text())]')
62+ non_text_children.to_a.reverse.each { |child| node.parent.add_next_sibling(child) }
63+ end
64+
6065 css('pre').each do |node|
6166 node['data-language'] = 'elixir'
6267 node.content = node.content
lib/docs/filters/homebrew/clean_html.rb+5−0
@@ -4,6 +4,11 @@ module Docs · 3 unmodified lines
44 def call
55 css('hr')
66
7+ if at_css('h1').nil?
8+ title = current_url.normalized_path[1..-1].gsub(/-/, ' ')
9+ doc.children.before("<h1>#{title}</h1>")
10+ end
11+
712 css('div.highlighter-rouge').each do |node|
813 lang = node['class'][/language-(\w+)/, 1]
914 node['data-language'] = lang if lang
lib/docs/filters/homebrew/entries.rb+3−1
@@ -2,7 +2,8 @@ module Docs · 1 unmodified line
22 class Homebrew
33 class EntriesFilter < Docs::EntriesFilter
44 def get_name
5− name = at_css('h1').content.strip
5+ header = at_css('h1')
6+ name = header.nil? ? current_url.normalized_path[1..-1].gsub(/-/, ' ') : header.content.strip
67 name.remove! %r{\(.*}
78 name
89 end
@@ -16,6 +17,7 @@ module Docs · 7 unmodified lines
1617 Python-for-Formula-Authors
1718 Migrating-A-Formula-To-A-Tap
1819 Rename-A-Formula
20+ Building-Against-Non-Homebrew-Dependencies
1921 How-to-Create-and-Maintain-a-Tap
2022 Brew-Test-Bot
2123 Prose-Style-Guidelines)
lib/docs/filters/qt/clean_html.rb+1−0
@@ -8,6 +8,7 @@ module Docs · 7 unmodified lines
88 # QML property/method header
99 css('.qmlproto').each do |node|
1010 id = node.at_css('tr')['id']
11+ id = node.at_css('a')['name'] if id.blank?
1112 node.inner_html = node.at_css('td').inner_html
1213 node.name = 'h3'
1314 node['id'] = id
lib/docs/filters/qt/entries.rb+1−0
@@ -111,6 +111,7 @@ module Docs · 110 unmodified lines
111111 css('.qmlproto').each do |node|
112112 title = node.content.strip
113113 id = node.at_css('tr')['id']
114+ id = node.at_css('a')['name'] if id.blank?
114115
115116 # Remove options
116117 title.remove!(%r{^\[.*\] })
lib/docs/scrapers/angular.rb+4−0
@@ -155,6 +155,10 @@ module Docs · 154 unmodified lines
155155 end
156156 end
157157
158+ def get_latest_version(opts)
159+ get_npm_version('@angular/core', opts)
160+ end
161+
158162 private
159163
160164 def parse(response)
lib/docs/scrapers/angularjs.rb+4−0
@@ -69,5 +69,9 @@ module Docs · 68 unmodified lines
6969 self.release = '1.2.32'
7070 self.base_url = "https://code.angularjs.org/#{release}/docs/partials/"
7171 end
72+
73+ def get_latest_version(opts)
74+ get_npm_version('angular', opts)
75+ end
7276 end
7377 end
lib/docs/scrapers/ansible.rb+5−0
@@ -87,5 +87,10 @@ module Docs · 86 unmodified lines
8787 quickstart.html
8888 list_of_all_modules.html)
8989 end
90+
91+ def get_latest_version(opts)
92+ doc = fetch_doc('https://docs.ansible.com/ansible/latest/index.html', opts)
93+ doc.at_css('.DocSiteProduct-CurrentVersion').content.strip
94+ end
9095 end
9196 end
lib/docs/scrapers/apache.rb+5−0
@@ -33,5 +33,10 @@ module Docs · 32 unmodified lines
3333 &copy; 2018 The Apache Software Foundation<br>
3434 Licensed under the Apache License, Version 2.0.
3535 HTML
36+
37+ def get_latest_version(opts)
38+ doc = fetch_doc('http://httpd.apache.org/docs/', opts)
39+ doc.at_css('#apcontents > ul a')['href'][0...-1]
40+ end
3641 end
3742 end
lib/docs/scrapers/apache_pig.rb+5−0
@@ -43,5 +43,10 @@ module Docs · 42 unmodified lines
4343 self.base_url = "https://pig.apache.org/docs/r#{release}/"
4444 end
4545
46+ def get_latest_version(opts)
47+ doc = fetch_doc('https://pig.apache.org/', opts)
48+ item = doc.at_css('div[id="menu_1.2"] > .menuitem:last-child')
49+ item.content.strip.sub(/Release /, '')
50+ end
4651 end
4752 end
lib/docs/scrapers/async.rb+5−0
@@ -17,5 +17,10 @@ module Docs · 16 unmodified lines
1717 &copy; 2010&ndash;2018 Caolan McMahon<br>
1818 Licensed under the MIT License.
1919 HTML
20+
21+ def get_latest_version(opts)
22+ doc = fetch_doc('https://caolan.github.io/async/v3/', opts)
23+ doc.at_css('#version-dropdown > a').content.strip[1..-1]
24+ end
2025 end
2126 end
lib/docs/scrapers/babel.rb+5−0
@@ -22,5 +22,10 @@ module Docs · 21 unmodified lines
2222 stub '' do
2323 '<div></div>'
2424 end
25+
26+ def get_latest_version(opts)
27+ doc = fetch_doc('https://babeljs.io/docs/en/', opts)
28+ doc.at_css('a[href="/versions"] > h3').content
29+ end
2530 end
2631 end
lib/docs/scrapers/backbone.rb+5−0
@@ -20,5 +20,10 @@ module Docs · 19 unmodified lines
2020 &copy; 2010&ndash;2016 Jeremy Ashkenas, DocumentCloud<br>
2121 Licensed under the MIT License.
2222 HTML
23+
24+ def get_latest_version(opts)
25+ doc = fetch_doc('https://backbonejs.org/', opts)
26+ doc.at_css('.version').content[1...-1]
27+ end
2328 end
2429 end
lib/docs/scrapers/bash.rb+6−1
@@ -1,7 +1,7 @@
11 module Docs
22 class Bash < UrlScraper
33 self.type = 'bash'
4− self.release = '4.4'
4+ self.release = '5.0'
55 self.base_url = 'https://www.gnu.org/software/bash/manual'
66 self.root_path = '/html_node/index.html'
77 self.links = {
@@ -17,5 +17,10 @@ module Docs · 9 unmodified lines
1717 Copyright &copy; 2000, 2001, 2002, 2007, 2008 Free Software Foundation, Inc.<br>
1818 Licensed under the GNU Free Documentation License.
1919 HTML
20+
21+ def get_latest_version(opts)
22+ body = fetch('https://www.gnu.org/software/bash/manual/html_node/index.html', opts)
23+ body.scan(/, Version ([0-9.]+)/)[0][0][0...-1]
24+ end
2025 end
2126 end
lib/docs/scrapers/bluebird.rb+4−0
@@ -18,5 +18,9 @@ module Docs · 17 unmodified lines
1818 &copy; 2013&ndash;2017 Petka Antonov<br>
1919 Licensed under the MIT License.
2020 HTML
21+
22+ def get_latest_version(opts)
23+ get_npm_version('bluebird', opts)
24+ end
2125 end
2226 end
lib/docs/scrapers/bootstrap.rb+5−0
@@ -34,5 +34,10 @@ module Docs · 33 unmodified lines
3434
3535 options[:only] = %w(getting-started/ css/ components/ javascript/)
3636 end
37+
38+ def get_latest_version(opts)
39+ doc = fetch_doc('https://getbootstrap.com/', opts)
40+ doc.at_css('#bd-versions').content.strip[1..-1]
41+ end
3742 end
3843 end
lib/docs/scrapers/bottle.rb+6−0
@@ -27,5 +27,11 @@ module Docs · 26 unmodified lines
2727 self.release = '0.11.7'
2828 self.base_url = "https://bottlepy.org/docs/#{self.version}/"
2929 end
30+
31+ def get_latest_version(opts)
32+ doc = fetch_doc('https://bottlepy.org/docs/stable/', opts)
33+ label = doc.at_css('.sphinxsidebarwrapper > ul > li > b')
34+ label.content.sub(/Bottle /, '')
35+ end
3036 end
3137 end
lib/docs/scrapers/bower.rb+4−0
@@ -19,5 +19,9 @@ module Docs · 18 unmodified lines
1919 &copy; 2018 Bower contributors<br>
2020 Licensed under the MIT License.
2121 HTML
22+
23+ def get_latest_version(opts)
24+ get_npm_version('bower', opts)
25+ end
2226 end
2327 end
lib/docs/scrapers/c.rb+7−0
@@ -26,6 +26,13 @@ module Docs · 25 unmodified lines
2626 Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0.
2727 HTML
2828
29+ def get_latest_version(opts)
30+ doc = fetch_doc('https://en.cppreference.com/w/Cppreference:Archives', opts)
31+ link = doc.at_css('a[title^="File:"]')
32+ date = link.content.scan(/(\d+)\./)[0][0]
33+ DateTime.strptime(date, '%Y%m%d').to_time.to_i
34+ end
35+
2936 private
3037
3138 def file_path_for(*)
lib/docs/scrapers/cakephp.rb+5−0
@@ -71,6 +71,11 @@ module Docs · 70 unmodified lines
7171 self.base_url = 'https://api.cakephp.org/2.7/'
7272 end
7373
74+ def get_latest_version(opts)
75+ doc = fetch_doc('https://api.cakephp.org/3.7/', opts)
76+ doc.at_css('.version-picker .dropdown-toggle').content.strip
77+ end
78+
7479 private
7580
7681 def parse(response)
lib/docs/scrapers/chai.rb+4−0
@@ -23,5 +23,9 @@ module Docs · 22 unmodified lines
2323 &copy; 2016 Chai.js Assertion Library<br>
2424 Licensed under the MIT License.
2525 HTML
26+
27+ def get_latest_version(opts)
28+ get_npm_version('chai', opts)
29+ end
2630 end
2731 end
lib/docs/scrapers/chef.rb+5−0
@@ -47,5 +47,10 @@ module Docs · 46 unmodified lines
4747
4848 options[:only_patterns] = [/\A#{client_path}\//, /\A#{server_path}\//]
4949 end
50+
51+ def get_latest_version(opts)
52+ doc = fetch_doc('https://downloads.chef.io/chef', opts)
53+ doc.at_css('h1.product-heading > span').content.strip
54+ end
5055 end
5156 end
lib/docs/scrapers/clojure.rb+14−4
@@ -13,19 +13,29 @@ module Docs · 12 unmodified lines
1313 Licensed under the Eclipse Public License 1.0.
1414 HTML
1515
16− version '1.9' do
17− self.release = '1.9'
16+ version '1.10' do
17+ self.release = '1.10 (stable)'
1818 self.base_url = 'https://clojure.github.io/clojure/'
1919 end
2020
21+ version '1.9' do
22+ self.release = '1.9 (legacy)'
23+ self.base_url = 'https://clojure.github.io/clojure/branch-clojure-1.9.0/'
24+ end
25+
2126 version '1.8' do
22− self.release = '1.8'
27+ self.release = '1.8 (legacy)'
2328 self.base_url = 'https://clojure.github.io/clojure/branch-clojure-1.8.0/'
2429 end
2530
2631 version '1.7' do
27− self.release = '1.7'
32+ self.release = '1.7 (legacy)'
2833 self.base_url = 'https://clojure.github.io/clojure/branch-clojure-1.7.0/'
2934 end
35+
36+ def get_latest_version(opts)
37+ doc = fetch_doc('http://clojure.github.io/clojure/index.html', opts)
38+ doc.at_css('#header-version').content[1..-1]
39+ end
3040 end
3141 end
lib/docs/scrapers/cmake.rb+6−0
@@ -59,5 +59,11 @@ module Docs · 58 unmodified lines
5959 self.release = '3.5.2'
6060 self.base_url = 'https://cmake.org/cmake/help/v3.5/'
6161 end
62+
63+ def get_latest_version(opts)
64+ doc = fetch_doc('https://cmake.org/documentation/', opts)
65+ link = doc.at_css('.entry-content ul > li > strong > a > big')
66+ link.content.scan(/([0-9.]+)/)[0][0]
67+ end
6268 end
6369 end
lib/docs/scrapers/codeception.rb+5−0
@@ -18,5 +18,10 @@ module Docs · 17 unmodified lines
1818 &copy; 2011 Michael Bodnarchuk and contributors<br>
1919 Licensed under the MIT License.
2020 HTML
21+
22+ def get_latest_version(opts)
23+ doc = fetch_doc('https://codeception.com/changelog', opts)
24+ doc.at_css('#page > h4').content
25+ end
2126 end
2227 end
lib/docs/scrapers/codeceptjs.rb+4−0
@@ -21,5 +21,9 @@ module Docs · 20 unmodified lines
2121 &copy; 2015 DavertMik &lt;davert@codegyre.com&gt; (http://codegyre.com)<br>
2222 Licensed under the MIT License.
2323 HTML
24+
25+ def get_latest_version(opts)
26+ get_npm_version('codeceptjs', opts)
27+ end
2428 end
2529 end
lib/docs/scrapers/codeigniter.rb+6−0
@@ -38,5 +38,11 @@ module Docs · 37 unmodified lines
3838 version '3' do
3939 self.release = '3.1.8'
4040 end
41+
42+ def get_latest_version(opts)
43+ doc = fetch_doc('https://codeigniter.com/user_guide/changelog.html', opts)
44+ header = doc.at_css('#change-log h2')
45+ header.content.scan(/([0-9.]+)/)[0][0]
46+ end
4147 end
4248 end
lib/docs/scrapers/coffeescript.rb+4−0
@@ -30,5 +30,9 @@ module Docs · 29 unmodified lines
3030
3131 options[:container] = '.container'
3232 end
33+
34+ def get_latest_version(opts)
35+ get_npm_version('coffeescript', opts)
36+ end
3337 end
3438 end
lib/docs/scrapers/cordova.rb+10−0
@@ -42,5 +42,15 @@ module Docs · 41 unmodified lines
4242 self.release = '6.5.0'
4343 self.base_url = 'https://cordova.apache.org/docs/en/6.x/'
4444 end
45+
46+ def get_latest_version(opts)
47+ doc = fetch_doc('https://cordova.apache.org/docs/en/latest/', opts)
48+
49+ label = doc.at_css('#versionDropdown').content.strip
50+ version = label.scan(/([0-9.]+)/)[0][0]
51+ version = version[0...-1] if version.end_with?('.')
52+
53+ version
54+ end
4555 end
4656 end
lib/docs/scrapers/cpp.rb+8−0
@@ -34,6 +34,14 @@ module Docs · 33 unmodified lines
3434 Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0.
3535 HTML
3636
37+ # Same as get_latest_version in lib/docs/scrapers/c.rb
38+ def get_latest_version(opts)
39+ doc = fetch_doc('https://en.cppreference.com/w/Cppreference:Archives', opts)
40+ link = doc.at_css('a[title^="File:"]')
41+ date = link.content.scan(/(\d+)\./)[0][0]
42+ DateTime.strptime(date, '%Y%m%d').to_time.to_i
43+ end
44+
3745 private
3846
3947 def file_path_for(*)
lib/docs/scrapers/crystal.rb+5−0
@@ -34,5 +34,10 @@ module Docs · 33 unmodified lines
3434 HTML
3535 end
3636 }
37+
38+ def get_latest_version(opts)
39+ body = fetch('https://crystal-lang.org/api', opts)
40+ body.scan(/Crystal Docs ([0-9.]+)/)[0][0]
41+ end
3742 end
3843 end
lib/docs/scrapers/d.rb+5−0
@@ -26,5 +26,10 @@ module Docs · 25 unmodified lines
2626 def initial_urls
2727 %w(https://dlang.org/phobos/index.html https://dlang.org/spec/intro.html)
2828 end
29+
30+ def get_latest_version(opts)
31+ doc = fetch_doc('https://dlang.org/changelog/', opts)
32+ doc.at_css('#content > ul > li:nth-child(2) > a')['id']
33+ end
2934 end
3035 end
lib/docs/scrapers/d3.rb+4−0
@@ -58,5 +58,9 @@ module Docs · 57 unmodified lines
5858 options[:root_title] = 'D3.js'
5959 options[:only_patterns] = [/\.md\z/]
6060 end
61+
62+ def get_latest_version(opts)
63+ get_npm_version('d3', opts)
64+ end
6165 end
6266 end
lib/docs/scrapers/dart.rb+6−0
@@ -31,5 +31,11 @@ module Docs · 30 unmodified lines
3131 self.release = '1.24.3'
3232 self.base_url = "https://api.dartlang.org/stable/#{release}/"
3333 end
34+
35+ def get_latest_version(opts)
36+ doc = fetch_doc('https://api.dartlang.org/', opts)
37+ label = doc.at_css('footer > span').content.strip
38+ label.sub(/Dart /, '')
39+ end
3440 end
3541 end
lib/docs/scrapers/django.rb+5−0
@@ -63,5 +63,10 @@ module Docs · 62 unmodified lines
6363 self.release = '1.8.18'
6464 self.base_url = 'https://docs.djangoproject.com/en/1.8/'
6565 end
66+
67+ def get_latest_version(opts)
68+ doc = fetch_doc('https://docs.djangoproject.com/', opts)
69+ doc.at_css('#doc-versions > li.current > span > strong').content
70+ end
6671 end
6772 end
lib/docs/scrapers/docker.rb+6−0
@@ -176,5 +176,11 @@ module Docs · 175 unmodified lines
176176 options[:container] = '#docs'
177177 options[:only_patterns] << /\Aswarm\//
178178 end
179+
180+ def get_latest_version(opts)
181+ doc = fetch_doc('https://docs.docker.com/', opts)
182+ label = doc.at_css('.nav-container button.dropdown-toggle').content.strip
183+ label.scan(/([0-9.]+)/)[0][0]
184+ end
179185 end
180186 end
lib/docs/scrapers/dojo.rb+5−0
@@ -36,6 +36,11 @@ module Docs · 35 unmodified lines
3636 urls.map { |url| "<a href='#{url}'>#{url}</a>" }.join
3737 end
3838
39+ def get_latest_version(opts)
40+ doc = fetch_doc('https://dojotoolkit.org/api/', opts)
41+ doc.at_css('#versionSelector > option[selected]').content
42+ end
43+
3944 private
4045
4146 def get_url_list(json, set = Set.new)
lib/docs/scrapers/drupal.rb+5−0
@@ -98,5 +98,10 @@ module Docs · 97 unmodified lines
9898 /\A[\w\-\.]+\.php\/7\.x\z/
9999 ]
100100 end
101+
102+ def get_latest_version(opts)
103+ json = fetch_json('https://packagist.org/packages/drupal/drupal.json', opts)
104+ json['package']['versions'].keys.find {|version| !version.end_with?('-dev')}
105+ end
101106 end
102107 end
lib/docs/scrapers/electron.rb+5−0
@@ -22,5 +22,10 @@ module Docs · 21 unmodified lines
2222 &copy; 2013&ndash;2018 GitHub Inc.<br>
2323 Licensed under the MIT license.
2424 HTML
25+
26+ def get_latest_version(opts)
27+ doc = fetch_doc('https://electronjs.org/docs', opts)
28+ doc.at_css('.docs-version').content
29+ end
2530 end
2631 end
lib/docs/scrapers/elixir.rb+33−2
@@ -33,8 +33,34 @@ module Docs · 32 unmodified lines
3333 "https://elixir-lang.org/getting-started/introduction.html" ]
3434 end
3535
36+ version '1.9' do
37+ self.release = '1.9.1'
38+ self.base_urls = [
39+ "https://hexdocs.pm/elixir/#{release}/",
40+ "https://hexdocs.pm/eex/#{release}/",
41+ "https://hexdocs.pm/ex_unit/#{release}/",
42+ "https://hexdocs.pm/iex/#{release}/",
43+ "https://hexdocs.pm/logger/#{release}/",
44+ "https://hexdocs.pm/mix/#{release}/",
45+ 'https://elixir-lang.org/getting-started/'
46+ ]
47+ end
48+
49+ version '1.8' do
50+ self.release = '1.8.2'
51+ self.base_urls = [
52+ "https://hexdocs.pm/elixir/#{release}/",
53+ "https://hexdocs.pm/eex/#{release}/",
54+ "https://hexdocs.pm/ex_unit/#{release}/",
55+ "https://hexdocs.pm/iex/#{release}/",
56+ "https://hexdocs.pm/logger/#{release}/",
57+ "https://hexdocs.pm/mix/#{release}/",
58+ 'https://elixir-lang.org/getting-started/'
59+ ]
60+ end
61+
3662 version '1.7' do
37− self.release = '1.7.3'
63+ self.release = '1.7.4'
3864 self.base_urls = [
3965 "https://hexdocs.pm/elixir/#{release}/",
4066 "https://hexdocs.pm/eex/#{release}/",
@@ -47,7 +73,7 @@ module Docs · 6 unmodified lines
4773 end
4874
4975 version '1.6' do
50− self.release = '1.6.5'
76+ self.release = '1.6.6'
5177 self.base_urls = [
5278 "https://hexdocs.pm/elixir/#{release}/",
5379 "https://hexdocs.pm/eex/#{release}/",
@@ -97,5 +123,10 @@ module Docs · 43 unmodified lines
97123 'https://elixir-lang.org/getting-started/'
98124 ]
99125 end
126+
127+ def get_latest_version(opts)
128+ doc = fetch_doc('https://hexdocs.pm/elixir/api-reference.html', opts)
129+ doc.at_css('h2.sidebar-projectVersion').content.strip[1..-1]
130+ end
100131 end
101132 end
lib/docs/scrapers/ember.rb+5−0
@@ -56,5 +56,10 @@ module Docs · 55 unmodified lines
5656 https://emberjs.com/api/ember-data/2.14/classes/DS
5757 )
5858 end
59+
60+ def get_latest_version(opts)
61+ doc = fetch_doc('https://emberjs.com/api/ember/release', opts)
62+ doc.at_css('.sidebar > .select-container .ember-power-select-selected-item').content.strip
63+ end
5964 end
6065 end
lib/docs/scrapers/erlang.rb+5−0
@@ -55,5 +55,10 @@ module Docs · 54 unmodified lines
5555 version '18' do
5656 self.release = '18.3'
5757 end
58+
59+ def get_latest_version(opts)
60+ doc = fetch_doc('https://www.erlang.org/downloads', opts)
61+ doc.at_css('.col-lg-3 > ul > li').content.strip.sub(/OTP /, '')
62+ end
5863 end
5964 end
lib/docs/scrapers/eslint.rb+4−0
@@ -20,5 +20,9 @@ module Docs · 19 unmodified lines
2020 &copy; JS Foundation and other contributors<br>
2121 Licensed under the MIT License.
2222 HTML
23+
24+ def get_latest_version(opts)
25+ get_npm_version('eslint', opts)
26+ end
2327 end
2428 end
lib/docs/scrapers/express.rb+4−0
@@ -28,5 +28,9 @@ module Docs · 27 unmodified lines
2828 &copy; 2017 StrongLoop, IBM, and other expressjs.com contributors.<br>
2929 Licensed under the Creative Commons Attribution-ShareAlike License v3.0.
3030 HTML
31+
32+ def get_latest_version(opts)
33+ get_npm_version('express', opts)
34+ end
3135 end
3236 end
lib/docs/scrapers/falcon.rb+5−0
@@ -33,5 +33,10 @@ module Docs · 32 unmodified lines
3333 self.release = '1.2.0'
3434 self.base_url = "https://falcon.readthedocs.io/en/#{self.release}/"
3535 end
36+
37+ def get_latest_version(opts)
38+ doc = fetch_doc('https://falcon.readthedocs.io/en/stable/changes/index.html', opts)
39+ doc.at_css('#changelogs ul > li > a').content
40+ end
3641 end
3742 end
lib/docs/scrapers/fish.rb+5−0
@@ -46,5 +46,10 @@ module Docs · 45 unmodified lines
4646 self.release = '2.2.0'
4747 self.base_url = "https://fishshell.com/docs/#{version}/"
4848 end
49+
50+ def get_latest_version(opts)
51+ doc = fetch_doc('http://fishshell.com/docs/current/index.html', opts)
52+ doc.at_css('#toc-index').content.scan(/([0-9.]+)/)[0][0]
53+ end
4954 end
5055 end
lib/docs/scrapers/flow.rb+4−0
@@ -18,5 +18,9 @@ module Docs · 17 unmodified lines
1818 &copy; 2013&ndash;present Facebook Inc.<br>
1919 Licensed under the MIT License.
2020 HTML
21+
22+ def get_latest_version(opts)
23+ get_npm_version('flow-bin', opts)
24+ end
2125 end
2226 end
lib/docs/scrapers/git.rb+5−0
@@ -19,5 +19,10 @@ module Docs · 18 unmodified lines
1919 &copy; 2005&ndash;2018 Linus Torvalds and others<br>
2020 Licensed under the GNU General Public License version 2.
2121 HTML
22+
23+ def get_latest_version(opts)
24+ doc = fetch_doc('https://git-scm.com/', opts)
25+ doc.at_css('.version').content.strip
26+ end
2227 end
2328 end
lib/docs/scrapers/gnu/gcc.rb+6−0
@@ -99,5 +99,11 @@ module Docs · 98 unmodified lines
9999
100100 options[:replace_paths] = CPP_PATHS
101101 end
102+
103+ def get_latest_version(opts)
104+ doc = fetch_doc('https://gcc.gnu.org/onlinedocs/', opts)
105+ label = doc.at_css('ul > li > ul > li > a').content.strip
106+ label.scan(/([0-9.]+)/)[0][0]
107+ end
102108 end
103109 end
lib/docs/scrapers/gnu/gnu_fortran.rb+6−0
@@ -25,5 +25,11 @@ module Docs · 24 unmodified lines
2525 self.release = '4.9.3'
2626 self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
2727 end
28+
29+ def get_latest_version(opts)
30+ doc = fetch_doc('https://gcc.gnu.org/onlinedocs/', opts)
31+ label = doc.at_css('ul > li > ul > li > a').content.strip
32+ label.scan(/([0-9.]+)/)[0][0]
33+ end
2834 end
2935 end
lib/docs/scrapers/go.rb+5−0
@@ -24,6 +24,11 @@ module Docs · 23 unmodified lines
2424 Licensed under the Creative Commons Attribution License 3.0.
2525 HTML
2626
27+ def get_latest_version(opts)
28+ doc = fetch_doc('https://golang.org/project/', opts)
29+ doc.at_css('#page ul > li > a').text[3..-1]
30+ end
31+
2732 private
2833
2934 def parse(response) # Hook here because Nokogori removes whitespace from textareas
lib/docs/scrapers/godot.rb+5−0
@@ -37,5 +37,10 @@ module Docs · 36 unmodified lines
3737 self.release = '2.1'
3838 self.base_url = "http://docs.godotengine.org/en/#{self.version}/"
3939 end
40+
41+ def get_latest_version(opts)
42+ doc = fetch_doc('https://docs.godotengine.org/', opts)
43+ doc.at_css('.version').content.strip
44+ end
4045 end
4146 end
lib/docs/scrapers/graphite.rb+5−0
@@ -17,5 +17,10 @@ module Docs · 16 unmodified lines
1717 &copy; 2011&ndash;2016 The Graphite Project<br>
1818 Licensed under the Apache License, Version 2.0.
1919 HTML
20+
21+ def get_latest_version(opts)
22+ doc = fetch_doc('https://graphite.readthedocs.io/en/latest/releases.html', opts)
23+ doc.at_css('#release-notes li > a').content
24+ end
2025 end
2126 end
lib/docs/scrapers/grunt.rb+4−0
@@ -26,5 +26,9 @@ module Docs · 25 unmodified lines
2626 &copy; GruntJS Team<br>
2727 Licensed under the MIT License.
2828 HTML
29+
30+ def get_latest_version(opts)
31+ get_npm_version('grunt-cli', opts)
32+ end
2933 end
3034 end
lib/docs/scrapers/handlebars.rb+4−0
@@ -19,5 +19,9 @@ module Docs · 18 unmodified lines
1919 &copy; 2011&ndash;2017 by Yehuda Katz<br>
2020 Licensed under the MIT License.
2121 HTML
22+
23+ def get_latest_version(opts)
24+ get_npm_version('handlebars', opts)
25+ end
2226 end
2327 end
lib/docs/scrapers/haskell.rb+8−1
@@ -10,7 +10,7 @@ module Docs · 9 unmodified lines
1010
1111 html_filters.push 'haskell/entries', 'haskell/clean_html'
1212
13− options[:container] = ->(filter) { filter.subpath.start_with?('users_guide') ? '.body' : '#content' }
13+ options[:container] = ->(filter) {filter.subpath.start_with?('users_guide') ? '.body' : '#content'}
1414
1515 options[:only_patterns] = [/\Alibraries\//, /\Ausers_guide\//]
1616 options[:skip_patterns] = [
@@ -68,5 +68,12 @@ module Docs · 51 unmodified lines
6868
6969 options[:only_patterns] = [/\Alibraries\//]
7070 end
71+
72+ def get_latest_version(opts)
73+ doc = fetch_doc('https://downloads.haskell.org/~ghc/latest/docs/html/', opts)
74+ links = doc.css('a').to_a
75+ versions = links.map {|link| link['href'].scan(/ghc-([0-9.]+)/)}
76+ versions.find {|version| !version.empty?}[0][0]
77+ end
7178 end
7279 end
lib/docs/scrapers/haxe.rb+6−0
@@ -66,5 +66,11 @@ module Docs · 65 unmodified lines
6666 version 'Python' do
6767 self.base_url = 'https://api.haxe.org/python/'
6868 end
69+
70+ def get_latest_version(opts)
71+ doc = fetch_doc('https://api.haxe.org/', opts)
72+ label = doc.at_css('.container.main-content h1 > small').content
73+ label.sub(/version /, '')
74+ end
6975 end
7076 end
lib/docs/scrapers/homebrew.rb+5−1
@@ -2,7 +2,7 @@ module Docs · 1 unmodified line
22 class Homebrew < UrlScraper
33 self.name = 'Homebrew'
44 self.type = 'simple'
5− self.release = '1.8.1'
5+ self.release = '2.1.9'
66 self.base_url = 'https://docs.brew.sh/'
77 self.links = {
88 home: 'https://brew.sh',
@@ -19,5 +19,9 @@ module Docs · 10 unmodified lines
1919 &copy; 2009&ndash;present Homebrew contributors<br>
2020 Licensed under the BSD 2-Clause License.
2121 HTML
22+
23+ def get_latest_version(opts)
24+ get_latest_github_release('Homebrew', 'brew', opts)
25+ end
2226 end
2327 end
lib/docs/scrapers/http.rb+2−0
@@ -7,6 +7,8 @@ module Docs · 6 unmodified lines
77
88 html_filters.push 'http/clean_html', 'http/entries', 'title'
99
10+ options[:mdn_tag] = 'HTTP'
11+
1012 options[:root_title] = 'HTTP'
1113 options[:title] = ->(filter) { filter.current_url.host == 'tools.ietf.org' ? false : filter.default_title }
1214 options[:container] = ->(filter) { filter.current_url.host == 'tools.ietf.org' ? '.content' : nil }
lib/docs/scrapers/immutable.rb+4−0
@@ -54,5 +54,9 @@ module Docs · 53 unmodified lines
5454 JS
5555 capybara.html
5656 end
57+
58+ def get_latest_version(opts)
59+ get_npm_version('immutable', opts)
60+ end
5761 end
5862 end
lib/docs/scrapers/influxdata.rb+6−0
@@ -46,5 +46,11 @@ module Docs · 45 unmodified lines
4646 &copy; 2015 InfluxData, Inc.<br>
4747 Licensed under the MIT license.
4848 HTML
49+
50+ def get_latest_version(opts)
51+ doc = fetch_doc('https://docs.influxdata.com/influxdb/', opts)
52+ label = doc.at_css('.navbar--current-product').content.strip
53+ label.scan(/([0-9.]+)/)[0][0]
54+ end
4955 end
5056 end
lib/docs/scrapers/jasmine.rb+4−0
@@ -17,5 +17,9 @@ module Docs · 16 unmodified lines
1717 &copy; 2008&ndash;2017 Pivotal Labs<br>
1818 Licensed under the MIT License.
1919 HTML
20+
21+ def get_latest_version(opts)
22+ get_latest_github_release('jasmine', 'jasmine', opts)
23+ end
2024 end
2125 end
lib/docs/scrapers/jekyll.rb+5−0
@@ -28,5 +28,10 @@ module Docs · 27 unmodified lines
2828 &copy; 2008&ndash;2018 Tom Preston-Werner and Jekyll contributors<br>
2929 Licensed under the MIT license.
3030 HTML
31+
32+ def get_latest_version(opts)
33+ doc = fetch_doc('https://jekyllrb.com/docs/', opts)
34+ doc.at_css('.meta a').content[1..-1]
35+ end
3136 end
3237 end
lib/docs/scrapers/jest.rb+5−0
@@ -17,5 +17,10 @@ module Docs · 16 unmodified lines
1717 &copy; 2014&ndash;present Facebook Inc.<br>
1818 Licensed under the BSD License.
1919 HTML
20+
21+ def get_latest_version(opts)
22+ doc = fetch_doc('https://jestjs.io/docs/en/getting-started', opts)
23+ doc.at_css('header > a > h3').content
24+ end
2025 end
2126 end
lib/docs/scrapers/jquery/jquery_core.rb+4−0
@@ -22,5 +22,9 @@ module Docs · 21 unmodified lines
2222 /Selectors\/odd/i,
2323 /index/i
2424 ]
25+
26+ def get_latest_version(opts)
27+ get_npm_version('jquery', opts)
28+ end
2529 end
2630 end
lib/docs/scrapers/jquery/jquery_mobile.rb+5−0
@@ -16,5 +16,10 @@ module Docs · 15 unmodified lines
1616 options[:fix_urls] = ->(url) do
1717 url.sub! 'http://api.jquerymobile.com/', 'https://api.jquerymobile.com/'
1818 end
19+
20+ def get_latest_version(opts)
21+ doc = fetch_doc('https://jquerymobile.com/', opts)
22+ doc.at_css('.download-box > .download-option:last-child > span').content.sub(/Version /, '')
23+ end
1924 end
2025 end
lib/docs/scrapers/jquery/jquery_ui.rb+4−0
@@ -15,5 +15,9 @@ module Docs · 14 unmodified lines
1515 options[:fix_urls] = ->(url) do
1616 url.sub! 'http://api.jqueryui.com/', 'https://api.jqueryui.com/'
1717 end
18+
19+ def get_latest_version(opts)
20+ get_npm_version('jquery-ui', opts)
21+ end
1822 end
1923 end
lib/docs/scrapers/jsdoc.rb+4−0
@@ -21,5 +21,9 @@ module Docs · 20 unmodified lines
2121 &copy; 2011&ndash;2017 the contributors to the JSDoc 3 documentation project<br>
2222 Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0.
2323 HTML
24+
25+ def get_latest_version(opts)
26+ get_latest_github_release('jsdoc3', 'jsdoc', opts)
27+ end
2428 end
2529 end
lib/docs/scrapers/julia.rb+4−0
@@ -49,5 +49,9 @@ module Docs · 48 unmodified lines
4949
5050 html_filters.push 'julia/entries_sphinx', 'julia/clean_html_sphinx', 'sphinx/clean_html'
5151 end
52+
53+ def get_latest_version(opts)
54+ get_latest_github_release('JuliaLang', 'julia', opts)
55+ end
5256 end
5357 end
lib/docs/scrapers/knockout.rb+4−0
@@ -33,5 +33,9 @@ module Docs · 32 unmodified lines
3333 &copy; Steven Sanderson, the Knockout.js team, and other contributors<br>
3434 Licensed under the MIT License.
3535 HTML
36+
37+ def get_latest_version(opts)
38+ get_latest_github_release('knockout', 'knockout', opts)
39+ end
3640 end
3741 end
lib/docs/scrapers/koa.rb+4−0
@@ -34,5 +34,9 @@ module Docs · 33 unmodified lines
3434 &copy; 2018 Koa contributors<br>
3535 Licensed under the MIT License.
3636 HTML
37+
38+ def get_latest_version(opts)
39+ get_npm_version('koa', opts)
40+ end
3741 end
3842 end
lib/docs/scrapers/kotlin.rb+4−0
@@ -28,5 +28,9 @@ module Docs · 27 unmodified lines
2828 &copy; 2010&ndash;2018 JetBrains s.r.o.<br>
2929 Licensed under the Apache License, Version 2.0.
3030 HTML
31+
32+ def get_latest_version(opts)
33+ get_latest_github_release('JetBrains', 'kotlin', opts)
34+ end
3135 end
3236 end
lib/docs/scrapers/laravel.rb+4−0
@@ -133,5 +133,9 @@ module Docs · 132 unmodified lines
133133 url
134134 end
135135 end
136+
137+ def get_latest_version(opts)
138+ get_latest_github_release('laravel', 'laravel', opts)
139+ end
136140 end
137141 end
lib/docs/scrapers/leaflet.rb+5−0
@@ -39,5 +39,10 @@ module Docs · 38 unmodified lines
3939 self.base_url = "https://leafletjs.com/reference-#{release}.html"
4040 end
4141
42+ def get_latest_version(opts)
43+ doc = fetch_doc('https://leafletjs.com/index.html', opts)
44+ link = doc.css('ul > li > a').to_a.select {|node| node.content == 'Docs'}.first
45+ link['href'].scan(/reference-([0-9.]+)\.html/)[0][0]
46+ end
4247 end
4348 end
lib/docs/scrapers/less.rb+6−0
@@ -21,5 +21,11 @@ module Docs · 20 unmodified lines
2121 &copy; 2009&ndash;2016 The Core Less Team<br>
2222 Licensed under the Creative Commons Attribution License 3.0.
2323 HTML
24+
25+ def get_latest_version(opts)
26+ doc = fetch_doc('http://lesscss.org/features/', opts)
27+ label = doc.at_css('.footer-links > li').content
28+ label.scan(/([0-9.]+)/)[0][0]
29+ end
2430 end
2531 end
lib/docs/scrapers/liquid.rb+5−0
@@ -19,5 +19,10 @@ module Docs · 18 unmodified lines
1919 &copy; 2005, 2006 Tobias Luetke<br>
2020 Licensed under the MIT License.
2121 HTML
22+
23+ def get_latest_version(opts)
24+ tags = get_github_tags('Shopify', 'liquid', opts)
25+ tags[0]['name'][1..-1]
26+ end
2227 end
2328 end
lib/docs/scrapers/lodash.rb+5−0
@@ -32,5 +32,10 @@ module Docs · 31 unmodified lines
3232 self.release = '2.4.2'
3333 self.base_url = "https://lodash.com/docs/#{release}"
3434 end
35+
36+ def get_latest_version(opts)
37+ doc = fetch_doc('https://lodash.com/docs/', opts)
38+ doc.at_css('#version > option[selected]').content
39+ end
3540 end
3641 end
lib/docs/scrapers/love.rb+5−0
@@ -39,5 +39,10 @@ module Docs · 38 unmodified lines
3939 &copy; 2006&ndash;2016 L&Ouml;VE Development Team<br>
4040 Licensed under the GNU Free Documentation License, Version 1.3.
4141 HTML
42+
43+ def get_latest_version(opts)
44+ doc = fetch_doc('https://love2d.org/wiki/Version_History', opts)
45+ doc.at_css('#mw-content-text table a').content
46+ end
4247 end
4348 end
lib/docs/scrapers/lua.rb+5−0
@@ -26,5 +26,10 @@ module Docs · 25 unmodified lines
2626 self.release = '5.1.5'
2727 self.base_url = 'https://www.lua.org/manual/5.1/'
2828 end
29+
30+ def get_latest_version(opts)
31+ doc = fetch_doc('https://www.lua.org/manual/', opts)
32+ doc.at_css('p.menubar > a').content
33+ end
2934 end
3035 end
lib/docs/scrapers/marionette.rb+4−0
@@ -38,5 +38,9 @@ module Docs · 37 unmodified lines
3838
3939 html_filters.push 'marionette/entries_v2'
4040 end
41+
42+ def get_latest_version(opts)
43+ get_npm_version('backbone.marionette', opts)
44+ end
4145 end
4246 end
lib/docs/scrapers/markdown.rb+4−0
@@ -13,5 +13,9 @@ module Docs · 12 unmodified lines
1313 &copy; 2004 John Gruber<br>
1414 Licensed under the BSD License.
1515 HTML
16+
17+ def get_latest_version(opts)
18+ '1.0.0'
19+ end
1620 end
1721 end
lib/docs/scrapers/matplotlib.rb+4−0
@@ -64,5 +64,9 @@ module Docs · 63 unmodified lines
6464 "https://matplotlib.org/#{release}/mpl_toolkits/axes_grid/api/"
6565 ]
6666 end
67+
68+ def get_latest_version(opts)
69+ get_latest_github_release('matplotlib', 'matplotlib', opts)
70+ end
6771 end
6872 end
lib/docs/scrapers/mdn/css.rb+2−0
@@ -6,6 +6,8 @@ module Docs · 5 unmodified lines
66
77 html_filters.push 'css/clean_html', 'css/entries', 'title'
88
9+ options[:mdn_tag] = 'CSS'
10+
911 options[:root_title] = 'CSS'
1012
1113 options[:skip] = %w(/CSS3 /Media/Visual /paged_media /Media/TV /Media/Tactile)
lib/docs/scrapers/mdn/dom.rb+2−0
@@ -8,6 +8,8 @@ module Docs · 7 unmodified lines
88
99 html_filters.push 'dom/clean_html', 'dom/entries', 'title'
1010
11+ options[:mdn_tag] = 'XSLT_Reference'
12+
1113 options[:root_title] = 'DOM'
1214
1315 options[:skip] = %w(
lib/docs/scrapers/mdn/dom_events.rb+2−0
@@ -9,6 +9,8 @@ module Docs · 8 unmodified lines
99 html_filters.insert_after 'clean_html', 'dom_events/clean_html'
1010 html_filters.push 'dom_events/entries', 'title'
1111
12+ options[:mdn_tag] = 'events'
13+
1214 options[:root_title] = 'DOM Events'
1315
1416 options[:skip] = %w(/MozOrientation)
lib/docs/scrapers/mdn/html.rb+2−0
@@ -7,6 +7,8 @@ module Docs · 6 unmodified lines
77
88 html_filters.push 'html/clean_html', 'html/entries', 'title'
99
10+ options[:mdn_tag] = 'HTML'
11+
1012 options[:root_title] = 'HTML'
1113
1214 options[:title] = ->(filter) do
lib/docs/scrapers/mdn/javascript.rb+2−0
@@ -8,6 +8,8 @@ module Docs · 7 unmodified lines
88
99 html_filters.push 'javascript/clean_html', 'javascript/entries', 'title'
1010
11+ options[:mdn_tag] = 'JavaScript'
12+
1113 options[:root_title] = 'JavaScript'
1214
1315 # Don't want
lib/docs/scrapers/mdn/mdn.rb+5−0
@@ -21,6 +21,11 @@ module Docs · 20 unmodified lines
2121 Licensed under the Creative Commons Attribution-ShareAlike License v2.5 or later.
2222 HTML
2323
24+ def get_latest_version(opts)
25+ json = fetch_json("https://developer.mozilla.org/en-US/docs/feeds/json/tag/#{options[:mdn_tag]}", opts)
26+ DateTime.parse(json[0]['pubdate']).to_time.to_i
27+ end
28+
2429 private
2530
2631 def process_response?(response)
lib/docs/scrapers/mdn/svg.rb+2−0
@@ -8,6 +8,8 @@ module Docs · 7 unmodified lines
88
99 html_filters.push 'svg/clean_html', 'svg/entries', 'title'
1010
11+ options[:mdn_tag] = 'XSLT_Reference'
12+
1113 options[:root_title] = 'SVG'
1214
1315 options[:title] = ->(filter) do
lib/docs/scrapers/mdn/xslt_xpath.rb+2−0
@@ -8,6 +8,8 @@ module Docs · 7 unmodified lines
88
99 html_filters.push 'xslt_xpath/clean_html', 'xslt_xpath/entries', 'title'
1010
11+ options[:mdn_tag] = 'XSLT_Reference'
12+
1113 options[:root_title] = 'XSLT'
1214
1315 options[:only_patterns] = [/\A\/XSLT/, /\A\/XPath/]
lib/docs/scrapers/meteor.rb+5−0
@@ -45,5 +45,10 @@ module Docs · 44 unmodified lines
4545 self.base_urls = ['https://guide.meteor.com/v1.3/', "https://docs.meteor.com/v#{self.release}/"]
4646 options[:fix_urls] = nil
4747 end
48+
49+ def get_latest_version(opts)
50+ doc = fetch_doc('https://docs.meteor.com/#/full/', opts)
51+ doc.at_css('select.version-select > option').content
52+ end
4853 end
4954 end
lib/docs/scrapers/mocha.rb+4−0
@@ -18,5 +18,9 @@ module Docs · 17 unmodified lines
1818 &copy; 2011&ndash;2018 JS Foundation and contributors<br>
1919 Licensed under the Creative Commons Attribution 4.0 International License.
2020 HTML
21+
22+ def get_latest_version(opts)
23+ get_npm_version('mocha', opts)
24+ end
2125 end
2226 end
lib/docs/scrapers/modernizr.rb+4−0
@@ -15,5 +15,9 @@ module Docs · 14 unmodified lines
1515 &copy; 2009&ndash;2017 The Modernizr team<br>
1616 Licensed under the MIT License.
1717 HTML
18+
19+ def get_latest_version(opts)
20+ get_npm_version('modernizr', opts)
21+ end
1822 end
1923 end
lib/docs/scrapers/moment.rb+5−0
@@ -22,5 +22,10 @@ module Docs · 21 unmodified lines
2222 &copy; JS Foundation and other contributors<br>
2323 Licensed under the MIT License.
2424 HTML
25+
26+ def get_latest_version(opts)
27+ doc = fetch_doc('http://momentjs.com/', opts)
28+ doc.at_css('.hero-title > h1 > span').content
29+ end
2530 end
2631 end
lib/docs/scrapers/mongoose.rb+6−0
@@ -26,5 +26,11 @@ module Docs · 25 unmodified lines
2626 &copy; 2010 LearnBoost<br>
2727 Licensed under the MIT License.
2828 HTML
29+
30+ def get_latest_version(opts)
31+ doc = fetch_doc('https://mongoosejs.com/docs/', opts)
32+ label = doc.at_css('.pure-menu-link').content.strip
33+ label.sub(/Version /, '')
34+ end
2935 end
3036 end
lib/docs/scrapers/nginx.rb+6−0
@@ -25,5 +25,11 @@ module Docs · 24 unmodified lines
2525 &copy; 2011-2018 Nginx, Inc.<br>
2626 Licensed under the BSD License.
2727 HTML
28+
29+ def get_latest_version(opts)
30+ doc = fetch_doc('https://nginx.org/en/download.html', opts)
31+ table = doc.at_css('#content > table').inner_html
32+ table.scan(/nginx-([0-9.]+)</)[0][0]
33+ end
2834 end
2935 end
lib/docs/scrapers/nginx_lua_module.rb+9−0
@@ -4,6 +4,9 @@ module Docs · 3 unmodified lines
44 self.slug = 'nginx_lua_module'
55 self.release = '0.10.13'
66 self.base_url = "https://github.com/openresty/lua-nginx-module/tree/v#{self.release}/"
7+ self.links = {
8+ code: 'https://github.com/openresty/lua-nginx-module'
9+ }
710
811 html_filters.push 'nginx_lua_module/clean_html', 'nginx_lua_module/entries', 'title'
912
@@ -15,5 +18,11 @@ module Docs · 5 unmodified lines
1518 &copy; 2009&ndash;2018 Yichun "agentzh" Zhang (章亦春), OpenResty Inc.<br>
1619 Licensed under the BSD License.
1720 HTML
21+
22+ def get_latest_version(opts)
23+ tags = get_github_tags('openresty', 'lua-nginx-module', opts)
24+ tag = tags.find {|tag| !tag['name'].include?('rc')}
25+ tag['name'][1..-1]
26+ end
1827 end
1928 end
lib/docs/scrapers/nim.rb+5−0
@@ -17,5 +17,10 @@ module Docs · 16 unmodified lines
1717 &copy; 2006&ndash;2018 Andreas Rumpf<br>
1818 Licensed under the MIT License.
1919 HTML
20+
21+ def get_latest_version(opts)
22+ doc = fetch_doc('https://nim-lang.org/docs/overview.html', opts)
23+ doc.at_css('.container > .docinfo > tbody > tr:last-child > td').content.strip
24+ end
2025 end
2126 end
lib/docs/scrapers/node.rb+5−0
@@ -46,5 +46,10 @@ module Docs · 45 unmodified lines
4646 self.release = '4.9.1'
4747 self.base_url = 'https://nodejs.org/dist/latest-v4.x/docs/api/'
4848 end
49+
50+ def get_latest_version(opts)
51+ doc = fetch_doc('https://nodejs.org/en/', opts)
52+ doc.at_css('#home-intro > .home-downloadblock:last-of-type > a')['data-version'][1..-1]
53+ end
4954 end
5055 end
lib/docs/scrapers/nokogiri2.rb+4−0
@@ -19,5 +19,9 @@ module Docs · 18 unmodified lines
1919 Patrick Mahoney, Yoko Harada, Akinori Musha, John Shahid, Lars Kanis<br>
2020 Licensed under the MIT License.
2121 HTML
22+
23+ def get_latest_version(opts)
24+ get_latest_github_release('sparklemotion', 'nokogiri', opts)
25+ end
2226 end
2327 end
lib/docs/scrapers/npm.rb+4−0
@@ -29,5 +29,9 @@ module Docs · 28 unmodified lines
2929 Licensed under the npm License.<br>
3030 npm is a trademark of npm, Inc.
3131 HTML
32+
33+ def get_latest_version(opts)
34+ get_latest_github_release('npm', 'cli', opts)
35+ end
3236 end
3337 end
lib/docs/scrapers/numpy.rb+4−0
@@ -49,5 +49,9 @@ module Docs · 48 unmodified lines
4949 self.release = '1.10.4'
5050 self.base_url = "https://docs.scipy.org/doc/numpy-#{self.release}/reference/"
5151 end
52+
53+ def get_latest_version(opts)
54+ get_latest_github_release('numpy', 'numpy', opts)
55+ end
5256 end
5357 end
lib/docs/scrapers/openjdk.rb+20−0
@@ -86,5 +86,25 @@ module Docs · 85 unmodified lines
8686 def read_file(path)
8787 File.read(path).force_encoding('iso-8859-1').encode('utf-8') rescue nil
8888 end
89+
90+ def get_latest_version(opts)
91+ latest_version = 8
92+ current_attempt = latest_version
93+ attempts = 0
94+
95+ while attempts < 3
96+ current_attempt += 1
97+
98+ doc = fetch_doc("https://packages.debian.org/sid/openjdk-#{current_attempt}-doc", opts)
99+ if doc.at_css('.perror').nil?
100+ latest_version = current_attempt
101+ attempts = 0
102+ else
103+ attempts += 1
104+ end
105+ end
106+
107+ latest_version
108+ end
89109 end
90110 end
lib/docs/scrapers/opentsdb.rb+4−0
@@ -18,5 +18,9 @@ module Docs · 17 unmodified lines
1818 &copy; 2010&ndash;2016 The OpenTSDB Authors<br>
1919 Licensed under the GNU LGPLv2.1+ and GPLv3+ licenses.
2020 HTML
21+
22+ def get_latest_version(opts)
23+ get_latest_github_release('OpenTSDB', 'opentsdb', opts)
24+ end
2125 end
2226 end
lib/docs/scrapers/padrino.rb+4−0
@@ -23,5 +23,9 @@ module Docs · 22 unmodified lines
2323 stub 'index2' do
2424 request_one(url_for('index')).body
2525 end
26+
27+ def get_latest_version(opts)
28+ get_github_tags('padrino', 'padrino-framework', opts)[0]['name']
29+ end
2630 end
2731 end
lib/docs/scrapers/pandas.rb+6−0
@@ -49,5 +49,11 @@ module Docs · 48 unmodified lines
4949 self.release = '0.18.1'
5050 self.base_url = "http://pandas.pydata.org/pandas-docs/version/#{self.release}/"
5151 end
52+
53+ def get_latest_version(opts)
54+ doc = fetch_doc('http://pandas.pydata.org/pandas-docs/stable/', opts)
55+ label = doc.at_css('.body > .section > p').content
56+ label.scan(/Version: ([0-9.]+)/)[0][0]
57+ end
5258 end
5359 end
lib/docs/scrapers/perl.rb+6−0
@@ -43,5 +43,11 @@ module Docs · 42 unmodified lines
4343 self.release = '5.20.2'
4444 self.base_url = "https://perldoc.perl.org/#{self.release}/"
4545 end
46+
47+ def get_latest_version(opts)
48+ doc = fetch_doc('https://perldoc.perl.org/', opts)
49+ header = doc.at_css('h2.h1').content
50+ header.scan(/Perl ([0-9.]+)/)[0][0]
51+ end
4652 end
4753 end
lib/docs/scrapers/phalcon.rb+5−0
@@ -29,5 +29,10 @@ module Docs · 28 unmodified lines
2929 self.release = '2.0.13'
3030 self.base_url = 'https://docs.phalconphp.com/en/2.0.0/'
3131 end
32+
33+ def get_latest_version(opts)
34+ doc = fetch_doc('https://docs.phalconphp.com/', opts)
35+ doc.at_css('.header__lang.expand > div > ul > li > a').content
36+ end
3237 end
3338 end
lib/docs/scrapers/phaser.rb+4−0
@@ -25,5 +25,9 @@ module Docs · 24 unmodified lines
2525 &copy; 2016 Richard Davey, Photon Storm Ltd.<br>
2626 Licensed under the MIT License.
2727 HTML
28+
29+ def get_latest_version(opts)
30+ get_latest_github_release('photonstorm', 'phaser', opts)
31+ end
2832 end
2933 end
lib/docs/scrapers/phoenix.rb+5−0
@@ -46,5 +46,10 @@ module Docs · 45 unmodified lines
4646 HTML
4747 end
4848 }
49+
50+ def get_latest_version(opts)
51+ doc = fetch_doc('https://hexdocs.pm/phoenix/Phoenix.html', opts)
52+ doc.at_css('.sidebar-projectVersion').content.strip[1..-1]
53+ end
4954 end
5055 end
lib/docs/scrapers/php.rb+6−0
@@ -66,5 +66,11 @@ module Docs · 65 unmodified lines
6666 &copy; 1997&ndash;2018 The PHP Documentation Group<br>
6767 Licensed under the Creative Commons Attribution License v3.0 or later.
6868 HTML
69+
70+ def get_latest_version(opts)
71+ doc = fetch_doc('https://secure.php.net/manual/en/doc.changelog.php', opts)
72+ label = doc.at_css('tbody.gen-changelog > tr > td').content
73+ label.split(',').last.strip
74+ end
6975 end
7076 end
lib/docs/scrapers/phpunit.rb+6−0
@@ -37,5 +37,11 @@ module Docs · 36 unmodified lines
3737 self.release = '4.8'
3838 self.base_url = "https://phpunit.de/manual/#{release}/en/"
3939 end
40+
41+ def get_latest_version(opts)
42+ doc = fetch_doc('https://phpunit.readthedocs.io/', opts)
43+ label = doc.at_css('.rst-current-version').content.strip
44+ label.scan(/v: ([0-9.]+)/)[0][0]
45+ end
4046 end
4147 end
lib/docs/scrapers/postgresql.rb+6−0
@@ -80,5 +80,11 @@ module Docs · 79 unmodified lines
8080
8181 html_filters.insert_before 'postgresql/extract_metadata', 'postgresql/normalize_class_names'
8282 end
83+
84+ def get_latest_version(opts)
85+ doc = fetch_doc('https://www.postgresql.org/docs/current/index.html', opts)
86+ label = doc.at_css('#pgContentWrap h1.title').content
87+ label.scan(/([0-9.]+)/)[0][0]
88+ end
8389 end
8490 end
lib/docs/scrapers/pug.rb+4−0
@@ -18,6 +18,10 @@ module Docs · 17 unmodified lines
1818 Licensed under the MIT license.
1919 HTML
2020
21+ def get_latest_version(opts)
22+ get_npm_version('pug', opts)
23+ end
24+
2125 private
2226
2327 def parse(response) # Hook here because Nokogori removes whitespace from textareas
lib/docs/scrapers/puppeteer.rb+5−0
@@ -14,5 +14,10 @@ module Docs · 13 unmodified lines
1414 &copy; 2017 Google Inc<br>
1515 Licensed under the Apache License 2.0.
1616 HTML
17+
18+ def get_latest_version(opts)
19+ contents = get_github_file_contents('GoogleChrome', 'puppeteer', 'README.md', opts)
20+ contents.scan(/\/v([0-9.]+)\/docs\/api\.md/)[0][0]
21+ end
1722 end
1823 end
lib/docs/scrapers/pygame.rb+5−0
@@ -2,6 +2,7 @@ module Docs · 1 unmodified line
22 class Pygame < UrlScraper
33 self.type = 'simple'
44 self.release = '1.9.4'
5+ self.base_url = 'https://www.pygame.org/docs/'
56 self.root_path = 'py-modindex.html'
67 self.links = {
78 home: 'https://www.pygame.org/',
@@ -16,5 +17,9 @@ module Docs · 8 unmodified lines
1617 &copy; Pygame Developpers.<br>
1718 Licensed under the GNU LGPL License version 2.1.
1819 HTML
20+
21+ def get_latest_version(opts)
22+ get_latest_github_release('pygame', 'pygame', opts)
23+ end
1924 end
2025 end
lib/docs/scrapers/python.rb+5−0
@@ -50,5 +50,10 @@ module Docs · 49 unmodified lines
5050
5151 html_filters.push 'python/entries_v2', 'sphinx/clean_html', 'python/clean_html'
5252 end
53+
54+ def get_latest_version(opts)
55+ doc = fetch_doc('https://docs.python.org/', opts)
56+ doc.at_css('.version_switcher_placeholder').content
57+ end
5358 end
5459 end
lib/docs/scrapers/q.rb+4−0
@@ -19,5 +19,9 @@ module Docs · 18 unmodified lines
1919 &copy; 2009&ndash;2017 Kristopher Michael Kowal<br>
2020 Licensed under the MIT License.
2121 HTML
22+
23+ def get_latest_version(opts)
24+ get_npm_version('q', opts)
25+ end
2226 end
2327 end
lib/docs/scrapers/qt.rb+17−2
@@ -103,9 +103,19 @@ module Docs · 102 unmodified lines
103103 Licensed under the GNU Free Documentation License, Version 1.3.
104104 HTML
105105
106+ version '5.13' do
107+ self.release = '5.13'
108+ self.base_url = 'https://doc.qt.io/qt-5.13/'
109+ end
110+
111+ version '5.12' do
112+ self.release = '5.12'
113+ self.base_url = 'https://doc.qt.io/qt-5.12/'
114+ end
115+
106116 version '5.11' do
107117 self.release = '5.11'
108− self.base_url = 'https://doc.qt.io/qt-5/'
118+ self.base_url = 'https://doc.qt.io/archives/qt-5.11/'
109119 end
110120
111121 version '5.9' do
@@ -115,7 +125,12 @@ module Docs · 3 unmodified lines
115125
116126 version '5.6' do
117127 self.release = '5.6'
118− self.base_url = 'https://doc.qt.io/qt-5.6/'
128+ self.base_url = 'https://doc.qt.io/archives/qt-5.6/'
129+ end
130+
131+ def get_latest_version(opts)
132+ doc = fetch_doc('https://doc.qt.io/qt-5/index.html', opts)
133+ doc.at_css('.mainContent h1.title').content.sub(/Qt /, '')
119134 end
120135 end
121136 end
lib/docs/scrapers/ramda.rb+5−0
@@ -15,6 +15,11 @@ module Docs · 14 unmodified lines
1515 &copy; 2013&ndash;2016 Scott Sauyet and Michael Hurley<br>
1616 Licensed under the MIT License.
1717 HTML
18+
19+ def get_latest_version(opts)
20+ doc = fetch_doc('https://ramdajs.com/docs/', opts)
21+ doc.at_css('.navbar-brand > .version').content[1..-1]
22+ end
1823 end
1924 end
2025
lib/docs/scrapers/rdoc/minitest.rb+5−0
@@ -21,5 +21,10 @@ module Docs · 20 unmodified lines
2121 &copy; Ryan Davis, seattle.rb<br>
2222 Licensed under the MIT License.
2323 HTML
24+
25+ def get_latest_version(opts)
26+ contents = get_github_file_contents('seattlerb', 'minitest', 'History.rdoc', opts)
27+ contents.scan(/([0-9.]+)/)[0][0]
28+ end
2429 end
2530 end
lib/docs/scrapers/rdoc/rails.rb+4−0
@@ -93,5 +93,9 @@ module Docs · 92 unmodified lines
9393 version '4.1' do
9494 self.release = '4.1.16'
9595 end
96+
97+ def get_latest_version(opts)
98+ get_latest_github_release('rails', 'rails', opts)
99+ end
96100 end
97101 end
lib/docs/scrapers/rdoc/ruby.rb+11−0
@@ -84,5 +84,16 @@ module Docs · 83 unmodified lines
8484 version '2.2' do
8585 self.release = '2.2.10'
8686 end
87+
88+ def get_latest_version(opts)
89+ tags = get_github_tags('ruby', 'ruby', opts)
90+ tags.each do |tag|
91+ version = tag['name'].gsub(/_/, '.')[1..-1]
92+
93+ if !/^([0-9.]+)$/.match(version).nil? && version.count('.') == 2
94+ return version
95+ end
96+ end
97+ end
8798 end
8899 end
lib/docs/scrapers/react.rb+5−0
@@ -30,5 +30,10 @@ module Docs · 29 unmodified lines
3030 &copy; 2013&ndash;present Facebook Inc.<br>
3131 Licensed under the Creative Commons Attribution 4.0 International Public License.
3232 HTML
33+
34+ def get_latest_version(opts)
35+ doc = fetch_doc('https://reactjs.org/docs/getting-started.html', opts)
36+ doc.at_css('a[href="/versions"]').content.strip[1..-1]
37+ end
3338 end
3439 end
lib/docs/scrapers/react_native.rb+5−0
@@ -30,5 +30,10 @@ module Docs · 29 unmodified lines
3030 &copy; 2015&ndash;2018 Facebook Inc.<br>
3131 Licensed under the Creative Commons Attribution 4.0 International Public License.
3232 HTML
33+
34+ def get_latest_version(opts)
35+ doc = fetch_doc('https://facebook.github.io/react-native/docs/getting-started.html', opts)
36+ doc.at_css('header > a > h3').content
37+ end
3338 end
3439 end
lib/docs/scrapers/redis.rb+6−0
@@ -19,5 +19,11 @@ module Docs · 18 unmodified lines
1919 &copy; 2009&ndash;2018 Salvatore Sanfilippo<br>
2020 Licensed under the Creative Commons Attribution-ShareAlike License 4.0.
2121 HTML
22+
23+ def get_latest_version(opts)
24+ body = fetch('http://download.redis.io/redis-stable/00-RELEASENOTES', opts)
25+ body = body.lines[1..-1].join
26+ body.scan(/Redis ([0-9.]+)/)[0][0]
27+ end
2228 end
2329 end
lib/docs/scrapers/redux.rb+4−0
@@ -20,5 +20,9 @@ module Docs · 19 unmodified lines
2020 stub '' do
2121 request_one('http://redux.js.org/index.html').body
2222 end
23+
24+ def get_latest_version(opts)
25+ get_npm_version('redux', opts)
26+ end
2327 end
2428 end
lib/docs/scrapers/relay.rb+5−0
@@ -18,5 +18,10 @@ module Docs · 17 unmodified lines
1818 &copy; 2013&ndash;present Facebook Inc.<br>
1919 Licensed under the BSD License.
2020 HTML
21+
22+ def get_latest_version(opts)
23+ doc = fetch_doc('http://facebook.github.io/relay/en/', opts)
24+ doc.at_css('header > a > h3').content[1..-1]
25+ end
2126 end
2227 end
lib/docs/scrapers/requirejs.rb+4−0
@@ -30,5 +30,9 @@ module Docs · 29 unmodified lines
3030 &copy; jQuery Foundation and other contributors<br>
3131 Licensed under the MIT License.
3232 HTML
33+
34+ def get_latest_version(opts)
35+ get_npm_version('requirejs', opts)
36+ end
3337 end
3438 end
lib/docs/scrapers/rethinkdb.rb+4−0
@@ -58,6 +58,10 @@ module Docs · 57 unmodified lines
5858 CODE
5959 end
6060
61+ def get_latest_version(opts)
62+ get_latest_github_release('rethinkdb', 'rethinkdb', opts)
63+ end
64+
6165 private
6266
6367 def process_response?(response)
lib/docs/scrapers/rust.rb+6−0
@@ -39,6 +39,12 @@ module Docs · 38 unmodified lines
3939 Licensed under the Apache License, Version 2.0 or the MIT license, at your option.
4040 HTML
4141
42+ def get_latest_version(opts)
43+ doc = fetch_doc('https://www.rust-lang.org/', opts)
44+ label = doc.at_css('.button-download + p > a').content
45+ label.sub(/Version /, '')
46+ end
47+
4248 private
4349
4450 REDIRECT_RGX = /http-equiv="refresh"/i
lib/docs/scrapers/sass.rb+4−0
@@ -23,5 +23,9 @@ module Docs · 22 unmodified lines
2323 &copy; 2006&ndash;2016 Hampton Catlin, Nathan Weizenbaum, and Chris Eppstein<br>
2424 Licensed under the MIT License.
2525 HTML
26+
27+ def get_latest_version(opts)
28+ get_latest_github_release('sass', 'libsass', opts)
29+ end
2630 end
2731 end
lib/docs/scrapers/scikit_image.rb+5−0
@@ -20,5 +20,10 @@ module Docs · 19 unmodified lines
2020 &copy; 2011 the scikit-image team<br>
2121 Licensed under the BSD 3-clause License.
2222 HTML
23+
24+ def get_latest_version(opts)
25+ tags = get_github_tags('scikit-image', 'scikit-image', opts)
26+ tags[0]['name'][1..-1]
27+ end
2328 end
2429 end
lib/docs/scrapers/scikit_learn.rb+4−0
@@ -25,5 +25,9 @@ module Docs · 24 unmodified lines
2525 Licensed under the 3-clause BSD License.
2626 HTML
2727
28+ def get_latest_version(opts)
29+ doc = fetch_doc('https://scikit-learn.org/stable/documentation.html', opts)
30+ doc.at_css('.body h1').content.scan(/([0-9.]+)/)[0][0]
31+ end
2832 end
2933 end
lib/docs/scrapers/sinon.rb+5−0
@@ -52,5 +52,10 @@ module Docs · 51 unmodified lines
5252 self.release = '1.17.7'
5353 self.base_url = "https://sinonjs.org/releases/v#{release}/"
5454 end
55+
56+ def get_latest_version(opts)
57+ body = fetch('https://sinonjs.org/', opts)
58+ body.scan(/\/releases\/v([0-9.]+)/)[0][0]
59+ end
5560 end
5661 end
lib/docs/scrapers/socketio.rb+4−0
@@ -20,5 +20,9 @@ module Docs · 19 unmodified lines
2020 &copy; 2014&ndash;2015 Automattic<br>
2121 Licensed under the MIT License.
2222 HTML
23+
24+ def get_latest_version(opts)
25+ get_npm_version('socket.io', opts)
26+ end
2327 end
2428 end
lib/docs/scrapers/sqlite.rb+5−0
@@ -41,6 +41,11 @@ module Docs · 40 unmodified lines
4141
4242 options[:attribution] = 'SQLite is in the Public Domain.'
4343
44+ def get_latest_version(opts)
45+ doc = fetch_doc('https://sqlite.org/chronology.html', opts)
46+ doc.at_css('#chrontab > tbody > tr > td:last-child > a').content
47+ end
48+
4449 private
4550
4651 def parse(response)
lib/docs/scrapers/statsmodels.rb+4−0
@@ -21,5 +21,9 @@ module Docs · 20 unmodified lines
2121 Licensed under the 3-clause BSD License.
2222 HTML
2323
24+ def get_latest_version(opts)
25+ doc = fetch_doc('http://www.statsmodels.org/stable/', opts)
26+ doc.at_css('.sphinxsidebarwrapper h3 + p > b').content[1..-1]
27+ end
2428 end
2529 end
lib/docs/scrapers/support_tables.rb+6−0
@@ -178,5 +178,11 @@ module Docs · 177 unmodified lines
178178 </p>
179179 </div>
180180 HTML
181+
182+ def get_latest_version(opts)
183+ body = fetch('https://feeds.feedburner.com/WhenCanIUse?format=xml', opts)
184+ timestamp = body.scan(/<updated>([^<]+)<\/updated>/)[0][0]
185+ DateTime.parse(timestamp).to_time.to_i
186+ end
181187 end
182188 end
lib/docs/scrapers/symfony.rb+4−0
@@ -70,5 +70,9 @@ module Docs · 69 unmodified lines
7070 self.release = '2.7.35'
7171 self.base_url = "https://api.symfony.com/#{version}/"
7272 end
73+
74+ def get_latest_version(opts)
75+ get_latest_github_release('symfony', 'symfony', opts)
76+ end
7377 end
7478 end
lib/docs/scrapers/tcl_tk.rb+5−0
@@ -25,5 +25,10 @@ module Docs · 24 unmodified lines
2525 options[:attribution] = <<-HTML
2626 Licensed under <a href="http://tcl.tk/software/tcltk/license.html">Tcl/Tk terms</a>
2727 HTML
28+
29+ def get_latest_version(opts)
30+ doc = fetch_doc('https://www.tcl.tk/man/tcl/contents.htm', opts)
31+ doc.at_css('h2').content.scan(/Tk([0-9.]+)/)[0][0]
32+ end
2833 end
2934 end
lib/docs/scrapers/tensorflow.rb+4−0
@@ -56,6 +56,10 @@ module Docs · 55 unmodified lines
5656 /\Aextend/]
5757 end
5858
59+ def get_latest_version(opts)
60+ get_latest_github_release('tensorflow', 'tensorflow', opts)
61+ end
62+
5963 private
6064
6165 def parse(response)
lib/docs/scrapers/terraform.rb+5−0
@@ -18,5 +18,10 @@ module Docs · 17 unmodified lines
1818 &copy; 2018 HashiCorp</br>
1919 Licensed under the MPL 2.0 License.
2020 HTML
21+
22+ def get_latest_version(opts)
23+ contents = get_github_file_contents('hashicorp', 'terraform-website', 'content/config.rb', opts)
24+ contents.scan(/version\s+=\s+"([0-9.]+)"/)[0][0]
25+ end
2126 end
2227 end
lib/docs/scrapers/twig.rb+5−0
@@ -28,5 +28,10 @@ module Docs · 27 unmodified lines
2828 self.release = '1.34.3'
2929 self.base_url = 'https://twig.symfony.com/doc/1.x/'
3030 end
31+
32+ def get_latest_version(opts)
33+ tags = get_github_tags('twigphp', 'Twig', opts)
34+ tags[0]['name'][1..-1]
35+ end
3136 end
3237 end
lib/docs/scrapers/typescript.rb+4−0
@@ -24,6 +24,10 @@ module Docs · 23 unmodified lines
2424 &copy; Microsoft and other contributors<br>
2525 Licensed under the Apache License, Version 2.0.
2626 HTML
27+
28+ def get_latest_version(opts)
29+ get_latest_github_release('Microsoft', 'TypeScript', opts)
30+ end
2731 end
2832 end
2933
lib/docs/scrapers/underscore.rb+5−0
@@ -20,5 +20,10 @@ module Docs · 19 unmodified lines
2020 &copy; 2009&ndash;2018 Jeremy Ashkenas, DocumentCloud and Investigative Reporters &amp; Editors<br>
2121 Licensed under the MIT License.
2222 HTML
23+
24+ def get_latest_version(opts)
25+ doc = fetch_doc('https://underscorejs.org/', opts)
26+ doc.at_css('.version').content[1...-1]
27+ end
2328 end
2429 end
lib/docs/scrapers/vagrant.rb+5−0
@@ -18,5 +18,10 @@ module Docs · 17 unmodified lines
1818 &copy; 2010&ndash;2018 Mitchell Hashimoto<br>
1919 Licensed under the MPL 2.0 License.
2020 HTML
21+
22+ def get_latest_version(opts)
23+ contents = get_github_file_contents('hashicorp', 'vagrant', 'website/config.rb', opts)
24+ contents.scan(/version\s+=\s+"([0-9.]+)"/)[0][0]
25+ end
2126 end
2227 end
lib/docs/scrapers/vue.rb+4−0
@@ -32,5 +32,9 @@ module Docs · 31 unmodified lines
3232 self.root_path = '/guide/index.html'
3333 self.initial_paths = %w(/api/index.html)
3434 end
35+
36+ def get_latest_version(opts)
37+ get_latest_github_release('vuejs', 'vue', opts)
38+ end
3539 end
3640 end
lib/docs/scrapers/vulkan.rb+5−0
@@ -20,5 +20,10 @@ module Docs · 19 unmodified lines
2020 Licensed under the Creative Commons Attribution 4.0 International License.<br>
2121 Vulkan and the Vulkan logo are registered trademarks of the Khronos Group Inc.
2222 HTML
23+
24+ def get_latest_version(opts)
25+ tags = get_github_tags('KhronosGroup', 'Vulkan-Docs', opts)
26+ tags[0]['name'][1..-1]
27+ end
2328 end
2429 end
lib/docs/scrapers/webpack.rb+4−0
@@ -68,5 +68,9 @@ module Docs · 67 unmodified lines
6868 Licensed under the MIT License.
6969 HTML
7070 end
71+
72+ def get_latest_version(opts)
73+ get_npm_version('webpack', opts)
74+ end
7175 end
7276 end
lib/docs/scrapers/yarn.rb+4−0
@@ -20,5 +20,9 @@ module Docs · 19 unmodified lines
2020 &copy; 2016&ndash;present Yarn Contributors<br>
2121 Licensed under the BSD License.
2222 HTML
23+
24+ def get_latest_version(opts)
25+ get_latest_github_release('yarnpkg', 'yarn', opts)
26+ end
2327 end
2428 end
lib/docs/scrapers/yii.rb+4−0
@@ -34,5 +34,9 @@ module Docs · 33 unmodified lines
3434
3535 options[:container] = '.grid_9'
3636 end
37+
38+ def get_latest_version(opts)
39+ get_latest_github_release('yiisoft', 'yii2', opts)
40+ end
3741 end
3842 end
lib/tasks/assets.thor+3−0
@@ -14,6 +14,9 @@ class AssetsCLI < Thor · 13 unmodified lines
1414 option :keep, type: :numeric, default: 0, desc: 'Number of old assets to keep'
1515 option :verbose, type: :boolean
1616 def compile
17+ load 'tasks/sprites.thor'
18+ invoke 'sprites:generate', [], :remove_public_icons => true, :verbose => options[:verbose]
19+
1720 manifest.compile App.assets_compile
1821 manifest.clean(options[:keep]) if options[:clean]
1922 end
lib/tasks/sprites.thor+213−0
@@ -0,0 +1,213 @@
1+class SpritesCLI < Thor
2+ def self.to_s
3+ 'Sprites'
4+ end
5+
6+ def initialize(*args)
7+ require 'docs'
8+ require 'chunky_png'
9+ require 'fileutils'
10+ super
11+ end
12+
13+ desc 'generate [--remove-public-icons] [--verbose]', 'Generate the documentation icon spritesheets'
14+ option :remove_public_icons, type: :boolean, desc: 'Remove public/icons after generating the spritesheets'
15+ option :verbose, type: :boolean
16+ def generate
17+ items = get_items
18+ items_with_icons = items.select {|item| item[:has_icons]}
19+ items_without_icons = items.select {|item| !item[:has_icons]}
20+ icons_per_row = Math.sqrt(items_with_icons.length).ceil
21+
22+ bg_color = get_sidebar_background
23+
24+ items_with_icons.each_with_index do |item, index|
25+ item[:row] = (index / icons_per_row).floor
26+ item[:col] = index - item[:row] * icons_per_row
27+
28+ item[:icon_16] = get_icon(item[:path_16], 16)
29+ item[:icon_32] = get_icon(item[:path_32], 32)
30+
31+ item[:dark_icon_fix] = needs_dark_icon_fix(item[:icon_32], bg_color)
32+ end
33+
34+ return unless items_with_icons.length > 0
35+
36+ log_details(items_with_icons, icons_per_row)
37+
38+ generate_spritesheet(16, items_with_icons, 'assets/images/sprites/docs.png') {|item| item[:icon_16]}
39+ generate_spritesheet(32, items_with_icons, 'assets/images/sprites/docs@2x.png') {|item| item[:icon_32]}
40+
41+ # Add Mongoose's icon details to docs without custom icons
42+ default_item = items_with_icons.find {|item| item[:type] == 'mongoose'}
43+ items_without_icons.each do |item|
44+ item[:row] = default_item[:row]
45+ item[:col] = default_item[:col]
46+ item[:dark_icon_fix] = default_item[:dark_icon_fix]
47+ end
48+
49+ save_manifest(items, icons_per_row, 'assets/images/sprites/docs.json')
50+
51+ if options[:remove_public_icons]
52+ logger.info('Removing public/icons')
53+ FileUtils.rm_rf('public/icons')
54+ end
55+ end
56+
57+ private
58+
59+ def get_items
60+ items = Docs.all.map do |doc|
61+ base_path = "public/icons/docs/#{doc.slug}"
62+ {
63+ :type => doc.slug,
64+ :path_16 => "#{base_path}/16.png",
65+ :path_32 => "#{base_path}/16@2x.png"
66+ }
67+ end
68+
69+ # Checking paths against an array of possible paths is faster than 200+ File.exist? calls
70+ files = Dir.glob('public/icons/docs/**/*.png')
71+
72+ items.each do |item|
73+ item[:has_icons] = files.include?(item[:path_16]) && files.include?(item[:path_32])
74+ end
75+ end
76+
77+ def get_icon(path, max_size)
78+ icon = ChunkyPNG::Image.from_file(path)
79+
80+ # Check if the icon is too big
81+ # If it is, resize the image without changing the aspect ratio
82+ if icon.width > max_size || icon.height > max_size
83+ ratio = icon.width.to_f / icon.height
84+ new_width = (icon.width >= icon.height ? max_size : max_size * ratio).floor
85+ new_height = (icon.width >= icon.height ? max_size / ratio : max_size).floor
86+
87+ logger.warn("Icon #{path} is too big: max size is #{max_size} x #{max_size}, icon is #{icon.width} x #{icon.height}, resizing to #{new_width} x #{new_height}")
88+
89+ icon.resample_nearest_neighbor!(new_width, new_height)
90+ end
91+
92+ icon
93+ end
94+
95+ def get_sidebar_background
96+ # This is a hacky way to get the background color of the sidebar
97+ # Unfortunately, it's not possible to get the value of a SCSS variable from a Thor task
98+ # Because hard-coding the value is even worse, we extract it using some regex
99+ path = 'assets/stylesheets/global/_variables-dark.scss'
100+ regex = /--sidebarBackground:\s+([^;]+);/
101+ ChunkyPNG::Color.parse(File.read(path)[regex, 1])
102+ end
103+
104+ def needs_dark_icon_fix(icon, bg_color)
105+ # Determine whether the icon needs to be grayscaled if the user has enabled the dark theme
106+ # The logic is roughly based on https://www.w3.org/TR/2008/REC-WCAG20-20081211/#visual-audio-contrast
107+ contrast = icon.pixels.select {|pixel| ChunkyPNG::Color.a(pixel) > 0}.map do |pixel|
108+ get_contrast(bg_color, pixel)
109+ end
110+
111+ avg = contrast.reduce(:+) / contrast.size.to_f
112+ avg < 3.5
113+ end
114+
115+ def get_contrast(base, other)
116+ # Calculating the contrast ratio as described in the WCAG 2.0:
117+ # https://www.w3.org/TR/2008/REC-WCAG20-20081211/#contrast-ratiodef
118+ l1 = get_luminance(base) + 0.05
119+ l2 = get_luminance(other) + 0.05
120+ ratio = l1 / l2
121+ l2 > l1 ? 1 / ratio : ratio
122+ end
123+
124+ def get_luminance(color)
125+ rgb = [
126+ ChunkyPNG::Color.r(color).to_f,
127+ ChunkyPNG::Color.g(color).to_f,
128+ ChunkyPNG::Color.b(color).to_f
129+ ]
130+
131+ # Calculating the relative luminance as described in the WCAG 2.0:
132+ # https://www.w3.org/TR/2008/REC-WCAG20-20081211/#relativeluminancedef
133+
134+ rgb.map! do |value|
135+ value /= 255
136+ value < 0.03928 ? value / 12.92 : ((value + 0.055) / 1.055) ** 2.4
137+ end
138+
139+ 0.2126 * rgb[0] + 0.7152 * rgb[1] + 0.0722 * rgb[2]
140+ end
141+
142+ def generate_spritesheet(size, items_with_icons, output_path, &item_to_icon)
143+ logger.info("Generating spritesheet #{output_path} with icons of size #{size} x #{size}")
144+
145+ icons_per_row = Math.sqrt(items_with_icons.length).ceil
146+ spritesheet = ChunkyPNG::Image.new(size * icons_per_row, size * icons_per_row)
147+
148+ items_with_icons.each do |item|
149+ icon = item_to_icon.call(item)
150+
151+ # Calculate the base coordinates
152+ base_x = item[:col] * size
153+ base_y = item[:row] * size
154+
155+ # Center the icon if it's not a perfect rectangle
156+ x = base_x + ((size - icon.width) / 2).floor
157+ y = base_y + ((size - icon.height) / 2).floor
158+
159+ spritesheet.compose!(icon, x, y)
160+ end
161+
162+ FileUtils.mkdir_p(File.dirname(output_path))
163+ spritesheet.save(output_path)
164+ end
165+
166+ def save_manifest(items, icons_per_row, path)
167+ logger.info("Saving spritesheet details to #{path}")
168+
169+ FileUtils.mkdir_p(File.dirname(path))
170+
171+ # Only save the details that the scss file needs
172+ manifest_items = items.map do |item|
173+ {
174+ :type => item[:type],
175+ :row => item[:row],
176+ :col => item[:col],
177+ :dark_icon_fix => item[:dark_icon_fix]
178+ }
179+ end
180+
181+ manifest = {:icons_per_row => icons_per_row, :items => manifest_items}
182+
183+ File.open(path, 'w') do |f|
184+ f.write(JSON.generate(manifest))
185+ end
186+ end
187+
188+ def log_details(items_with_icons, icons_per_row)
189+ logger.debug("Amount of icons: #{items_with_icons.length}")
190+ logger.debug("Amount of icons needing the dark icon fix: #{items_with_icons.count {|item| item[:dark_icon_fix]}}")
191+ logger.debug("Amount of icons per row: #{icons_per_row}")
192+
193+ max_type_length = items_with_icons.map {|item| item[:type].length}.max
194+ border = "+#{'-' * (max_type_length + 2)}+#{'-' * 5}+#{'-' * 8}+#{'-' * 15}+"
195+
196+ logger.debug(border)
197+ logger.debug("| #{'Type'.ljust(max_type_length)} | Row | Column | Dark icon fix |")
198+ logger.debug(border)
199+
200+ items_with_icons.each do |item|
201+ logger.debug("| #{item[:type].ljust(max_type_length)} | #{item[:row].to_s.ljust(3)} | #{item[:col].to_s.ljust(6)} | #{(item[:dark_icon_fix] ? 'Yes' : 'No').ljust(13)} |")
202+ end
203+
204+ logger.debug(border)
205+ end
206+
207+ def logger
208+ @logger ||= Logger.new($stdout).tap do |logger|
209+ logger.level = options[:verbose] ? Logger::DEBUG : Logger::INFO
210+ logger.formatter = proc {|severity, datetime, progname, msg| "#{msg}\n"}
211+ end
212+ end
213+end
lib/tasks/updates.thor+315−0
@@ -0,0 +1,315 @@
1+class UpdatesCLI < Thor
2+ # The GitHub user that is allowed to upload reports
3+ UPLOAD_USER = 'devdocs-bot'
4+
5+ # The repository to create an issue in when uploading the results
6+ UPLOAD_REPO = 'freeCodeCamp/devdocs'
7+
8+ def self.to_s
9+ 'Updates'
10+ end
11+
12+ def initialize(*args)
13+ require 'docs'
14+ require 'progress_bar'
15+ require 'terminal-table'
16+ require 'date'
17+ super
18+ end
19+
20+ desc 'check [--github-token] [--upload] [--verbose] [doc]...', 'Check for outdated documentations'
21+ option :github_token, :type => :string
22+ option :upload, :type => :boolean
23+ option :verbose, :type => :boolean
24+ def check(*names)
25+ # Convert names to a list of Scraper instances
26+ # Versions are omitted, if v10 is outdated than v8 is aswell
27+ docs = names.map {|name| Docs.find(name.split(/@|~/)[0], false)}.uniq
28+
29+ # Check all documentations for updates when no arguments are given
30+ docs = Docs.all if docs.empty?
31+
32+ opts = {
33+ logger: logger
34+ }
35+
36+ if options.key?(:github_token)
37+ opts[:github_token] = options[:github_token]
38+ end
39+
40+ with_progress_bar do |bar|
41+ bar.max = docs.length
42+ bar.write
43+ end
44+
45+ results = docs.map do |doc|
46+ result = check_doc(doc, opts)
47+ with_progress_bar(&:increment!)
48+ result
49+ end
50+
51+ process_results(results)
52+ rescue Docs::DocNotFound => error
53+ logger.error(error)
54+ logger.info('Run "thor docs:list" to see the list of docs.')
55+ end
56+
57+ private
58+
59+ def check_doc(doc, opts)
60+ logger.debug("Checking #{doc.name}")
61+
62+ instance = doc.versions.first.new
63+ scraper_version = instance.get_scraper_version(opts)
64+ latest_version = instance.get_latest_version(opts)
65+
66+ {
67+ name: doc.name,
68+ scraper_version: format_version(scraper_version),
69+ latest_version: format_version(latest_version),
70+ is_outdated: instance.is_outdated(scraper_version, latest_version)
71+ }
72+ rescue NotImplementedError
73+ logger.warn("Couldn't check #{doc.name}, get_latest_version is not implemented")
74+ error_result(doc, '`get_latest_version` is not implemented')
75+ rescue => error
76+ logger.error("Error while checking #{doc.name}\n#{error.full_message.strip}")
77+ error_result(doc, error.message.gsub(/'/, '`'))
78+ end
79+
80+ def format_version(version)
81+ str = version.to_s
82+
83+ # If the version is numeric and greater than or equal to 1e9 it's probably a timestamp
84+ return str if str.match(/^(\d)+$/).nil? or str.to_i < 1e9
85+
86+ DateTime.strptime(str, '%s').strftime('%F')
87+ end
88+
89+ def error_result(doc, reason)
90+ {
91+ name: doc.name,
92+ error: reason
93+ }
94+ end
95+
96+ def process_results(results)
97+ successful_results = results.select {|result| result.key?(:is_outdated)}
98+ failed_results = results.select {|result| result.key?(:error)}
99+
100+ up_to_date_results = successful_results.select {|result| !result[:is_outdated]}
101+ outdated_results = successful_results.select {|result| result[:is_outdated]}
102+
103+ log_results(outdated_results, up_to_date_results, failed_results)
104+ upload_results(outdated_results, up_to_date_results, failed_results) if options[:upload]
105+ end
106+
107+ #
108+ # Result logging methods
109+ #
110+
111+ def log_results(outdated_results, up_to_date_results, failed_results)
112+ log_failed_results(failed_results) unless failed_results.empty?
113+ log_successful_results('Up-to-date', up_to_date_results) unless up_to_date_results.empty?
114+ log_successful_results('Outdated', outdated_results) unless outdated_results.empty?
115+ end
116+
117+ def log_successful_results(label, results)
118+ title = "#{label} documentations (#{results.length})"
119+ headings = ['Documentation', 'Scraper version', 'Latest version']
120+ rows = results.map {|result| [result[:name], result[:scraper_version], result[:latest_version]]}
121+
122+ table = Terminal::Table.new :title => title, :headings => headings, :rows => rows
123+ puts table
124+ end
125+
126+ def log_failed_results(results)
127+ title = "Documentations that could not be checked (#{results.length})"
128+ headings = %w(Documentation Reason)
129+ rows = results.map {|result| [result[:name], result[:error]]}
130+
131+ table = Terminal::Table.new :title => title, :headings => headings, :rows => rows
132+ puts table
133+ end
134+
135+ #
136+ # Upload methods
137+ #
138+
139+ def upload_results(outdated_results, up_to_date_results, failed_results)
140+ # We can't create issues without a GitHub token
141+ unless options.key?(:github_token)
142+ logger.error("Please specify a GitHub token with the public_repo permission for #{UPLOAD_USER} with the --github-token parameter")
143+ return
144+ end
145+
146+ logger.info('Uploading the results to a new GitHub issue')
147+
148+ logger.info('Checking if the GitHub token belongs to the correct user')
149+ user = github_get('/user')
150+
151+ # Only allow the DevDocs bot to upload reports
152+ unless user['login'] == UPLOAD_USER
153+ logger.error("Only #{UPLOAD_USER} is supposed to upload the results to a new issue. The specified github token is not for #{UPLOAD_USER}.")
154+ return
155+ end
156+
157+ logger.info('Creating a new GitHub issue')
158+
159+ issue = results_to_issue(outdated_results, up_to_date_results, failed_results)
160+ created_issue = github_post("/repos/#{UPLOAD_REPO}/issues", issue)
161+
162+ logger.info('Checking if the previous issue is still open')
163+
164+ search_params = {
165+ q: "Documentation versions report in:title author:#{UPLOAD_USER} is:issue repo:#{UPLOAD_REPO}",
166+ sort: 'created',
167+ order: 'desc'
168+ }
169+
170+ matching_issues = github_get('/search/issues', search_params)
171+ previous_issue = matching_issues['items'].find {|item| item['number'] != created_issue['number']}
172+
173+ if previous_issue.nil?
174+ logger.info('No previous issue found')
175+ log_upload_success(created_issue)
176+ else
177+ logger.info('Commenting on the previous issue')
178+
179+ comment = "This report was superseded by ##{created_issue['number']}."
180+ github_post("/repos/#{UPLOAD_REPO}/issues/#{previous_issue['number']}/comments", {body: comment})
181+ if previous_issue['closed_at'].nil?
182+ logger.info('Closing the previous issue')
183+ github_patch("/repos/#{UPLOAD_REPO}/issues/#{previous_issue['number']}", {state: 'closed'})
184+ log_upload_success(created_issue)
185+ else
186+ logger.info('The previous issue has already been closed')
187+ log_upload_success(created_issue)
188+ end
189+ end
190+ end
191+
192+ def results_to_issue(outdated_results, up_to_date_results, failed_results)
193+ results = [
194+ successful_results_to_markdown('Outdated', outdated_results),
195+ successful_results_to_markdown('Up-to-date', up_to_date_results),
196+ failed_results_to_markdown(failed_results)
197+ ]
198+
199+ results_str = results.select {|result| !result.nil?}.join("\n\n")
200+ travis_str = ENV['TRAVIS'].nil? ? '' : "\n\nThis issue was created by Travis CI build [##{ENV['TRAVIS_BUILD_NUMBER']}](#{ENV['TRAVIS_BUILD_WEB_URL']})."
201+
202+ title = "Documentation versions report for #{Date.today.strftime('%B %Y')}"
203+ body = <<-MARKDOWN
204+## What is this?
205+
206+This is an automatically created issue which contains information about the version status of the documentations available on DevDocs. The results of this report can be used by maintainers when updating outdated documentations.
207+
208+Maintainers can close this issue when all documentations are up-to-date. The issue is also automatically closed when the next report is created.#{travis_str}
209+
210+## Results
211+
212+The #{outdated_results.length + up_to_date_results.length + failed_results.length} documentations are divided as follows:
213+- #{outdated_results.length} that #{outdated_results.length == 1 ? 'is' : 'are'} outdated
214+- #{up_to_date_results.length} that #{up_to_date_results.length == 1 ? 'is' : 'are'} up-to-date (patch updates are ignored)
215+- #{failed_results.length} that could not be checked
216+ MARKDOWN
217+
218+ {
219+ title: title,
220+ body: body.strip + "\n\n" + results_str
221+ }
222+ end
223+
224+ def successful_results_to_markdown(label, results)
225+ return nil if results.empty?
226+
227+ title = "#{label} documentations (#{results.length})"
228+ headings = ['Documentation', 'Scraper version', 'Latest version']
229+ rows = results.map {|result| [result[:name], result[:scraper_version], result[:latest_version]]}
230+
231+ results_to_markdown(title, headings, rows)
232+ end
233+
234+ def failed_results_to_markdown(results)
235+ return nil if results.empty?
236+
237+ title = "Documentations that could not be checked (#{results.length})"
238+ headings = %w(Documentation Reason)
239+ rows = results.map {|result| [result[:name], result[:error]]}
240+
241+ results_to_markdown(title, headings, rows)
242+ end
243+
244+ def results_to_markdown(title, headings, rows)
245+ "<details>\n<summary>#{title}</summary>\n\n#{create_markdown_table(headings, rows)}\n</details>"
246+ end
247+
248+ def create_markdown_table(headings, rows)
249+ header = headings.join(' | ')
250+ separator = '-|' * headings.length
251+ body = rows.map {|row| row.join(' | ')}
252+
253+ header + "\n" + separator[0...-1] + "\n" + body.join("\n")
254+ end
255+
256+ def log_upload_success(created_issue)
257+ logger.info("Successfully uploaded the results to #{created_issue['html_url']}")
258+ end
259+
260+ #
261+ # HTTP utilities
262+ #
263+
264+ def github_get(endpoint, params = {})
265+ github_request(endpoint, {method: :get, params: params})
266+ end
267+
268+ def github_post(endpoint, params)
269+ github_request(endpoint, {method: :post, body: params.to_json})
270+ end
271+
272+ def github_patch(endpoint, params)
273+ github_request(endpoint, {method: :patch, body: params.to_json})
274+ end
275+
276+ def github_request(endpoint, opts)
277+ url = "https://api.github.com#{endpoint}"
278+
279+ # GitHub token authentication
280+ opts[:headers] = {
281+ Authorization: "token #{options[:github_token]}"
282+ }
283+
284+ # GitHub requires the Content-Type to be application/json when a body is passed
285+ if opts.key?(:body)
286+ opts[:headers]['Content-Type'] = 'application/json'
287+ end
288+
289+ logger.debug("Making a #{opts[:method]} request to #{url}")
290+ response = Docs::Request.run(url, opts)
291+
292+ # response.success? is false if the response code is 201
293+ # GitHub returns 201 Created after an issue is created
294+ if response.success? || response.code == 201
295+ JSON.parse(response.body)
296+ else
297+ logger.error("Couldn't make a #{opts[:method]} request to #{url} (response code #{response.code})")
298+ nil
299+ end
300+ end
301+
302+ # A utility method which ensures no progress bar is shown when stdout is not a tty
303+ def with_progress_bar(&block)
304+ return unless $stdout.tty?
305+ @progress_bar ||= ::ProgressBar.new
306+ block.call @progress_bar
307+ end
308+
309+ def logger
310+ @logger ||= Logger.new($stdout).tap do |logger|
311+ logger.level = options[:verbose] ? Logger::DEBUG : Logger::INFO
312+ logger.formatter = proc {|severity, datetime, progname, msg| "[#{severity}] #{msg}\n"}
313+ end
314+ end
315+end