History

Add better logging and get_latest_version implementations for 10 scrapers

Jasper van Merle committed 7 years ago

Commit 2d1e8aa1 parent d0300dd13 files · +128 −21Browse files

Unified · Split

13 files
  1. lib/docs/core/scraper.rb +33 −1
  2. lib/docs/scrapers/angular.rb +4 −0
  3. lib/docs/scrapers/angularjs.rb +4 −0
  4. lib/docs/scrapers/ansible.rb +6 −0
  5. lib/docs/scrapers/apache.rb +6 −0
  6. lib/docs/scrapers/apache_pig.rb +6 −0
  7. lib/docs/scrapers/async.rb +7 −0
  8. lib/docs/scrapers/babel.rb +6 −0
  9. lib/docs/scrapers/backbone.rb +7 −0
  10. lib/docs/scrapers/bash.rb +7 −0
  11. lib/docs/scrapers/bluebird.rb +4 −0
  12. lib/docs/scrapers/pygame.rb +1 −0
  13. lib/tasks/updates.thor +37 −20
lib/docs/core/scraper.rb+33−1
@@ -132,7 +132,7 @@ module Docs · 131 unmodified lines
132132 end
133133 end
134134
135− def get_latest_version
135+ def get_latest_version(&block)
136136 raise NotImplementedError
137137 end
138138
@@ -231,6 +231,38 @@ module Docs · 92 unmodified lines
231231 {}
232232 end
233233
234+ # Utility methods for get_latest_version
235+
236+ def fetch(url, &block)
237+ Request.run(url) do |response|
238+ if response.success?
239+ block.call response.body
240+ else
241+ block.call nil
242+ end
243+ end
244+ end
245+
246+ def fetch_doc(url, &block)
247+ fetch(url) do |body|
248+ parser = Parser.new(body)
249+ block.call parser.html
250+ end
251+ end
252+
253+ def fetch_json(url, &block)
254+ fetch(url) do |body|
255+ json = JSON.parse(body)
256+ block.call json
257+ end
258+ end
259+
260+ def get_npm_version(package, &block)
261+ fetch_json("https://registry.npmjs.com/#{package}") do |json|
262+ block.call json['dist-tags']['latest']
263+ end
264+ end
265+
234266 module FixInternalUrlsBehavior
235267 def self.included(base)
236268 base.extend ClassMethods
lib/docs/scrapers/angular.rb+4−0
@@ -155,6 +155,10 @@ module Docs · 154 unmodified lines
155155 end
156156 end
157157
158+ def get_latest_version(&block)
159+ get_npm_version('@angular/core', &block)
160+ end
161+
158162 private
159163
160164 def parse(response)
lib/docs/scrapers/angularjs.rb+4−0
@@ -69,5 +69,9 @@ module Docs · 68 unmodified lines
6969 self.release = '1.2.32'
7070 self.base_url = "https://code.angularjs.org/#{release}/docs/partials/"
7171 end
72+
73+ def get_latest_version(&block)
74+ get_npm_version('angular', &block)
75+ end
7276 end
7377 end
lib/docs/scrapers/ansible.rb+6−0
@@ -87,5 +87,11 @@ module Docs · 86 unmodified lines
8787 quickstart.html
8888 list_of_all_modules.html)
8989 end
90+
91+ def get_latest_version(&block)
92+ fetch_doc('https://docs.ansible.com/ansible/latest/index.html') do |doc|
93+ block.call doc.at_css('.DocSiteProduct-CurrentVersion').content.strip
94+ end
95+ end
9096 end
9197 end
lib/docs/scrapers/apache.rb+6−0
@@ -33,5 +33,11 @@ module Docs · 32 unmodified lines
3333 &copy; 2018 The Apache Software Foundation<br>
3434 Licensed under the Apache License, Version 2.0.
3535 HTML
36+
37+ def get_latest_version(&block)
38+ fetch_doc('http://httpd.apache.org/docs/') do |doc|
39+ block.call doc.at_css('#apcontents > ul a')['href'][0...-1]
40+ end
41+ end
3642 end
3743 end
lib/docs/scrapers/apache_pig.rb+6−0
@@ -43,5 +43,11 @@ module Docs · 42 unmodified lines
4343 self.base_url = "https://pig.apache.org/docs/r#{release}/"
4444 end
4545
46+ def get_latest_version(&block)
47+ fetch_doc('https://pig.apache.org/') do |doc|
48+ item = doc.at_css('div[id="menu_1.2"] > .menuitem:last-child')
49+ block.call item.content.strip.sub(/Release /, '')
50+ end
51+ end
4652 end
4753 end
lib/docs/scrapers/async.rb+7−0
@@ -17,5 +17,12 @@ module Docs · 16 unmodified lines
1717 &copy; 2010&ndash;2018 Caolan McMahon<br>
1818 Licensed under the MIT License.
1919 HTML
20+
21+ def get_latest_version(&block)
22+ fetch_doc('https://caolan.github.io/async/') do |doc|
23+ version = doc.at_css('#version-dropdown > a').content.strip[1..-1]
24+ block.call version
25+ end
26+ end
2027 end
2128 end
lib/docs/scrapers/babel.rb+6−0
@@ -22,5 +22,11 @@ module Docs · 21 unmodified lines
2222 stub '' do
2323 '<div></div>'
2424 end
25+
26+ def get_latest_version(&block)
27+ fetch_doc('https://babeljs.io/docs/en/') do |doc|
28+ block.call doc.at_css('a[href="/versions"] > h3').content
29+ end
30+ end
2531 end
2632 end
lib/docs/scrapers/backbone.rb+7−0
@@ -20,5 +20,12 @@ module Docs · 19 unmodified lines
2020 &copy; 2010&ndash;2016 Jeremy Ashkenas, DocumentCloud<br>
2121 Licensed under the MIT License.
2222 HTML
23+
24+ def get_latest_version(&block)
25+ fetch_doc('https://backbonejs.org/') do |doc|
26+ version = doc.at_css('.version').content
27+ block.call version[1...-1]
28+ end
29+ end
2330 end
2431 end
lib/docs/scrapers/bash.rb+7−0
@@ -17,5 +17,12 @@ module Docs · 16 unmodified lines
1717 Copyright &copy; 2000, 2001, 2002, 2007, 2008 Free Software Foundation, Inc.<br>
1818 Licensed under the GNU Free Documentation License.
1919 HTML
20+
21+ def get_latest_version(&block)
22+ fetch('https://www.gnu.org/software/bash/manual/html_node/index.html') do |body|
23+ version = body.scan(/, Version ([0-9.]+)/)[0][0]
24+ block.call version[0...-1]
25+ end
26+ end
2027 end
2128 end
lib/docs/scrapers/bluebird.rb+4−0
@@ -18,5 +18,9 @@ module Docs · 17 unmodified lines
1818 &copy; 2013&ndash;2017 Petka Antonov<br>
1919 Licensed under the MIT License.
2020 HTML
21+
22+ def get_latest_version(&block)
23+ get_npm_version('bluebird', &block)
24+ end
2125 end
2226 end
lib/docs/scrapers/pygame.rb+1−0
@@ -2,6 +2,7 @@ module Docs · 1 unmodified line
22 class Pygame < UrlScraper
33 self.type = 'simple'
44 self.release = '1.9.4'
5+ self.base_url = 'https://www.pygame.org/docs/'
56 self.root_path = 'py-modindex.html'
67 self.links = {
78 home: 'https://www.pygame.org/',
lib/tasks/updates.thor+37−20
@@ -9,7 +9,8 @@ class UpdatesCLI < Thor · 8 unmodified lines
99 super
1010 end
1111
12− desc 'check [doc]...', 'Check for outdated documentations'
12+ desc 'check [--verbose] [doc]...', 'Check for outdated documentations'
13+ option :verbose, :type => :boolean
1314 def check(*names)
1415 # Convert names to a list of Scraper instances
1516 # Versions are omitted, if v10 is outdated than v8 is aswell
@@ -27,13 +28,14 @@ class UpdatesCLI < Thor · 11 unmodified lines
2728 result
2829 end
2930
30− outdated = results.select {|result| result.is_a?(Hash) && result[:is_outdated]}
31− return if outdated.empty?
31+ valid_results = results.select {|result| result.is_a?(Hash)}
3232
33− logger.info("Outdated documentations (#{outdated.length}):")
34− outdated.each do |result|
35− logger.info("#{result[:name]}: #{result[:current_version]} -> #{result[:latest_version]}")
36− end
33+ up_to_date_results = valid_results.select {|result| !result[:is_outdated]}
34+ outdated_results = valid_results.select {|result| result[:is_outdated]}
35+
36+ log_results('Up-to-date', up_to_date_results) if options[:verbose] and !up_to_date_results.empty?
37+ logger.info("") if options[:verbose] and !up_to_date_results.empty? and !outdated_results.empty?
38+ log_results('Outdated', outdated_results) unless outdated_results.empty?
3739 rescue Docs::DocNotFound => error
3840 logger.error(error)
3941 logger.info('Run "thor docs:list" to see the list of docs.')
@@ -42,33 +44,48 @@ class UpdatesCLI < Thor · 2 unmodified lines
4244 private
4345
4446 def check_doc(doc)
45− # Scraper versions are always sorted from new to old
47+ # Newer scraper versions always come before older scraper versions
4648 # Therefore, the first item's release value is the latest current scraper version
4749 #
4850 # For example, a scraper could scrape 3 versions: 10, 11 and 12
49− # doc.versions.first would be the scraper for version 12 if the scraper is written like all the other scrapers are
51+ # doc.versions.first would be the scraper for version 12
5052 instance = doc.versions.first.new
5153
54+ return nil unless instance.class.method_defined?(:options)
55+
5256 current_version = instance.options[:release]
5357 return nil if current_version.nil?
5458
55− latest_version = instance.get_latest_version
56− return nil if latest_version.nil?
59+ logger.debug("Checking #{doc.name}")
60+
61+ instance.get_latest_version do |latest_version|
62+ return {
63+ name: doc.name,
64+ current_version: current_version,
65+ latest_version: latest_version,
66+ is_outdated: instance.is_outdated(current_version, latest_version)
67+ }
68+ end
5769
58− {
59− name: doc.name,
60− current_version: current_version,
61− latest_version: latest_version,
62− is_outdated: instance.is_outdated(current_version, latest_version)
63− }
70+ return nil
6471 rescue NotImplementedError
65− logger.warn("Can't check #{doc.name}, get_latest_version is not implemented")
66− rescue => error
67− logger.error("Error while checking #{doc.name}: #{error}")
72+ logger.warn("Couldn't check #{doc.name}, get_latest_version is not implemented")
73+ rescue
74+ logger.error("Error while checking #{doc.name}")
75+ raise
76+ end
77+
78+ def log_results(label, results)
79+ logger.info("#{label} documentations (#{results.length}):")
80+
81+ results.each do |result|
82+ logger.info("#{result[:name]}: #{result[:current_version]} -> #{result[:latest_version]}")
83+ end
6884 end
6985
7086 def logger
7187 @logger ||= Logger.new($stdout).tap do |logger|
88+ logger.level = options[:verbose] ? Logger::DEBUG : Logger::INFO
7289 logger.formatter = proc do |severity, datetime, progname, msg|
7390 prefix = severity != "INFO" ? "[#{severity}] " : ""
7491 "#{prefix}#{msg}\n"