History

Add better logging and get_latest_version implementations for 10 scrapers

Jasper van Merle committed 7 years ago

Commit 2d1e8aa1 parent d0300dd13 files · +128 −21Browse files

Unified · Split

13 files
  1. lib/docs/core/scraper.rb +33 −1
  2. lib/docs/scrapers/angular.rb +4 −0
  3. lib/docs/scrapers/angularjs.rb +4 −0
  4. lib/docs/scrapers/ansible.rb +6 −0
  5. lib/docs/scrapers/apache.rb +6 −0
  6. lib/docs/scrapers/apache_pig.rb +6 −0
  7. lib/docs/scrapers/async.rb +7 −0
  8. lib/docs/scrapers/babel.rb +6 −0
  9. lib/docs/scrapers/backbone.rb +7 −0
  10. lib/docs/scrapers/bash.rb +7 −0
  11. lib/docs/scrapers/bluebird.rb +4 −0
  12. lib/docs/scrapers/pygame.rb +1 −0
  13. lib/tasks/updates.thor +37 −20
lib/docs/core/scraper.rb+33−1
@@ -132,7 +132,7 @@ module Docs · 131 unmodified lines
132 end132 end
133 end133 end
134 134
135 def get_latest_version135 def get_latest_version(&block)
136 raise NotImplementedError136 raise NotImplementedError
137 end137 end
138 138
@@ -231,6 +231,38 @@ module Docs · 92 unmodified lines
231 {}231 {}
232 end232 end
233 233
234 # Utility methods for get_latest_version
235
236 def fetch(url, &block)
237 Request.run(url) do |response|
238 if response.success?
239 block.call response.body
240 else
241 block.call nil
242 end
243 end
244 end
245
246 def fetch_doc(url, &block)
247 fetch(url) do |body|
248 parser = Parser.new(body)
249 block.call parser.html
250 end
251 end
252
253 def fetch_json(url, &block)
254 fetch(url) do |body|
255 json = JSON.parse(body)
256 block.call json
257 end
258 end
259
260 def get_npm_version(package, &block)
261 fetch_json("https://registry.npmjs.com/#{package}") do |json|
262 block.call json['dist-tags']['latest']
263 end
264 end
265
234 module FixInternalUrlsBehavior266 module FixInternalUrlsBehavior
235 def self.included(base)267 def self.included(base)
236 base.extend ClassMethods268 base.extend ClassMethods
lib/docs/scrapers/angular.rb+4−0
@@ -155,6 +155,10 @@ module Docs · 154 unmodified lines
155 end155 end
156 end156 end
157 157
158 def get_latest_version(&block)
159 get_npm_version('@angular/core', &block)
160 end
161
158 private162 private
159 163
160 def parse(response)164 def parse(response)
lib/docs/scrapers/angularjs.rb+4−0
@@ -69,5 +69,9 @@ module Docs · 68 unmodified lines
69 self.release = '1.2.32'69 self.release = '1.2.32'
70 self.base_url = "https://code.angularjs.org/#{release}/docs/partials/"70 self.base_url = "https://code.angularjs.org/#{release}/docs/partials/"
71 end71 end
72
73 def get_latest_version(&block)
74 get_npm_version('angular', &block)
75 end
72 end76 end
73end77end
lib/docs/scrapers/ansible.rb+6−0
@@ -87,5 +87,11 @@ module Docs · 86 unmodified lines
87 quickstart.html87 quickstart.html
88 list_of_all_modules.html)88 list_of_all_modules.html)
89 end89 end
90
91 def get_latest_version(&block)
92 fetch_doc('https://docs.ansible.com/ansible/latest/index.html') do |doc|
93 block.call doc.at_css('.DocSiteProduct-CurrentVersion').content.strip
94 end
95 end
90 end96 end
91end97end
lib/docs/scrapers/apache.rb+6−0
@@ -33,5 +33,11 @@ module Docs · 32 unmodified lines
33 &copy; 2018 The Apache Software Foundation<br>33 &copy; 2018 The Apache Software Foundation<br>
34 Licensed under the Apache License, Version 2.0.34 Licensed under the Apache License, Version 2.0.
35 HTML35 HTML
36
37 def get_latest_version(&block)
38 fetch_doc('http://httpd.apache.org/docs/') do |doc|
39 block.call doc.at_css('#apcontents > ul a')['href'][0...-1]
40 end
41 end
36 end42 end
37end43end
lib/docs/scrapers/apache_pig.rb+6−0
@@ -43,5 +43,11 @@ module Docs · 42 unmodified lines
43 self.base_url = "https://pig.apache.org/docs/r#{release}/"43 self.base_url = "https://pig.apache.org/docs/r#{release}/"
44 end44 end
45 45
46 def get_latest_version(&block)
47 fetch_doc('https://pig.apache.org/') do |doc|
48 item = doc.at_css('div[id="menu_1.2"] > .menuitem:last-child')
49 block.call item.content.strip.sub(/Release /, '')
50 end
51 end
46 end52 end
47end53end
lib/docs/scrapers/async.rb+7−0
@@ -17,5 +17,12 @@ module Docs · 16 unmodified lines
17 &copy; 2010&ndash;2018 Caolan McMahon<br>17 &copy; 2010&ndash;2018 Caolan McMahon<br>
18 Licensed under the MIT License.18 Licensed under the MIT License.
19 HTML19 HTML
20
21 def get_latest_version(&block)
22 fetch_doc('https://caolan.github.io/async/') do |doc|
23 version = doc.at_css('#version-dropdown > a').content.strip[1..-1]
24 block.call version
25 end
26 end
20 end27 end
21end28end
lib/docs/scrapers/babel.rb+6−0
@@ -22,5 +22,11 @@ module Docs · 21 unmodified lines
22 stub '' do22 stub '' do
23 '<div></div>'23 '<div></div>'
24 end24 end
25
26 def get_latest_version(&block)
27 fetch_doc('https://babeljs.io/docs/en/') do |doc|
28 block.call doc.at_css('a[href="/versions"] > h3').content
29 end
30 end
25 end31 end
26end32end
lib/docs/scrapers/backbone.rb+7−0
@@ -20,5 +20,12 @@ module Docs · 19 unmodified lines
20 &copy; 2010&ndash;2016 Jeremy Ashkenas, DocumentCloud<br>20 &copy; 2010&ndash;2016 Jeremy Ashkenas, DocumentCloud<br>
21 Licensed under the MIT License.21 Licensed under the MIT License.
22 HTML22 HTML
23
24 def get_latest_version(&block)
25 fetch_doc('https://backbonejs.org/') do |doc|
26 version = doc.at_css('.version').content
27 block.call version[1...-1]
28 end
29 end
23 end30 end
24end31end
lib/docs/scrapers/bash.rb+7−0
@@ -17,5 +17,12 @@ module Docs · 16 unmodified lines
17 Copyright &copy; 2000, 2001, 2002, 2007, 2008 Free Software Foundation, Inc.<br>17 Copyright &copy; 2000, 2001, 2002, 2007, 2008 Free Software Foundation, Inc.<br>
18 Licensed under the GNU Free Documentation License.18 Licensed under the GNU Free Documentation License.
19 HTML19 HTML
20
21 def get_latest_version(&block)
22 fetch('https://www.gnu.org/software/bash/manual/html_node/index.html') do |body|
23 version = body.scan(/, Version ([0-9.]+)/)[0][0]
24 block.call version[0...-1]
25 end
26 end
20 end27 end
21end28end
lib/docs/scrapers/bluebird.rb+4−0
@@ -18,5 +18,9 @@ module Docs · 17 unmodified lines
18 &copy; 2013&ndash;2017 Petka Antonov<br>18 &copy; 2013&ndash;2017 Petka Antonov<br>
19 Licensed under the MIT License.19 Licensed under the MIT License.
20 HTML20 HTML
21
22 def get_latest_version(&block)
23 get_npm_version('bluebird', &block)
24 end
21 end25 end
22end26end
lib/docs/scrapers/pygame.rb+1−0
@@ -2,6 +2,7 @@ module Docs · 1 unmodified line
2 class Pygame < UrlScraper2 class Pygame < UrlScraper
3 self.type = 'simple'3 self.type = 'simple'
4 self.release = '1.9.4'4 self.release = '1.9.4'
5 self.base_url = 'https://www.pygame.org/docs/'
5 self.root_path = 'py-modindex.html'6 self.root_path = 'py-modindex.html'
6 self.links = {7 self.links = {
7 home: 'https://www.pygame.org/',8 home: 'https://www.pygame.org/',
lib/tasks/updates.thor+37−20
@@ -9,7 +9,8 @@ class UpdatesCLI < Thor · 8 unmodified lines
9 super9 super
10 end10 end
11 11
12 desc 'check [doc]...', 'Check for outdated documentations'12 desc 'check [--verbose] [doc]...', 'Check for outdated documentations'
13 option :verbose, :type => :boolean
13 def check(*names)14 def check(*names)
14 # Convert names to a list of Scraper instances15 # Convert names to a list of Scraper instances
15 # Versions are omitted, if v10 is outdated than v8 is aswell16 # Versions are omitted, if v10 is outdated than v8 is aswell
@@ -27,13 +28,14 @@ class UpdatesCLI < Thor · 11 unmodified lines
27 result28 result
28 end29 end
29 30
30 outdated = results.select {|result| result.is_a?(Hash) && result[:is_outdated]}31 valid_results = results.select {|result| result.is_a?(Hash)}
31 return if outdated.empty?
32 32
33 logger.info("Outdated documentations (#{outdated.length}):")33 up_to_date_results = valid_results.select {|result| !result[:is_outdated]}
34 outdated.each do |result|34 outdated_results = valid_results.select {|result| result[:is_outdated]}
35 logger.info("#{result[:name]}: #{result[:current_version]} -> #{result[:latest_version]}")35
36 end36 log_results('Up-to-date', up_to_date_results) if options[:verbose] and !up_to_date_results.empty?
37 logger.info("") if options[:verbose] and !up_to_date_results.empty? and !outdated_results.empty?
38 log_results('Outdated', outdated_results) unless outdated_results.empty?
37 rescue Docs::DocNotFound => error39 rescue Docs::DocNotFound => error
38 logger.error(error)40 logger.error(error)
39 logger.info('Run "thor docs:list" to see the list of docs.')41 logger.info('Run "thor docs:list" to see the list of docs.')
@@ -42,33 +44,48 @@ class UpdatesCLI < Thor · 2 unmodified lines
42 private44 private
43 45
44 def check_doc(doc)46 def check_doc(doc)
45 # Scraper versions are always sorted from new to old47 # Newer scraper versions always come before older scraper versions
46 # Therefore, the first item's release value is the latest current scraper version48 # Therefore, the first item's release value is the latest current scraper version
47 #49 #
48 # For example, a scraper could scrape 3 versions: 10, 11 and 1250 # For example, a scraper could scrape 3 versions: 10, 11 and 12
49 # doc.versions.first would be the scraper for version 12 if the scraper is written like all the other scrapers are51 # doc.versions.first would be the scraper for version 12
50 instance = doc.versions.first.new52 instance = doc.versions.first.new
51 53
54 return nil unless instance.class.method_defined?(:options)
55
52 current_version = instance.options[:release]56 current_version = instance.options[:release]
53 return nil if current_version.nil?57 return nil if current_version.nil?
54 58
55 latest_version = instance.get_latest_version59 logger.debug("Checking #{doc.name}")
56 return nil if latest_version.nil?60
61 instance.get_latest_version do |latest_version|
62 return {
63 name: doc.name,
64 current_version: current_version,
65 latest_version: latest_version,
66 is_outdated: instance.is_outdated(current_version, latest_version)
67 }
68 end
57 69
58 {70 return nil
59 name: doc.name,
60 current_version: current_version,
61 latest_version: latest_version,
62 is_outdated: instance.is_outdated(current_version, latest_version)
63 }
64 rescue NotImplementedError71 rescue NotImplementedError
65 logger.warn("Can't check #{doc.name}, get_latest_version is not implemented")72 logger.warn("Couldn't check #{doc.name}, get_latest_version is not implemented")
66 rescue => error73 rescue
67 logger.error("Error while checking #{doc.name}: #{error}")74 logger.error("Error while checking #{doc.name}")
75 raise
76 end
77
78 def log_results(label, results)
79 logger.info("#{label} documentations (#{results.length}):")
80
81 results.each do |result|
82 logger.info("#{result[:name]}: #{result[:current_version]} -> #{result[:latest_version]}")
83 end
68 end84 end
69 85
70 def logger86 def logger
71 @logger ||= Logger.new($stdout).tap do |logger|87 @logger ||= Logger.new($stdout).tap do |logger|
88 logger.level = options[:verbose] ? Logger::DEBUG : Logger::INFO
72 logger.formatter = proc do |severity, datetime, progname, msg|89 logger.formatter = proc do |severity, datetime, progname, msg|
73 prefix = severity != "INFO" ? "[#{severity}] " : ""90 prefix = severity != "INFO" ? "[#{severity}] " : ""
74 "#{prefix}#{msg}\n"91 "#{prefix}#{msg}\n"