Add better logging and get_latest_version implementations for 10 scrapers

Jasper van Merle committed 7 years ago

Commit 2d1e8aa1 parent d0300dd13 files · +128 −21Browse files

Showing the changes to lib/docs (12 of 13 files). Show all

Unified · Split

12 files

@@ -132,7 +132,7 @@ module Docs · 131 unmodified lines
132	end	132	end
133	end	133	end
134		134
135	def get_latest_version	135	def get_latest_version(&block)
136	raise NotImplementedError	136	raise NotImplementedError
137	end	137	end
138		138
@@ -231,6 +231,38 @@ module Docs · 92 unmodified lines
231	{}	231	{}
232	end	232	end
233		233
		234	# Utility methods for get_latest_version
		235
		236	def fetch(url, &block)
		237	Request.run(url) do \|response\|
		238	if response.success?
		239	block.call response.body
		240	else
		241	block.call nil
		242	end
		243	end
		244	end
		245
		246	def fetch_doc(url, &block)
		247	fetch(url) do \|body\|
		248	parser = Parser.new(body)
		249	block.call parser.html
		250	end
		251	end
		252
		253	def fetch_json(url, &block)
		254	fetch(url) do \|body\|
		255	json = JSON.parse(body)
		256	block.call json
		257	end
		258	end
		259
		260	def get_npm_version(package, &block)
		261	fetch_json("https://registry.npmjs.com/#{package}") do \|json\|
		262	block.call json['dist-tags']['latest']
		263	end
		264	end
		265
234	module FixInternalUrlsBehavior	266	module FixInternalUrlsBehavior
235	def self.included(base)	267	def self.included(base)
236	base.extend ClassMethods	268	base.extend ClassMethods

@@ -155,6 +155,10 @@ module Docs · 154 unmodified lines
155	end	155	end
156	end	156	end
157		157
		158	def get_latest_version(&block)
		159	get_npm_version('@angular/core', &block)
		160	end
		161
158	private	162	private
159		163
160	def parse(response)	164	def parse(response)

       self.release = '1.2.32'
       self.base_url = "https://code.angularjs.org/#{release}/docs/partials/"
     end
+    def get_latest_version(&block)
+      get_npm_version('angular', &block)
+    end
   end
 end

@@ -87,5 +87,11 @@ module Docs · 86 unmodified lines
87	quickstart.html	87	quickstart.html
88	list_of_all_modules.html)	88	list_of_all_modules.html)
89	end	89	end
		90
		91	def get_latest_version(&block)
		92	fetch_doc('https://docs.ansible.com/ansible/latest/index.html') do \|doc\|
		93	block.call doc.at_css('.DocSiteProduct-CurrentVersion').content.strip
		94	end
		95	end
90	end	96	end
91	end	97	end

@@ -33,5 +33,11 @@ module Docs · 32 unmodified lines
33	© 2018 The Apache Software Foundation<br>	33	© 2018 The Apache Software Foundation<br>
34	Licensed under the Apache License, Version 2.0.	34	Licensed under the Apache License, Version 2.0.
35	HTML	35	HTML
		36
		37	def get_latest_version(&block)
		38	fetch_doc('http://httpd.apache.org/docs/') do \|doc\|
		39	block.call doc.at_css('#apcontents > ul a')['href'][0...-1]
		40	end
		41	end
36	end	42	end
37	end	43	end

@@ -43,5 +43,11 @@ module Docs · 42 unmodified lines
43	self.base_url = "https://pig.apache.org/docs/r#{release}/"	43	self.base_url = "https://pig.apache.org/docs/r#{release}/"
44	end	44	end
45		45
		46	def get_latest_version(&block)
		47	fetch_doc('https://pig.apache.org/') do \|doc\|
		48	item = doc.at_css('div[id="menu_1.2"] > .menuitem:last-child')
		49	block.call item.content.strip.sub(/Release /, '')
		50	end
		51	end
46	end	52	end
47	end	53	end

@@ -17,5 +17,12 @@ module Docs · 16 unmodified lines
17	© 2010–2018 Caolan McMahon<br>	17	© 2010–2018 Caolan McMahon<br>
18	Licensed under the MIT License.	18	Licensed under the MIT License.
19	HTML	19	HTML
		20
		21	def get_latest_version(&block)
		22	fetch_doc('https://caolan.github.io/async/') do \|doc\|
		23	version = doc.at_css('#version-dropdown > a').content.strip[1..-1]
		24	block.call version
		25	end
		26	end
20	end	27	end
21	end	28	end

@@ -22,5 +22,11 @@ module Docs · 21 unmodified lines
22	stub '' do	22	stub '' do
23	'<div></div>'	23	'<div></div>'
24	end	24	end
		25
		26	def get_latest_version(&block)
		27	fetch_doc('https://babeljs.io/docs/en/') do \|doc\|
		28	block.call doc.at_css('a[href="/versions"] > h3').content
		29	end
		30	end
25	end	31	end
26	end	32	end

@@ -20,5 +20,12 @@ module Docs · 19 unmodified lines
20	© 2010–2016 Jeremy Ashkenas, DocumentCloud<br>	20	© 2010–2016 Jeremy Ashkenas, DocumentCloud<br>
21	Licensed under the MIT License.	21	Licensed under the MIT License.
22	HTML	22	HTML
		23
		24	def get_latest_version(&block)
		25	fetch_doc('https://backbonejs.org/') do \|doc\|
		26	version = doc.at_css('.version').content
		27	block.call version[1...-1]
		28	end
		29	end
23	end	30	end
24	end	31	end

@@ -17,5 +17,12 @@ module Docs · 16 unmodified lines
17	Copyright © 2000, 2001, 2002, 2007, 2008 Free Software Foundation, Inc.<br>	17	Copyright © 2000, 2001, 2002, 2007, 2008 Free Software Foundation, Inc.<br>
18	Licensed under the GNU Free Documentation License.	18	Licensed under the GNU Free Documentation License.
19	HTML	19	HTML
		20
		21	def get_latest_version(&block)
		22	fetch('https://www.gnu.org/software/bash/manual/html_node/index.html') do \|body\|
		23	version = body.scan(/, Version ([0-9.]+)/)[0][0]
		24	block.call version[0...-1]
		25	end
		26	end
20	end	27	end
21	end	28	end

       &copy; 2013&ndash;2017 Petka Antonov<br>
       Licensed under the MIT License.
     HTML
+    def get_latest_version(&block)
+      get_npm_version('bluebird', &block)
+    end
   end
 end

@@ -2,6 +2,7 @@ module Docs · 1 unmodified line
2	class Pygame < UrlScraper	2	class Pygame < UrlScraper
3	self.type = 'simple'	3	self.type = 'simple'
4	self.release = '1.9.4'	4	self.release = '1.9.4'
		5	self.base_url = 'https://www.pygame.org/docs/'
5	self.root_path = 'py-modindex.html'	6	self.root_path = 'py-modindex.html'
6	self.links = {	7	self.links = {
7	home: 'https://www.pygame.org/',	8	home: 'https://www.pygame.org/',