Add better logging and get_latest_version implementations for 10 scrapers

Jasper van Merle committed 7 years ago

Commit 2d1e8aa1 parent d0300dd13 files · +128 −21Browse files

Showing the changes to lib/docs (12 of 13 files). Show all

Unified · Split

12 files

@@ -132,7 +132,7 @@ module Docs · 131 unmodified lines
132	132	end
133	133	end
134	134
135		− def get_latest_version
	135	+ def get_latest_version(&block)
136	136	raise NotImplementedError
137	137	end
138	138
@@ -231,6 +231,38 @@ module Docs · 92 unmodified lines
231	231	{}
232	232	end
233	233
	234	+ # Utility methods for get_latest_version
	235	+
	236	+ def fetch(url, &block)
	237	+ Request.run(url) do \|response\|
	238	+ if response.success?
	239	+ block.call response.body
	240	+ else
	241	+ block.call nil
	242	+ end
	243	+ end
	244	+ end
	245	+
	246	+ def fetch_doc(url, &block)
	247	+ fetch(url) do \|body\|
	248	+ parser = Parser.new(body)
	249	+ block.call parser.html
	250	+ end
	251	+ end
	252	+
	253	+ def fetch_json(url, &block)
	254	+ fetch(url) do \|body\|
	255	+ json = JSON.parse(body)
	256	+ block.call json
	257	+ end
	258	+ end
	259	+
	260	+ def get_npm_version(package, &block)
	261	+ fetch_json("https://registry.npmjs.com/#{package}") do \|json\|
	262	+ block.call json['dist-tags']['latest']
	263	+ end
	264	+ end
	265	+
234	266	module FixInternalUrlsBehavior
235	267	def self.included(base)
236	268	base.extend ClassMethods

@@ -155,6 +155,10 @@ module Docs · 154 unmodified lines
155	155	end
156	156	end
157	157
	158	+ def get_latest_version(&block)
	159	+ get_npm_version('@angular/core', &block)
	160	+ end
	161	+
158	162	private
159	163
160	164	def parse(response)

@@ -69,5 +69,9 @@ module Docs · 68 unmodified lines
69	69	self.release = '1.2.32'
70	70	self.base_url = "https://code.angularjs.org/#{release}/docs/partials/"
71	71	end
	72	+
	73	+ def get_latest_version(&block)
	74	+ get_npm_version('angular', &block)
	75	+ end
72	76	end
73	77	end

@@ -87,5 +87,11 @@ module Docs · 86 unmodified lines
87	87	quickstart.html
88	88	list_of_all_modules.html)
89	89	end
	90	+
	91	+ def get_latest_version(&block)
	92	+ fetch_doc('https://docs.ansible.com/ansible/latest/index.html') do \|doc\|
	93	+ block.call doc.at_css('.DocSiteProduct-CurrentVersion').content.strip
	94	+ end
	95	+ end
90	96	end
91	97	end

@@ -33,5 +33,11 @@ module Docs · 32 unmodified lines
33	33	© 2018 The Apache Software Foundation<br>
34	34	Licensed under the Apache License, Version 2.0.
35	35	HTML
	36	+
	37	+ def get_latest_version(&block)
	38	+ fetch_doc('http://httpd.apache.org/docs/') do \|doc\|
	39	+ block.call doc.at_css('#apcontents > ul a')['href'][0...-1]
	40	+ end
	41	+ end
36	42	end
37	43	end

@@ -43,5 +43,11 @@ module Docs · 42 unmodified lines
43	43	self.base_url = "https://pig.apache.org/docs/r#{release}/"
44	44	end
45	45
	46	+ def get_latest_version(&block)
	47	+ fetch_doc('https://pig.apache.org/') do \|doc\|
	48	+ item = doc.at_css('div[id="menu_1.2"] > .menuitem:last-child')
	49	+ block.call item.content.strip.sub(/Release /, '')
	50	+ end
	51	+ end
46	52	end
47	53	end

@@ -17,5 +17,12 @@ module Docs · 16 unmodified lines
17	17	© 2010–2018 Caolan McMahon<br>
18	18	Licensed under the MIT License.
19	19	HTML
	20	+
	21	+ def get_latest_version(&block)
	22	+ fetch_doc('https://caolan.github.io/async/') do \|doc\|
	23	+ version = doc.at_css('#version-dropdown > a').content.strip[1..-1]
	24	+ block.call version
	25	+ end
	26	+ end
20	27	end
21	28	end

@@ -22,5 +22,11 @@ module Docs · 21 unmodified lines
22	22	stub '' do
23	23	'<div></div>'
24	24	end
	25	+
	26	+ def get_latest_version(&block)
	27	+ fetch_doc('https://babeljs.io/docs/en/') do \|doc\|
	28	+ block.call doc.at_css('a[href="/versions"] > h3').content
	29	+ end
	30	+ end
25	31	end
26	32	end

@@ -20,5 +20,12 @@ module Docs · 19 unmodified lines
20	20	© 2010–2016 Jeremy Ashkenas, DocumentCloud<br>
21	21	Licensed under the MIT License.
22	22	HTML
	23	+
	24	+ def get_latest_version(&block)
	25	+ fetch_doc('https://backbonejs.org/') do \|doc\|
	26	+ version = doc.at_css('.version').content
	27	+ block.call version[1...-1]
	28	+ end
	29	+ end
23	30	end
24	31	end

@@ -17,5 +17,12 @@ module Docs · 16 unmodified lines
17	17	Copyright © 2000, 2001, 2002, 2007, 2008 Free Software Foundation, Inc.<br>
18	18	Licensed under the GNU Free Documentation License.
19	19	HTML
	20	+
	21	+ def get_latest_version(&block)
	22	+ fetch('https://www.gnu.org/software/bash/manual/html_node/index.html') do \|body\|
	23	+ version = body.scan(/, Version ([0-9.]+)/)[0][0]
	24	+ block.call version[0...-1]
	25	+ end
	26	+ end
20	27	end
21	28	end

@@ -18,5 +18,9 @@ module Docs · 17 unmodified lines
18	18	© 2013–2017 Petka Antonov<br>
19	19	Licensed under the MIT License.
20	20	HTML
	21	+
	22	+ def get_latest_version(&block)
	23	+ get_npm_version('bluebird', &block)
	24	+ end
21	25	end
22	26	end

@@ -2,6 +2,7 @@ module Docs · 1 unmodified line
2	2	class Pygame < UrlScraper
3	3	self.type = 'simple'
4	4	self.release = '1.9.4'
	5	+ self.base_url = 'https://www.pygame.org/docs/'
5	6	self.root_path = 'py-modindex.html'
6	7	self.links = {
7	8	home: 'https://www.pygame.org/',