Added dojo to devdocs & ability to define headers in scraper requests

ShaneQful committed 11 years ago

Commit 34659331 parent 59778f77 files · +38 −135Browse files

Unified · Split

7 files

lib/docs/core/scrapers/url_scraper.rb +4 −1
lib/docs/filters/dojo/clean_html.rb +8 −1
lib/docs/filters/dojo/entries.rb +0 −1
lib/docs/scrapers/dojo.rb +16 −134
public/icons/docs/dojo/16.png +0 −0
public/icons/docs/dojo/16@2x.png +0 −0
test/lib/docs/core/scrapers/url_scraper_test.rb +12 −0

lib/docs/core/scrapers/url_scraper.rb+4−1

@@ -2,14 +2,17 @@ module Docs · 1 unmodified line
2	2	class UrlScraper < Scraper
3	3	class << self
4	4	attr_accessor :params
	5	+ attr_accessor :headers
5	6
6	7	def inherited(subclass)
7	8	super
8	9	subclass.params = params.deep_dup
	10	+ subclass.headers = headers.deep_dup
9	11	end
10	12	end
11	13
12	14	self.params = {}
	15	+ self.headers = { 'User-Agent' => 'devdocs.io' }
13	16
14	17	private
15	18
@@ -22,7 +25,7 @@ module Docs · 6 unmodified lines
22	25	end
23	26
24	27	def request_options
25		− { params: self.class.params }
	28	+ { params: self.class.params, headers: self.class.headers }
26	29	end
27	30
28	31	def process_response?(response)

lib/docs/filters/dojo/clean_html.rb+8−1

@@ -2,8 +2,15 @@ module Docs · 1 unmodified line
2	2	class Dojo
3	3	class CleanHtmlFilter < Filter
4	4	def call
5		− # TODO: Probably needs a little more cleanup but should do for the moment
6	5	css('script').remove
	6	+
	7	+ css('.version').remove
	8	+
	9	+ #Remove links which are broken on the methods
	10	+ doc.css(".functionIcon a").each do \|a\|
	11	+ a.replace a.content
	12	+ end
	13	+
7	14	doc
8	15	end
9	16	end

lib/docs/filters/dojo/entries.rb+0−1

@@ -8,7 +8,6 @@ module Docs · 7 unmodified lines
8	8	def get_type
9	9	name
10	10	end
11		− # TODO:Figure out how to solve the internal links issue later
12	11	end
13	12	end
14	13	end

lib/docs/scrapers/dojo.rb+16−134

@@ -4,7 +4,13 @@ module Docs · 3 unmodified lines
4	4	self.slug = 'dojo'
5	5	self.type = 'dojo'
6	6	self.version = '1.10'
7		− self.base_url = 'http://dojotoolkit.org/api/1.10/' #tree.json
	7	+ self.base_url = 'http://dojotoolkit.org/api/1.10/'
	8	+
	9	+ # This is a cut down list of the actually paths taken from the tree.json api on the dojo site
	10	+ # Dojo used javascript and xhr requests to allow users to browse it's documentation so it can't
	11	+ # be scrapped by just following links from the base page. This list was generating with a little
	12	+ # bash and then cut down in order to remove a lot of the more unused documentation e.g. kernel,
	13	+ # main, dnd and some others
8	14	self.initial_paths = %w(
9	15	dojo/AdapterRegistry
10	16	dojo/aspect
@@ -17,65 +23,18 @@ module Docs · 6 unmodified lines
17	23	dojo/_base/config.modulePaths
18	24	dojo/_base/connect
19	25	dojo/_base/declare
20		− dojo/_base/declare.__DeclareCreatedObject
21	26	dojo/_base/Deferred
22	27	dojo/_base/event
23	28	dojo/_base/fx
24	29	dojo/_base/html
25	30	dojo/_base/json
26	31	dojo/_base/kernel
27		− dojo/_base/kernel.back
28		− dojo/_base/kernel.cldr
29		− dojo/_base/kernel.colors
30		− dojo/_base/kernel.config
31		− dojo/_base/kernel.contentHandlers
32		− dojo/_base/kernel._contentHandlers
33		− dojo/_base/kernel.currency
34		− dojo/_base/kernel.data
35		− dojo/_base/kernel.date
36		− dojo/_base/kernel.dijit
37		− dojo/_base/kernel.dnd
38		− dojo/_base/kernel.doc
39		− dojo/_base/kernel.dojox
40		− dojo/_base/kernel.fx
41		− dojo/_base/kernel.gears
42		− dojo/_base/kernel.global
43		− dojo/_base/kernel._hasResource
44		− dojo/_base/kernel.html
45		− dojo/_base/kernel.i18n
46		− dojo/_base/kernel.io
47		− dojo/_base/kernel.__IoArgs
48		− dojo/_base/kernel.__IoCallbackArgs
49		− dojo/_base/kernel.__IoPublish
50		− dojo/_base/kernel.keys
51		− dojo/_base/kernel.mouseButtons
52		− dojo/_base/kernel._nodeDataCache
53		− dojo/_base/kernel.number
54		− dojo/_base/kernel.regexp
55		− dojo/_base/kernel.rpc
56		− dojo/_base/kernel.scopeMap
57		− dojo/_base/kernel.Stateful
58		− dojo/_base/kernel.store
59		− dojo/_base/kernel.string
60		− dojo/_base/kernel.tests
61		− dojo/_base/kernel.touch
62		− dojo/_base/kernel.version
63		− dojo/_base/kernel.window
64		− dojo/_base/kernel.__XhrArgs
65	32	dojo/_base/lang
66	33	dojo/_base/loader
67	34	dojo/_base/NodeList
68	35	dojo/_base/query
69	36	dojo/_base/sniff
70	37	dojo/_base/unload
71		− dojo/_base/url
72		− dojo/_base/url.authority
73		− dojo/_base/url.fragment
74		− dojo/_base/url.password
75		− dojo/_base/url.port
76		− dojo/_base/url.query
77		− dojo/_base/url.scheme
78		− dojo/_base/url.user
79	38	dojo/_base/window
80	39	dojo/_base/window.doc
81	40	dojo/_base/window.global
@@ -88,8 +47,6 @@ module Docs · 6 unmodified lines
88	47	dojo/colors
89	48	dojo/cookie
90	49	dojo/currency
91		− dojo/currency.__FormatOptions
92		− dojo/currency.__ParseOptions
93	50	dojo/data/api/Identity
94	51	dojo/data/api/Item
95	52	dojo/data/api/Notification
@@ -104,33 +61,10 @@ module Docs · 8 unmodified lines
104	61	dojo/data/util/sorter
105	62	dojo/date
106	63	dojo/date/locale
107		− dojo/date/locale.__FormatOptions
108	64	dojo/date/stamp
109	65	dojo/debounce
110	66	dojo/Deferred
111	67	dojo/DeferredList
112		− dojo/dnd/autoscroll
113		− dojo/dnd/autoscroll._validNodes
114		− dojo/dnd/autoscroll._validOverflow
115		− dojo/dnd/AutoSource
116		− dojo/dnd/Avatar
117		− dojo/dnd/common
118		− dojo/dnd/common._defaultCreatorNodes
119		− dojo/dnd/common._empty
120		− dojo/dnd/Container
121		− dojo/dnd/Container.__ContainerArgs
122		− dojo/dnd/Manager
123		− dojo/dnd/move
124		− dojo/dnd/Moveable
125		− dojo/dnd/Moveable.__MoveableArgs
126		− dojo/dnd/move.boxConstrainedMoveable
127		− dojo/dnd/move.constrainedMoveable
128		− dojo/dnd/move.parentConstrainedMoveable
129		− dojo/dnd/Mover
130		− dojo/dnd/Selector
131		− dojo/dnd/Source
132		− dojo/dnd/Target
133		− dojo/dnd/TimedMoveable
134	68	dojo/dom
135	69	dojo/dom-attr
136	70	dojo/dom-class
@@ -146,7 +80,6 @@ module Docs · 9 unmodified lines
146	80	dojo/errors/RequestError
147	81	dojo/errors/RequestTimeoutError
148	82	dojo/Evented
149		− dojo/_firebug/firebug
150	83	dojo/fx
151	84	dojo/fx/easing
152	85	dojo/fx.easing
@@ -168,44 +101,6 @@ module Docs · 15 unmodified lines
168	101	dojo/keys
169	102	dojo/loadInit
170	103	dojo/main
171		− dojo/main.back
172		− dojo/main.cldr
173		− dojo/main.colors
174		− dojo/main.config
175		− dojo/main.contentHandlers
176		− dojo/main._contentHandlers
177		− dojo/main.currency
178		− dojo/main.data
179		− dojo/main.date
180		− dojo/main.dijit
181		− dojo/main.dnd
182		− dojo/main.doc
183		− dojo/main.dojox
184		− dojo/main.fx
185		− dojo/main.gears
186		− dojo/main.global
187		− dojo/main._hasResource
188		− dojo/main.html
189		− dojo/main.i18n
190		− dojo/main.io
191		− dojo/main.__IoArgs
192		− dojo/main.__IoCallbackArgs
193		− dojo/main.__IoPublish
194		− dojo/main.keys
195		− dojo/main.mouseButtons
196		− dojo/main._nodeDataCache
197		− dojo/main.number
198		− dojo/main.regexp
199		− dojo/main.rpc
200		− dojo/main.scopeMap
201		− dojo/main.Stateful
202		− dojo/main.store
203		− dojo/main.string
204		− dojo/main.tests
205		− dojo/main.touch
206		− dojo/main.version
207		− dojo/main.window
208		− dojo/main.__XhrArgs
209	104	dojo/mouse
210	105	dojo/node
211	106	dojo/NodeList
@@ -217,12 +112,6 @@ module Docs · 5 unmodified lines
217	112	dojo/NodeList._nodeDataCache
218	113	dojo/NodeList-traverse
219	114	dojo/number
220		− dojo/number.__FormatAbsoluteOptions
221		− dojo/number.__FormatOptions
222		− dojo/number.__IntegerRegexpFlags
223		− dojo/number.__ParseOptions
224		− dojo/number.__RealNumberRegexpFlags
225		− dojo/number.__RegexpOptions
226	115	dojo/on
227	116	dojo/on/asyncEventListener
228	117	dojo/on/debounce
@@ -237,32 +126,16 @@ module Docs · 8 unmodified lines
237	126	dojo/ready
238	127	dojo/regexp
239	128	dojo/request
240		− dojo/request.__BaseOptions
241	129	dojo/request/default
242	130	dojo/request/handlers
243	131	dojo/request/iframe
244		− dojo/request/iframe.__BaseOptions
245		− dojo/request/iframe.__MethodOptions
246		− dojo/request/iframe.__Options
247		− dojo/request.__MethodOptions
248	132	dojo/request/node
249		− dojo/request/node.__BaseOptions
250		− dojo/request/node.__MethodOptions
251		− dojo/request/node.__Options
252	133	dojo/request/notify
253		− dojo/request.__Options
254		− dojo/request.__Promise
255	134	dojo/request/registry
256	135	dojo/request/script
257		− dojo/request/script.__BaseOptions
258		− dojo/request/script.__MethodOptions
259		− dojo/request/script.__Options
260	136	dojo/request/util
261	137	dojo/request/watch
262	138	dojo/request/xhr
263		− dojo/request/xhr.__BaseOptions
264		− dojo/request/xhr.__MethodOptions
265		− dojo/request/xhr.__Options
266	139	dojo/require
267	140	dojo/robot
268	141	dojo/robot._runsemaphore
@@ -299,6 +172,11 @@ module Docs · 30 unmodified lines
299	172	dojo/uacss
300	173	dojo/when
301	174	dojo/window)
	175	+ # Add the rest of the url to the path
	176	+ self.initial_paths = self.initial_paths.map { \|l\| l + ".html?xhr=true" }
	177	+ # Dojo expects all the requests to be xhrs or it redirects you back to the docs home page
	178	+ # where it uses js to call the backend based on the URL so you get the appropriate documentation
	179	+ self.headers = { 'User-Agent' => 'devdocs.io' , 'X-Requested-With' => 'XMLHttpRequest' }
302	180	self.links = {
303	181	home: 'http://dojotoolkit.org',
304	182	code: 'https://github.com/dojo/dojo'
@@ -306,7 +184,11 @@ module Docs · 1 unmodified line
306	184
307	185	html_filters.push 'dojo/clean_html', 'dojo/entries'
308	186
	187	+ # Don't use default selector on xhrs as no body or html document exists
	188	+ options[:container] = ->(filter) { filter.root_page? ? '#content' : false }
	189	+ options[:follow_links] = false
309	190	options[:skip_links] = true
	191	+ options[:only] = self.initial_paths
310	192
311	193	options[:attribution] = <<-HTML
312	194	The Dojo Toolkit is Copyright © 2005–2013 <br>

public/icons/docs/dojo/16.png+0−0

Binary file not shown.

public/icons/docs/dojo/16@2x.png+0−0

Binary file not shown.

test/lib/docs/core/scrapers/url_scraper_test.rb+12−0

@@ -58,12 +58,24 @@ class DocsUrlScraperTest < MiniTest::Spec · 57 unmodified lines
58	58	result
59	59	end
60	60
	61	+ it "runs a Requester with .headers as :request_options" do
	62	+ stub(Scraper).headers { { testheader: true } }
	63	+ mock(Docs::Requester).run anything, satisfy { \|options\| options[:request_options][:headers][:testheader] }
	64	+ result
	65	+ end
	66	+
	67	+ it "runs a Requester with default .headers as :request_options" do
	68	+ mock(Docs::Requester).run anything, satisfy { \|options\| options[:request_options][:headers]["User-Agent"] }
	69	+ result
	70	+ end
	71	+
61	72	it "runs a Requester with .params as :request_options" do
62	73	stub(Scraper).params { { test: true } }
63	74	mock(Docs::Requester).run anything, satisfy { \|options\| options[:request_options][:params][:test] }
64	75	result
65	76	end
66	77
	78	+
67	79	it "runs a Requester with the given block" do
68	80	stub(Docs::Requester).run { \|*args\| @block = args.last }
69	81	result