Added dojo to devdocs & ability to define headers in scraper requests

ShaneQful committed 11 years ago

Commit 34659331 parent 59778f77 files · +38 −135Browse files

Unified · Split

7 files

lib/docs/core/scrapers/url_scraper.rb +4 −1
lib/docs/filters/dojo/clean_html.rb +8 −1
lib/docs/filters/dojo/entries.rb +0 −1
lib/docs/scrapers/dojo.rb +16 −134
public/icons/docs/dojo/16.png +0 −0
public/icons/docs/dojo/16@2x.png +0 −0
test/lib/docs/core/scrapers/url_scraper_test.rb +12 −0

lib/docs/core/scrapers/url_scraper.rb+4−1

   class UrlScraper < Scraper
     class << self
       attr_accessor :params
+      attr_accessor :headers
       def inherited(subclass)
         super
         subclass.params = params.deep_dup
+        subclass.headers = headers.deep_dup
       end
     end
     self.params = {}
+    self.headers = { 'User-Agent' => 'devdocs.io' }
     private
     end
     def request_options
-      { params: self.class.params }
+      { params: self.class.params, headers: self.class.headers }
     end
     def process_response?(response)

lib/docs/filters/dojo/clean_html.rb+8−1

   class Dojo
     class CleanHtmlFilter < Filter
       def call
-        # TODO: Probably needs a little more cleanup but should do for the moment
         css('script').remove
+        css('.version').remove
+        #Remove links which are broken on the methods
+        doc.css(".functionIcon a").each do |a|
+            a.replace a.content
+        end
         doc
       end
     end

lib/docs/filters/dojo/entries.rb+0−1

@@ -8,7 +8,6 @@ module Docs · 7 unmodified lines
8	def get_type	8	def get_type
9	name	9	name
10	end	10	end
11	# TODO:Figure out how to solve the internal links issue later
12	end	11	end
13	end	12	end
14	end	13	end

lib/docs/scrapers/dojo.rb+16−134

@@ -4,7 +4,13 @@ module Docs · 3 unmodified lines
4	self.slug = 'dojo'	4	self.slug = 'dojo'
5	self.type = 'dojo'	5	self.type = 'dojo'
6	self.version = '1.10'	6	self.version = '1.10'
7	self.base_url = 'http://dojotoolkit.org/api/1.10/' #tree.json	7	self.base_url = 'http://dojotoolkit.org/api/1.10/'
		8
		9	# This is a cut down list of the actually paths taken from the tree.json api on the dojo site
		10	# Dojo used javascript and xhr requests to allow users to browse it's documentation so it can't
		11	# be scrapped by just following links from the base page. This list was generating with a little
		12	# bash and then cut down in order to remove a lot of the more unused documentation e.g. kernel,
		13	# main, dnd and some others
8	self.initial_paths = %w(	14	self.initial_paths = %w(
9	dojo/AdapterRegistry	15	dojo/AdapterRegistry
10	dojo/aspect	16	dojo/aspect
@@ -17,65 +23,18 @@ module Docs · 6 unmodified lines
17	dojo/_base/config.modulePaths	23	dojo/_base/config.modulePaths
18	dojo/_base/connect	24	dojo/_base/connect
19	dojo/_base/declare	25	dojo/_base/declare
20	dojo/_base/declare.__DeclareCreatedObject
21	dojo/_base/Deferred	26	dojo/_base/Deferred
22	dojo/_base/event	27	dojo/_base/event
23	dojo/_base/fx	28	dojo/_base/fx
24	dojo/_base/html	29	dojo/_base/html
25	dojo/_base/json	30	dojo/_base/json
26	dojo/_base/kernel	31	dojo/_base/kernel
27	dojo/_base/kernel.back
28	dojo/_base/kernel.cldr
29	dojo/_base/kernel.colors
30	dojo/_base/kernel.config
31	dojo/_base/kernel.contentHandlers
32	dojo/_base/kernel._contentHandlers
33	dojo/_base/kernel.currency
34	dojo/_base/kernel.data
35	dojo/_base/kernel.date
36	dojo/_base/kernel.dijit
37	dojo/_base/kernel.dnd
38	dojo/_base/kernel.doc
39	dojo/_base/kernel.dojox
40	dojo/_base/kernel.fx
41	dojo/_base/kernel.gears
42	dojo/_base/kernel.global
43	dojo/_base/kernel._hasResource
44	dojo/_base/kernel.html
45	dojo/_base/kernel.i18n
46	dojo/_base/kernel.io
47	dojo/_base/kernel.__IoArgs
48	dojo/_base/kernel.__IoCallbackArgs
49	dojo/_base/kernel.__IoPublish
50	dojo/_base/kernel.keys
51	dojo/_base/kernel.mouseButtons
52	dojo/_base/kernel._nodeDataCache
53	dojo/_base/kernel.number
54	dojo/_base/kernel.regexp
55	dojo/_base/kernel.rpc
56	dojo/_base/kernel.scopeMap
57	dojo/_base/kernel.Stateful
58	dojo/_base/kernel.store
59	dojo/_base/kernel.string
60	dojo/_base/kernel.tests
61	dojo/_base/kernel.touch
62	dojo/_base/kernel.version
63	dojo/_base/kernel.window
64	dojo/_base/kernel.__XhrArgs
65	dojo/_base/lang	32	dojo/_base/lang
66	dojo/_base/loader	33	dojo/_base/loader
67	dojo/_base/NodeList	34	dojo/_base/NodeList
68	dojo/_base/query	35	dojo/_base/query
69	dojo/_base/sniff	36	dojo/_base/sniff
70	dojo/_base/unload	37	dojo/_base/unload
71	dojo/_base/url
72	dojo/_base/url.authority
73	dojo/_base/url.fragment
74	dojo/_base/url.password
75	dojo/_base/url.port
76	dojo/_base/url.query
77	dojo/_base/url.scheme
78	dojo/_base/url.user
79	dojo/_base/window	38	dojo/_base/window
80	dojo/_base/window.doc	39	dojo/_base/window.doc
81	dojo/_base/window.global	40	dojo/_base/window.global
@@ -88,8 +47,6 @@ module Docs · 6 unmodified lines
88	dojo/colors	47	dojo/colors
89	dojo/cookie	48	dojo/cookie
90	dojo/currency	49	dojo/currency
91	dojo/currency.__FormatOptions
92	dojo/currency.__ParseOptions
93	dojo/data/api/Identity	50	dojo/data/api/Identity
94	dojo/data/api/Item	51	dojo/data/api/Item
95	dojo/data/api/Notification	52	dojo/data/api/Notification
@@ -104,33 +61,10 @@ module Docs · 8 unmodified lines
104	dojo/data/util/sorter	61	dojo/data/util/sorter
105	dojo/date	62	dojo/date
106	dojo/date/locale	63	dojo/date/locale
107	dojo/date/locale.__FormatOptions
108	dojo/date/stamp	64	dojo/date/stamp
109	dojo/debounce	65	dojo/debounce
110	dojo/Deferred	66	dojo/Deferred
111	dojo/DeferredList	67	dojo/DeferredList
112	dojo/dnd/autoscroll
113	dojo/dnd/autoscroll._validNodes
114	dojo/dnd/autoscroll._validOverflow
115	dojo/dnd/AutoSource
116	dojo/dnd/Avatar
117	dojo/dnd/common
118	dojo/dnd/common._defaultCreatorNodes
119	dojo/dnd/common._empty
120	dojo/dnd/Container
121	dojo/dnd/Container.__ContainerArgs
122	dojo/dnd/Manager
123	dojo/dnd/move
124	dojo/dnd/Moveable
125	dojo/dnd/Moveable.__MoveableArgs
126	dojo/dnd/move.boxConstrainedMoveable
127	dojo/dnd/move.constrainedMoveable
128	dojo/dnd/move.parentConstrainedMoveable
129	dojo/dnd/Mover
130	dojo/dnd/Selector
131	dojo/dnd/Source
132	dojo/dnd/Target
133	dojo/dnd/TimedMoveable
134	dojo/dom	68	dojo/dom
135	dojo/dom-attr	69	dojo/dom-attr
136	dojo/dom-class	70	dojo/dom-class
@@ -146,7 +80,6 @@ module Docs · 9 unmodified lines
146	dojo/errors/RequestError	80	dojo/errors/RequestError
147	dojo/errors/RequestTimeoutError	81	dojo/errors/RequestTimeoutError
148	dojo/Evented	82	dojo/Evented
149	dojo/_firebug/firebug
150	dojo/fx	83	dojo/fx
151	dojo/fx/easing	84	dojo/fx/easing
152	dojo/fx.easing	85	dojo/fx.easing
@@ -168,44 +101,6 @@ module Docs · 15 unmodified lines
168	dojo/keys	101	dojo/keys
169	dojo/loadInit	102	dojo/loadInit
170	dojo/main	103	dojo/main
171	dojo/main.back
172	dojo/main.cldr
173	dojo/main.colors
174	dojo/main.config
175	dojo/main.contentHandlers
176	dojo/main._contentHandlers
177	dojo/main.currency
178	dojo/main.data
179	dojo/main.date
180	dojo/main.dijit
181	dojo/main.dnd
182	dojo/main.doc
183	dojo/main.dojox
184	dojo/main.fx
185	dojo/main.gears
186	dojo/main.global
187	dojo/main._hasResource
188	dojo/main.html
189	dojo/main.i18n
190	dojo/main.io
191	dojo/main.__IoArgs
192	dojo/main.__IoCallbackArgs
193	dojo/main.__IoPublish
194	dojo/main.keys
195	dojo/main.mouseButtons
196	dojo/main._nodeDataCache
197	dojo/main.number
198	dojo/main.regexp
199	dojo/main.rpc
200	dojo/main.scopeMap
201	dojo/main.Stateful
202	dojo/main.store
203	dojo/main.string
204	dojo/main.tests
205	dojo/main.touch
206	dojo/main.version
207	dojo/main.window
208	dojo/main.__XhrArgs
209	dojo/mouse	104	dojo/mouse
210	dojo/node	105	dojo/node
211	dojo/NodeList	106	dojo/NodeList
@@ -217,12 +112,6 @@ module Docs · 5 unmodified lines
217	dojo/NodeList._nodeDataCache	112	dojo/NodeList._nodeDataCache
218	dojo/NodeList-traverse	113	dojo/NodeList-traverse
219	dojo/number	114	dojo/number
220	dojo/number.__FormatAbsoluteOptions
221	dojo/number.__FormatOptions
222	dojo/number.__IntegerRegexpFlags
223	dojo/number.__ParseOptions
224	dojo/number.__RealNumberRegexpFlags
225	dojo/number.__RegexpOptions
226	dojo/on	115	dojo/on
227	dojo/on/asyncEventListener	116	dojo/on/asyncEventListener
228	dojo/on/debounce	117	dojo/on/debounce
@@ -237,32 +126,16 @@ module Docs · 8 unmodified lines
237	dojo/ready	126	dojo/ready
238	dojo/regexp	127	dojo/regexp
239	dojo/request	128	dojo/request
240	dojo/request.__BaseOptions
241	dojo/request/default	129	dojo/request/default
242	dojo/request/handlers	130	dojo/request/handlers
243	dojo/request/iframe	131	dojo/request/iframe
244	dojo/request/iframe.__BaseOptions
245	dojo/request/iframe.__MethodOptions
246	dojo/request/iframe.__Options
247	dojo/request.__MethodOptions
248	dojo/request/node	132	dojo/request/node
249	dojo/request/node.__BaseOptions
250	dojo/request/node.__MethodOptions
251	dojo/request/node.__Options
252	dojo/request/notify	133	dojo/request/notify
253	dojo/request.__Options
254	dojo/request.__Promise
255	dojo/request/registry	134	dojo/request/registry
256	dojo/request/script	135	dojo/request/script
257	dojo/request/script.__BaseOptions
258	dojo/request/script.__MethodOptions
259	dojo/request/script.__Options
260	dojo/request/util	136	dojo/request/util
261	dojo/request/watch	137	dojo/request/watch
262	dojo/request/xhr	138	dojo/request/xhr
263	dojo/request/xhr.__BaseOptions
264	dojo/request/xhr.__MethodOptions
265	dojo/request/xhr.__Options
266	dojo/require	139	dojo/require
267	dojo/robot	140	dojo/robot
268	dojo/robot._runsemaphore	141	dojo/robot._runsemaphore
@@ -299,6 +172,11 @@ module Docs · 30 unmodified lines
299	dojo/uacss	172	dojo/uacss
300	dojo/when	173	dojo/when
301	dojo/window)	174	dojo/window)
		175	# Add the rest of the url to the path
		176	self.initial_paths = self.initial_paths.map { \|l\| l + ".html?xhr=true" }
		177	# Dojo expects all the requests to be xhrs or it redirects you back to the docs home page
		178	# where it uses js to call the backend based on the URL so you get the appropriate documentation
		179	self.headers = { 'User-Agent' => 'devdocs.io' , 'X-Requested-With' => 'XMLHttpRequest' }
302	self.links = {	180	self.links = {
303	home: 'http://dojotoolkit.org',	181	home: 'http://dojotoolkit.org',
304	code: 'https://github.com/dojo/dojo'	182	code: 'https://github.com/dojo/dojo'
@@ -306,7 +184,11 @@ module Docs · 1 unmodified line
306		184
307	html_filters.push 'dojo/clean_html', 'dojo/entries'	185	html_filters.push 'dojo/clean_html', 'dojo/entries'
308		186
		187	# Don't use default selector on xhrs as no body or html document exists
		188	options[:container] = ->(filter) { filter.root_page? ? '#content' : false }
		189	options[:follow_links] = false
309	options[:skip_links] = true	190	options[:skip_links] = true
		191	options[:only] = self.initial_paths
310		192
311	options[:attribution] = <<-HTML	193	options[:attribution] = <<-HTML
312	The Dojo Toolkit is Copyright © 2005–2013 <br>	194	The Dojo Toolkit is Copyright © 2005–2013 <br>

public/icons/docs/dojo/16.png+0−0

Binary file not shown.

public/icons/docs/dojo/16@2x.png+0−0

Binary file not shown.

test/lib/docs/core/scrapers/url_scraper_test.rb+12−0

       result
     end
+    it "runs a Requester with .headers as :request_options" do
+      stub(Scraper).headers { { testheader: true } }
+      mock(Docs::Requester).run anything, satisfy { |options| options[:request_options][:headers][:testheader] }
+      result
+    end
+    it "runs a Requester with default .headers as :request_options" do
+      mock(Docs::Requester).run anything, satisfy { |options| options[:request_options][:headers]["User-Agent"] }
+      result
+    end
     it "runs a Requester with .params as :request_options" do
       stub(Scraper).params { { test: true } }
       mock(Docs::Requester).run anything, satisfy { |options| options[:request_options][:params][:test] }
       result
     end
     it "runs a Requester with the given block" do
       stub(Docs::Requester).run { |*args| @block = args.last }
       result