Add HTTP documentation from MDN

Thibaut Courouble committed 9 years ago

Commit 8a59a661 parent 9a970676 files · +93 −103Browse files

Unified · Split

6 files

assets/javascripts/news.json +3 −0
assets/javascripts/templates/pages/about_tmpl.coffee +1 −1
lib/docs/filters/http/clean_html.rb +21 −5
lib/docs/filters/http/entries.rb +31 −86
lib/docs/filters/mdn/contribute_link.rb +3 −0
lib/docs/scrapers/http.rb +34 −11

@@ -1,5 +1,8 @@
1	1	[
2	2	[
	3	+ "2017-01-22",
	4	+ "New <a href=\"/http/\">HTTP</a> documentation (thanks Mozilla)"
	5	+ ], [
3	6	"2016-12-04",
4	7	"New documentations: <a href=\"/sqlite/\">SQLite</a>, <a href=\"/codeception/\">Codeception</a> and <a href=\"/codeceptjs/\">CodeceptJS</a>"
5	8	], [

assets/javascripts/templates/pages/about_tmpl.coffee+1−1

@@ -180,7 +180,7 @@ credits = [ · 179 unmodified lines
180	180	'Apache',
181	181	'https://raw.githubusercontent.com/apache/cordova-docs/master/LICENSE'
182	182	], [
183		− 'CSS<br>DOM<br>HTML<br>JavaScript<br>SVG<br>XPath',
	183	+ 'CSS<br>DOM<br>HTTP<br>HTML<br>JavaScript<br>SVG<br>XPath',
184	184	'2005-2017 Mozilla Developer Network and individual contributors',
185	185	'CC BY-SA',
186	186	'https://creativecommons.org/licenses/by-sa/2.5/'

lib/docs/filters/http/clean_html.rb+21−5

@@ -2,11 +2,29 @@ module Docs · 1 unmodified line
2	2	class Http
3	3	class CleanHtmlFilter < Filter
4	4	def call
5		− if root_page?
6		− doc.inner_html = '<h1>Hypertext Transfer Protocol</h1>'
7		− return doc
	5	+ current_url.host == 'tools.ietf.org' ? ietf : mdn
	6	+ doc
	7	+ end
	8	+
	9	+ def mdn
	10	+ css('.column-container', '.column-half').each do \|node\|
	11	+ node.before(node.children).remove
	12	+ end
	13	+
	14	+ css('p > code + strong').each do \|node\|
	15	+ code = node.previous_element
	16	+ if code.content =~ /\A[\s\d]+\z/
	17	+ code.content = "#{code.content.strip} #{node.content.strip}"
	18	+ node.remove
	19	+ end
8	20	end
9	21
	22	+ css('strong > code').each do \|node\|
	23	+ node.parent.before(node.parent.children).remove
	24	+ end
	25	+ end
	26	+
	27	+ def ietf
10	28	doc.child.remove while doc.child.name != 'pre'
11	29
12	30	css('span.grey', '.invisible', '.noprint', 'a[href^="#page-"]').remove
@@ -33,8 +51,6 @@ module Docs · 20 unmodified lines
33	51	html.remove! %r[\.{2,}$]
34	52	html.gsub! %r[(^\n$){3,}], "\n"
35	53	doc.inner_html = %(<div class="_rfc-pre">#{html}</div>)
36		−
37		− doc
38	54	end
39	55	end
40	56	end

lib/docs/filters/http/entries.rb+31−86

@@ -2,95 +2,40 @@ module Docs · 1 unmodified line
2	2	class Http
3	3	class EntriesFilter < Docs::EntriesFilter
4	4	def get_name
5		− name = at_css('h1').content
6		− name.remove! %r{\A.+\:}
7		− name.remove! %r{\A.+\-\-}
8		− "#{rfc}: #{name.strip}"
	5	+ if current_url.host == 'tools.ietf.org'
	6	+ name = at_css('h1').content
	7	+ name.remove! %r{\A.+\:}
	8	+ name.remove! %r{\A.+\-\-}
	9	+ rfc = slug.sub('rfc', 'RFC ')
	10	+ "#{rfc}: #{name.strip}"
	11	+ elsif slug.start_with?('Status')
	12	+ at_css('code').content
	13	+ else
	14	+ name = super
	15	+ name.remove! %r{\A\w+\.}
	16	+ name.remove! 'Basics of HTTP.'
	17	+ name.sub! 'Content-Security-Policy.', 'CSP.'
	18	+ name.sub! '.', ': '
	19	+ name.sub! '1: x', '1.x'
	20	+ name
	21	+ end
9	22	end
10	23
11	24	def get_type
12		− 'RFC'
13		− end
14		−
15		− def rfc
16		− slug.sub('rfc', 'RFC ')
17		− end
18		−
19		− SECTIONS = {
20		− 'rfc2616' => [
21		− [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15],
22		− [14],
23		− []
24		− ],
25		− 'rfc4918' => [
26		− [],
27		− [11],
28		− []
29		− ],
30		− 'rfc7230' => [
31		− (2..9).to_a,
32		− [],
33		− []
34		− ],
35		− 'rfc7231' => [
36		− [3, 8, 9],
37		− [],
38		− [4, 5, 6, 7]
39		− ],
40		− 'rfc7232' => [
41		− [5, 6, 7, 8],
42		− [2, 3, 4],
43		− []
44		− ],
45		− 'rfc7233' => [
46		− [5, 6],
47		− [2, 3, 4],
48		− []
49		− ],
50		− 'rfc7234' => [
51		− [3, 6, 7, 8],
52		− [4, 5],
53		− []
54		− ],
55		− 'rfc7235' => [
56		− [2, 5, 6],
57		− [3, 4],
58		− []
59		− ]
60		− }
61		−
62		− LEVEL_1 = /\A(\d+)\z/
63		− LEVEL_2 = /\A(\d+)\.\d+\z/
64		− LEVEL_3 = /\A(\d+)\.\d+\.\d+\z/
65		−
66		− def additional_entries
67		− return [] if root_page?
68		− type = nil
69		−
70		− css('a[href^="#section-"]').each_with_object([]) do \|node, entries\|
71		− id = node['href'].remove('#')
72		− break entries if entries.any? { \|e\| e[1] == id }
73		−
74		− content = node.next.content.strip
75		− content.remove! %r{\s\.+\d\z}
76		− content.remove! %r{\A[\.\s]+}
77		−
78		− name = "#{content} (#{rfc})"
79		− number = node.content.strip
80		−
81		− if number =~ LEVEL_1
82		− if SECTIONS[slug][0].include?($1.to_i)
83		− entries << [name, id, self.name]
84		− end
85		−
86		− type = content.sub(/\ Definitions\z/, 's')
87		− type = 'Request Header Fields' if type.include?('Header Fields') && type.exclude?('Response')
88		− type = 'Response Status Codes' if type.include?('Status Codes')
89		− type = self.name unless type.start_with?('Request ') \|\| type.start_with?('Response ')
90		− elsif (number =~ LEVEL_2 && SECTIONS[slug][1].include?($1.to_i)) \|\|
91		− (number =~ LEVEL_3 && SECTIONS[slug][2].include?($1.to_i))
92		− entries << [name, id, (name =~ /\A\d\d\d/ ? 'Response Status Codes' : type )]
93		− end
	25	+ return 'RFC' if current_url.host == 'tools.ietf.org'
	26	+
	27	+ if slug.start_with?('Headers/Content-Security-Policy')
	28	+ 'CSP'
	29	+ elsif slug.start_with?('Headers')
	30	+ 'Headers'
	31	+ elsif slug.start_with?('Methods')
	32	+ 'Methods'
	33	+ elsif slug.start_with?('Status')
	34	+ 'Status'
	35	+ elsif slug.start_with?('Basics_of_HTTP')
	36	+ 'Guides: Basics'
	37	+ else
	38	+ 'Guides'
94	39	end
95	40	end
96	41	end

lib/docs/filters/mdn/contribute_link.rb+3−0

@@ -2,6 +2,8 @@ module Docs · 1 unmodified line
2	2	class Mdn
3	3	class ContributeLinkFilter < Filter
4	4	def call
	5	+ return html if current_url.host != 'developer.mozilla.org'
	6	+
5	7	html << <<-HTML.strip_heredoc
6	8	<div class="_attribution">
7	9	<p class="_attribution-p">
@@ -9,6 +11,7 @@ module Docs · 1 unmodified line
9	11	</p>
10	12	</div>
11	13	HTML
	14	+
12	15	html
13	16	end
14	17	end

lib/docs/scrapers/http.rb+34−11

@@ -1,17 +1,40 @@
1	1	module Docs
2		− class Http < UrlScraper
	2	+ class Http < Mdn
	3	+ include MultipleBaseUrls
	4	+
3	5	self.name = 'HTTP'
4		− self.type = 'rfc'
5		− self.base_url = 'https://tools.ietf.org/html/'
6		− self.initial_paths = %w(rfc2616 rfc4918 rfc7230 rfc7231
7		− rfc7232 rfc7233 rfc7234 rfc7235)
	6	+ self.base_urls = ['https://developer.mozilla.org/en-US/docs/Web/HTTP', 'https://tools.ietf.org/html/']
	7	+
	8	+ html_filters.push 'http/clean_html', 'http/entries', 'title'
8	9
9		− html_filters.push 'http/clean_html', 'http/entries'
	10	+ options[:root_title] = 'HTTP'
	11	+ options[:title] = ->(filter) {
	12	+ filter.current_url.host == 'tools.ietf.org' ? false : filter.default_title
	13	+ }
	14	+ options[:container] = ->(filter) {
	15	+ filter.current_url.host == 'tools.ietf.org' ? '.content' : nil
	16	+ }
	17	+ options[:skip_links] = ->(filter) {
	18	+ filter.current_url.host == 'tools.ietf.org' ? true : false
	19	+ }
	20	+ options[:attribution] = ->(filter) {
	21	+ if filter.current_url.host == 'tools.ietf.org'
	22	+ "© document authors. All rights reserved."
	23	+ else
	24	+ Docs::Mdn.options[:attribution]
	25	+ end
	26	+ }
10	27
11		− options[:container] = '.content'
12		− options[:skip_links] = true
13		− options[:attribution] = <<-HTML
14		− © document authors. All rights reserved.
15		− HTML
	28	+ def initial_urls
	29	+ %w(https://developer.mozilla.org/en-US/docs/Web/HTTP
	30	+ https://tools.ietf.org/html/rfc2616
	31	+ https://tools.ietf.org/html/rfc4918
	32	+ https://tools.ietf.org/html/rfc7230
	33	+ https://tools.ietf.org/html/rfc7231
	34	+ https://tools.ietf.org/html/rfc7232
	35	+ https://tools.ietf.org/html/rfc7233
	36	+ https://tools.ietf.org/html/rfc7234
	37	+ https://tools.ietf.org/html/rfc7235)
	38	+ end
16	39	end
17	40	end