History

Add HTTP documentation from MDN

Thibaut Courouble committed 9 years ago

Commit 8a59a661 parent 9a970676 files · +93 −103Browse files

Unified · Split

6 files
  1. assets/javascripts/news.json +3 −0
  2. assets/javascripts/templates/pages/about_tmpl.coffee +1 −1
  3. lib/docs/filters/http/clean_html.rb +21 −5
  4. lib/docs/filters/http/entries.rb +31 −86
  5. lib/docs/filters/mdn/contribute_link.rb +3 −0
  6. lib/docs/scrapers/http.rb +34 −11
assets/javascripts/news.json+3−0
@@ -1,5 +1,8 @@
1[1[
2 [2 [
3 "2017-01-22",
4 "New <a href=\"/http/\">HTTP</a> documentation (thanks Mozilla)"
5 ], [
3 "2016-12-04",6 "2016-12-04",
4 "New documentations: <a href=\"/sqlite/\">SQLite</a>, <a href=\"/codeception/\">Codeception</a> and <a href=\"/codeceptjs/\">CodeceptJS</a>"7 "New documentations: <a href=\"/sqlite/\">SQLite</a>, <a href=\"/codeception/\">Codeception</a> and <a href=\"/codeceptjs/\">CodeceptJS</a>"
5 ], [8 ], [
assets/javascripts/templates/pages/about_tmpl.coffee+1−1
@@ -180,7 +180,7 @@ credits = [ · 179 unmodified lines
180 'Apache',180 'Apache',
181 'https://raw.githubusercontent.com/apache/cordova-docs/master/LICENSE'181 'https://raw.githubusercontent.com/apache/cordova-docs/master/LICENSE'
182 ], [182 ], [
183 'CSS<br>DOM<br>HTML<br>JavaScript<br>SVG<br>XPath',183 'CSS<br>DOM<br>HTTP<br>HTML<br>JavaScript<br>SVG<br>XPath',
184 '2005-2017 Mozilla Developer Network and individual contributors',184 '2005-2017 Mozilla Developer Network and individual contributors',
185 'CC BY-SA',185 'CC BY-SA',
186 'https://creativecommons.org/licenses/by-sa/2.5/'186 'https://creativecommons.org/licenses/by-sa/2.5/'
lib/docs/filters/http/clean_html.rb+21−5
@@ -2,11 +2,29 @@ module Docs · 1 unmodified line
2 class Http2 class Http
3 class CleanHtmlFilter < Filter3 class CleanHtmlFilter < Filter
4 def call4 def call
5 if root_page?5 current_url.host == 'tools.ietf.org' ? ietf : mdn
6 doc.inner_html = '<h1>Hypertext Transfer Protocol</h1>'6 doc
7 return doc7 end
8
9 def mdn
10 css('.column-container', '.column-half').each do |node|
11 node.before(node.children).remove
12 end
13
14 css('p > code + strong').each do |node|
15 code = node.previous_element
16 if code.content =~ /\A[\s\d]+\z/
17 code.content = "#{code.content.strip} #{node.content.strip}"
18 node.remove
19 end
8 end20 end
9 21
22 css('strong > code').each do |node|
23 node.parent.before(node.parent.children).remove
24 end
25 end
26
27 def ietf
10 doc.child.remove while doc.child.name != 'pre'28 doc.child.remove while doc.child.name != 'pre'
11 29
12 css('span.grey', '.invisible', '.noprint', 'a[href^="#page-"]').remove30 css('span.grey', '.invisible', '.noprint', 'a[href^="#page-"]').remove
@@ -33,8 +51,6 @@ module Docs · 20 unmodified lines
33 html.remove! %r[\.{2,}$]51 html.remove! %r[\.{2,}$]
34 html.gsub! %r[(^\n$){3,}], "\n"52 html.gsub! %r[(^\n$){3,}], "\n"
35 doc.inner_html = %(<div class="_rfc-pre">#{html}</div>)53 doc.inner_html = %(<div class="_rfc-pre">#{html}</div>)
36
37 doc
38 end54 end
39 end55 end
40 end56 end
lib/docs/filters/http/entries.rb+31−86
@@ -2,95 +2,40 @@ module Docs · 1 unmodified line
2 class Http2 class Http
3 class EntriesFilter < Docs::EntriesFilter3 class EntriesFilter < Docs::EntriesFilter
4 def get_name4 def get_name
5 name = at_css('h1').content5 if current_url.host == 'tools.ietf.org'
6 name.remove! %r{\A.+\:}6 name = at_css('h1').content
7 name.remove! %r{\A.+\-\-}7 name.remove! %r{\A.+\:}
8 "#{rfc}: #{name.strip}"8 name.remove! %r{\A.+\-\-}
9 rfc = slug.sub('rfc', 'RFC ')
10 "#{rfc}: #{name.strip}"
11 elsif slug.start_with?('Status')
12 at_css('code').content
13 else
14 name = super
15 name.remove! %r{\A\w+\.}
16 name.remove! 'Basics of HTTP.'
17 name.sub! 'Content-Security-Policy.', 'CSP.'
18 name.sub! '.', ': '
19 name.sub! '1: x', '1.x'
20 name
21 end
9 end22 end
10 23
11 def get_type24 def get_type
12 'RFC'25 return 'RFC' if current_url.host == 'tools.ietf.org'
13 end26
14 27 if slug.start_with?('Headers/Content-Security-Policy')
15 def rfc28 'CSP'
16 slug.sub('rfc', 'RFC ')29 elsif slug.start_with?('Headers')
17 end30 'Headers'
18 31 elsif slug.start_with?('Methods')
19 SECTIONS = {32 'Methods'
20 'rfc2616' => [33 elsif slug.start_with?('Status')
21 [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15],34 'Status'
22 [14],35 elsif slug.start_with?('Basics_of_HTTP')
23 []36 'Guides: Basics'
24 ],37 else
25 'rfc4918' => [38 'Guides'
26 [],
27 [11],
28 []
29 ],
30 'rfc7230' => [
31 (2..9).to_a,
32 [],
33 []
34 ],
35 'rfc7231' => [
36 [3, 8, 9],
37 [],
38 [4, 5, 6, 7]
39 ],
40 'rfc7232' => [
41 [5, 6, 7, 8],
42 [2, 3, 4],
43 []
44 ],
45 'rfc7233' => [
46 [5, 6],
47 [2, 3, 4],
48 []
49 ],
50 'rfc7234' => [
51 [3, 6, 7, 8],
52 [4, 5],
53 []
54 ],
55 'rfc7235' => [
56 [2, 5, 6],
57 [3, 4],
58 []
59 ]
60 }
61
62 LEVEL_1 = /\A(\d+)\z/
63 LEVEL_2 = /\A(\d+)\.\d+\z/
64 LEVEL_3 = /\A(\d+)\.\d+\.\d+\z/
65
66 def additional_entries
67 return [] if root_page?
68 type = nil
69
70 css('a[href^="#section-"]').each_with_object([]) do |node, entries|
71 id = node['href'].remove('#')
72 break entries if entries.any? { |e| e[1] == id }
73
74 content = node.next.content.strip
75 content.remove! %r{\s*\.+\d*\z}
76 content.remove! %r{\A[\.\s]+}
77
78 name = "#{content} (#{rfc})"
79 number = node.content.strip
80
81 if number =~ LEVEL_1
82 if SECTIONS[slug][0].include?($1.to_i)
83 entries << [name, id, self.name]
84 end
85
86 type = content.sub(/\ Definitions\z/, 's')
87 type = 'Request Header Fields' if type.include?('Header Fields') && type.exclude?('Response')
88 type = 'Response Status Codes' if type.include?('Status Codes')
89 type = self.name unless type.start_with?('Request ') || type.start_with?('Response ')
90 elsif (number =~ LEVEL_2 && SECTIONS[slug][1].include?($1.to_i)) ||
91 (number =~ LEVEL_3 && SECTIONS[slug][2].include?($1.to_i))
92 entries << [name, id, (name =~ /\A\d\d\d/ ? 'Response Status Codes' : type )]
93 end
94 end39 end
95 end40 end
96 end41 end
lib/docs/filters/mdn/contribute_link.rb+3−0
@@ -2,6 +2,8 @@ module Docs · 1 unmodified line
2 class Mdn2 class Mdn
3 class ContributeLinkFilter < Filter3 class ContributeLinkFilter < Filter
4 def call4 def call
5 return html if current_url.host != 'developer.mozilla.org'
6
5 html << <<-HTML.strip_heredoc7 html << <<-HTML.strip_heredoc
6 <div class="_attribution">8 <div class="_attribution">
7 <p class="_attribution-p">9 <p class="_attribution-p">
@@ -9,6 +11,7 @@ module Docs · 1 unmodified line
9 </p>11 </p>
10 </div>12 </div>
11 HTML13 HTML
14
12 html15 html
13 end16 end
14 end17 end
lib/docs/scrapers/http.rb+34−11
@@ -1,17 +1,40 @@
1module Docs1module Docs
2 class Http < UrlScraper2 class Http < Mdn
3 include MultipleBaseUrls
4
3 self.name = 'HTTP'5 self.name = 'HTTP'
4 self.type = 'rfc'6 self.base_urls = ['https://developer.mozilla.org/en-US/docs/Web/HTTP', 'https://tools.ietf.org/html/']
5 self.base_url = 'https://tools.ietf.org/html/'7
6 self.initial_paths = %w(rfc2616 rfc4918 rfc7230 rfc72318 html_filters.push 'http/clean_html', 'http/entries', 'title'
7 rfc7232 rfc7233 rfc7234 rfc7235)
8 9
9 html_filters.push 'http/clean_html', 'http/entries'10 options[:root_title] = 'HTTP'
11 options[:title] = ->(filter) {
12 filter.current_url.host == 'tools.ietf.org' ? false : filter.default_title
13 }
14 options[:container] = ->(filter) {
15 filter.current_url.host == 'tools.ietf.org' ? '.content' : nil
16 }
17 options[:skip_links] = ->(filter) {
18 filter.current_url.host == 'tools.ietf.org' ? true : false
19 }
20 options[:attribution] = ->(filter) {
21 if filter.current_url.host == 'tools.ietf.org'
22 "&copy; document authors. All rights reserved."
23 else
24 Docs::Mdn.options[:attribution]
25 end
26 }
10 27
11 options[:container] = '.content'28 def initial_urls
12 options[:skip_links] = true29 %w(https://developer.mozilla.org/en-US/docs/Web/HTTP
13 options[:attribution] = <<-HTML30 https://tools.ietf.org/html/rfc2616
14 &copy; document authors. All rights reserved.31 https://tools.ietf.org/html/rfc4918
15 HTML32 https://tools.ietf.org/html/rfc7230
33 https://tools.ietf.org/html/rfc7231
34 https://tools.ietf.org/html/rfc7232
35 https://tools.ietf.org/html/rfc7233
36 https://tools.ietf.org/html/rfc7234
37 https://tools.ietf.org/html/rfc7235)
38 end
16 end39 end
17end40end