History

Update PostgreSQL documentation (10.1)

Thibaut Courouble committed 9 years ago

Commit e7048a81 parent 03428546 files · +105 −43Browse files

Showing the changes to lib/docs (5 of 6 files). Show all

Unified · Split

5 files
  1. lib/docs/filters/postgresql/clean_html.rb +34 −5
  2. lib/docs/filters/postgresql/entries.rb +23 −21
  3. lib/docs/filters/postgresql/extract_metadata.rb +10 −8
  4. lib/docs/filters/postgresql/normalize_class_names.rb +13 −0
  5. lib/docs/scrapers/postgresql.rb +14 −3
lib/docs/filters/postgresql/clean_html.rb+34−5
@@ -13,18 +13,22 @@ module Docs · 12 unmodified lines
13 def other13 def other
14 @doc = at_css('#docContent')14 @doc = at_css('#docContent')
15 15
16 css('.NAVHEADER', 'hr', '.NAVFOOTER a[accesskey="H"]').remove16 css('.navheader', 'hr', '.navfooter a[accesskey="H"]').remove
17
18 unless at_css('h1')
19 at_css('.refnamediv h2, .titlepage h2').name = 'h1'
20 end
17 21
18 css('a[name]').each do |node|22 css('a[name]').each do |node|
19 node.parent['id'] = node['name']23 node.parent['id'] = node['name']
20 node.before(node.children).remove24 node.before(node.children).remove
21 end25 end
22 26
23 css('div.SECT1', 'pre > kbd', 'tt > code', 'h1 > tt', '> .CHAPTER', 'div.NOTE', '.APPENDIX').each do |node|27 css('div.sect1', '.refentry', '.refnamediv', '.refentrytitle', '.refsynopsisdiv', 'pre > kbd', 'tt > code', 'h1 > tt', '> .chapter', '.appendix', '.titlepage', 'div:not([class]):not([id])', 'br', 'a.indexterm', 'acronym', '.productname', 'div.itemizedlist', 'span.sect2', 'span.application', 'em.replaceable', 'span.term').each do |node|
24 node.before(node.children).remove28 node.before(node.children).remove
25 end29 end
26 30
27 css('div.CAUTION table.CAUTION').each do |node|31 css('div.caution table.caution').each do |node|
28 parent = node.parent32 parent = node.parent
29 title = node.at_css('.c2, .c3, .c4, .c5').content33 title = node.at_css('.c2, .c3, .c4, .c5').content
30 node.replace(node.css('p'))34 node.replace(node.css('p'))
@@ -43,11 +47,27 @@ module Docs · 12 unmodified lines
43 node.remove_attribute 'valign'47 node.remove_attribute 'valign'
44 end48 end
45 49
50 css('.sect2 > h3').each do |node|
51 node.name = 'h2'
52 end
53
54 css('.sect3 > h4').each do |node|
55 node.name = 'h3'
56 end
57
46 css('tt').each do |node|58 css('tt').each do |node|
47 node.name = 'code'59 node.name = 'code'
48 end60 end
49 61
50 css('.REFSYNOPSISDIV > p').each do |node|62 css('div.note', 'div.important', 'div.tip', 'div.caution').each do |node|
63 if node.at_css('blockquote')
64 node.before(node.children).remove
65 else
66 node.name = 'blockquote'
67 end
68 end
69
70 css('.refsynopsisdiv > p').each do |node|
51 node.name = 'pre'71 node.name = 'pre'
52 node.content = node.content72 node.content = node.content
53 end73 end
@@ -56,9 +76,18 @@ module Docs · 2 unmodified lines
56 node.before(node.children).remove76 node.before(node.children).remove
57 end77 end
58 78
59 css('pre.SYNOPSIS', 'pre.PROGRAMLISTING').each do |node|79 css('code').each do |node|
80 node.inner_html = node.inner_html.gsub(/\s*\n\s*/, ' ')
81 end
82
83 css('pre.synopsis', 'pre.programlisting').each do |node|
60 node['data-language'] = 'sql'84 node['data-language'] = 'sql'
61 end85 end
86
87 css('h1', 'ul', 'li', 'pre').each do |node|
88 node.remove_attribute 'class'
89 node.remove_attribute 'style'
90 end
62 end91 end
63 end92 end
64 end93 end
lib/docs/filters/postgresql/entries.rb+23−21
@@ -90,33 +90,33 @@ module Docs · 89 unmodified lines
90 return config_additional_entries if type && type.include?('Configuration')90 return config_additional_entries if type && type.include?('Configuration')
91 return data_types_additional_entries if type == 'Data Types'91 return data_types_additional_entries if type == 'Data Types'
92 return command_additional_entries if type == 'Commands'92 return command_additional_entries if type == 'Commands'
93 return get_heading_entries('h3[id]') if slug == 'functions-xml'93 return get_heading_entries('h3[id], .sect3[id] > h3:first-child') if slug == 'functions-xml'
94 94
95 entries = get_heading_entries('h2[id]')95 entries = get_heading_entries('h2[id], .sect2[id] > h2:first-child')
96 96
97 case slug97 case slug
98 when 'queries-union'98 when 'queries-union'
99 entries.concat get_custom_entries('p > .LITERAL:first-child')99 entries.concat get_custom_entries('p > .literal:first-child')
100 when 'queries-table-expressions'100 when 'queries-table-expressions'
101 entries.concat get_heading_entries('h3[id]')101 entries.concat get_heading_entries('h3[id], .sect3[id] > h3:first-child')
102 entries.concat get_custom_entries('dt > .LITERAL:first-child')102 entries.concat get_custom_entries('dt > .literal:first-child')
103 when 'functions-logical'103 when 'functions-logical'
104 entries.concat get_custom_entries('> table td:first-child > code')104 entries.concat get_custom_entries('> table td:first-child > code')
105 when 'functions-formatting'105 when 'functions-formatting'
106 entries.concat get_custom_entries('#FUNCTIONS-FORMATTING-TABLE td:first-child > code')106 entries.concat get_custom_entries('#FUNCTIONS-FORMATTING-TABLE td:first-child > code')
107 when 'functions-admin'107 when 'functions-admin'
108 entries.concat get_custom_entries('.TABLE td:first-child > code')108 entries.concat get_custom_entries('.table td:first-child > code')
109 when 'functions-string'109 when 'functions-string'
110 entries.concat get_custom_entries('> div[id^="FUNC"] td:first-child > code')110 entries.concat get_custom_entries('> div[id^="FUNC"] td:first-child > code')
111 entries.concat get_custom_entries('> div[id^="FORMAT"] td:first-child > code')111 entries.concat get_custom_entries('> div[id^="FORMAT"] td:first-child > code')
112 else112 else
113 if type && type.start_with?('Functions')113 if type && type.start_with?('Functions')
114 entries.concat get_custom_entries('> .TABLE td:first-child > code.LITERAL:first-child')114 entries.concat get_custom_entries('> .table td:first-child > code.literal:first-child')
115 entries.concat get_custom_entries('> .TABLE td:first-child > code.FUNCTION:first-child')115 entries.concat get_custom_entries('> .table td:first-child > code.function:first-child')
116 entries.concat get_custom_entries('> .TABLE td:first-child > code:not(.LITERAL):first-child + code.LITERAL')116 entries.concat get_custom_entries('> .table td:first-child > code:not(.literal):first-child + code.literal')
117 entries.concat get_custom_entries('> .TABLE td:first-child > p > code.LITERAL:first-child')117 entries.concat get_custom_entries('> .table td:first-child > p > code.literal:first-child')
118 entries.concat get_custom_entries('> .TABLE td:first-child > p > code.FUNCTION:first-child')118 entries.concat get_custom_entries('> .table td:first-child > p > code.function:first-child')
119 entries.concat get_custom_entries('> .TABLE td:first-child > p > code:not(.LITERAL):first-child + code.LITERAL')119 entries.concat get_custom_entries('> .table td:first-child > p > code:not(.literal):first-child + code.literal')
120 if slug == 'functions-comparison' && !at_css('#FUNCTIONS-COMPARISON-PRED-TABLE') # before 9.6120 if slug == 'functions-comparison' && !at_css('#FUNCTIONS-COMPARISON-PRED-TABLE') # before 9.6
121 entries.concat %w(IS NULL BETWEEN DISTINCT\ FROM).map { |name| ["#{self.name}: #{name}"] }121 entries.concat %w(IS NULL BETWEEN DISTINCT\ FROM).map { |name| ["#{self.name}: #{name}"] }
122 end122 end
@@ -127,8 +127,8 @@ module Docs · 4 unmodified lines
127 end127 end
128 128
129 def config_additional_entries129 def config_additional_entries
130 css('.VARIABLELIST dt[id]').map do |node|130 css('.variablelist dt[id]').map do |node|
131 name = node.at_css('.VARNAME').content131 name = node.at_css('.varname').content
132 ["Config: #{name}", node['id']]132 ["Config: #{name}", node['id']]
133 end133 end
134 end134 end
@@ -136,27 +136,27 @@ module Docs · 1 unmodified line
136 def data_types_additional_entries136 def data_types_additional_entries
137 selector = case slug137 selector = case slug
138 when 'rangetypes'138 when 'rangetypes'
139 'li > p > .TYPE:first-child'139 'li > p > .type:first-child'
140 when 'datatype-textsearch'140 when 'datatype-textsearch'
141 '.SECT2 > .TYPE'141 '.title > .type, .sect2 > .type'
142 else142 else
143 '.CALSTABLE td:first-child > .TYPE'143 '.table-contents td:first-child > .type, .calstable td:first-child > .type'
144 end144 end
145 get_custom_entries(selector)145 get_custom_entries(selector)
146 end146 end
147 147
148 def command_additional_entries148 def command_additional_entries
149 css('.REFSECT2[id^="SQL"]').each_with_object([]) do |node, entries|149 css('.refsect2[id^="SQL"]').each_with_object([]) do |node, entries|
150 next unless heading = node.at_css('h3')150 next unless heading = node.at_css('h3')
151 next unless heading.content.strip =~ /[A-Z_\-]+ Clause/151 next unless heading.content.strip =~ /[A-Z_\-]+ Clause/
152 name = heading.at_css('.LITERAL').content152 name = heading.at_css('.literal').content
153 name.prepend "#{self.name} ... "153 name.prepend "#{self.name} ... "
154 entries << [name, node['id']]154 entries << [name, node['id']]
155 end155 end
156 end156 end
157 157
158 def include_default_entry?158 def include_default_entry?
159 !initial_page? && !at_css('.TOC') && type159 !initial_page? && (!at_css('.toc') || at_css('.sect2, .variablelist, .refsect1')) && type
160 end160 end
161 161
162 SKIP_ENTRIES_SLUGS = [162 SKIP_ENTRIES_SLUGS = [
@@ -199,7 +199,9 @@ module Docs · 36 unmodified lines
199 css(selector).each_with_object([]) do |node, entries|199 css(selector).each_with_object([]) do |node, entries|
200 name = node.content200 name = node.content
201 clean_heading_name(name)201 clean_heading_name(name)
202 entries << ["#{additional_entry_prefix}: #{name}", node['id']] unless skip_heading?(name)202 id = node['id'] || node.parent['id']
203 raise "missing ids for selector #{selector}" unless id
204 entries << ["#{additional_entry_prefix}: #{name}", id] unless skip_heading?(name)
203 end205 end
204 end206 end
205 207
lib/docs/filters/postgresql/extract_metadata.rb+10−8
@@ -8,19 +8,21 @@ module Docs · 7 unmodified lines
8 end8 end
9 9
10 def extract_up_path10 def extract_up_path
11 if node = at_css('.NAVHEADER a[accesskey="U"]')11 if node = at_css('.navheader a[accesskey="u"], .navheader a[accesskey="U"]')
12 result[:pg_up_path] = node['href']12 result[:pg_up_path] = node['href']
13 end13 end
14 end14 end
15 15
16 def extract_chapter16 def extract_chapter
17 return unless text = at_css('.NAVHEADER td[align="center"]').content17 css('.navheader td[align="center"], .navheader th[align="center"]').each do |node|
18 if match = text.match(/\AChapter (\d+)\. (.+)\z/)18 text = node.content.strip
19 result[:pg_chapter] = match[1].to_i19 if match = text.match(/\AChapter (\d+)\. (.+)\z/)
20 result[:pg_chapter_name] = match[2].strip20 result[:pg_chapter] = match[1].to_i
21 elsif match = text.match(/\AAppendix ([A-Z])\. (.+)\z/)21 result[:pg_chapter_name] = match[2].strip
22 result[:pg_appendix] = match[1]22 elsif match = text.match(/\AAppendix ([A-Z])\. (.+)\z/)
23 result[:pg_appendix_name] = match[2].strip23 result[:pg_appendix] = match[1]
24 result[:pg_appendix_name] = match[2].strip
25 end
24 end26 end
25 end27 end
26 end28 end
lib/docs/filters/postgresql/normalize_class_names.rb+13−0
@@ -0,0 +1,13 @@
1module Docs
2 class Postgresql
3 class NormalizeClassNamesFilter < Filter
4 def call
5 doc.css('*').each do |node|
6 node['class'] = node['class'].downcase if node['class'].present?
7 end
8
9 doc
10 end
11 end
12 end
13end
lib/docs/scrapers/postgresql.rb+14−3
@@ -55,19 +55,30 @@ module Docs · 54 unmodified lines
55 Licensed under the PostgreSQL License.55 Licensed under the PostgreSQL License.
56 HTML56 HTML
57 57
58 version '10' do
59 self.release = '10.1'
60 self.base_url = 'https://www.postgresql.org/docs/10/static/'
61 end
62
58 version '9.6' do63 version '9.6' do
59 self.release = '9.6.5'64 self.release = '9.6.6'
60 self.base_url = 'https://www.postgresql.org/docs/9.6/static/'65 self.base_url = 'https://www.postgresql.org/docs/9.6/static/'
66
67 html_filters.insert_before 'postgresql/extract_metadata', 'postgresql/normalize_class_names'
61 end68 end
62 69
63 version '9.5' do70 version '9.5' do
64 self.release = '9.5.9'71 self.release = '9.5.10'
65 self.base_url = 'https://www.postgresql.org/docs/9.5/static/'72 self.base_url = 'https://www.postgresql.org/docs/9.5/static/'
73
74 html_filters.insert_before 'postgresql/extract_metadata', 'postgresql/normalize_class_names'
66 end75 end
67 76
68 version '9.4' do77 version '9.4' do
69 self.release = '9.4.14'78 self.release = '9.4.15'
70 self.base_url = 'https://www.postgresql.org/docs/9.4/static/'79 self.base_url = 'https://www.postgresql.org/docs/9.4/static/'
80
81 html_filters.insert_before 'postgresql/extract_metadata', 'postgresql/normalize_class_names'
71 end82 end
72 end83 end
73end84end