History

Update and improve PostgreSQL documentation (9.4)

Thibaut committed 11 years ago

Commit df1f2031 parent 38b16ed4 files · +125 −77Browse files

Showing the changes to lib (4 of 4 files). Show all

Unified · Split

4 files
  1. lib/docs/filters/postgresql/clean_html.rb +4 −0
  2. lib/docs/filters/postgresql/entries.rb +88 −62
  3. lib/docs/filters/postgresql/extract_metadata.rb +1 −2
  4. lib/docs/scrapers/postgresql.rb +32 −13
lib/docs/filters/postgresql/clean_html.rb+4−0
@@ -11,6 +11,10 @@ module Docs · 10 unmodified lines
11 end11 end
12 12
13 def other13 def other
14 @doc = at_css('#docContent')
15
16 css('.NAVHEADER', '.NAVFOOTER').remove
17
14 css('a[name]').each do |node|18 css('a[name]').each do |node|
15 node.parent['id'] = node['name']19 node.parent['id'] = node['name']
16 node.before(node.children).remove20 node.before(node.children).remove
lib/docs/filters/postgresql/entries.rb+88−62
@@ -11,98 +11,144 @@ module Docs · 10 unmodified lines
11 'System Administration Functions' => 'Administration Functions',11 'System Administration Functions' => 'Administration Functions',
12 'System Information Functions' => 'Information Functions' }12 'System Information Functions' => 'Information Functions' }
13 13
14 def get_name14 PREPEND_TYPES = [
15 name = at_css('h1').content15 'Type Conversion',
16 clean_heading_name(name)16 'Full Text Search',
17 'Performance Tips',
18 'Server Configuration',
19 'Monitoring' ]
20
21 REPLACE_TYPES = {
22 'Routine Database Maintenance Tasks' => 'Maintenance',
23 'High Availability, Load Balancing, and Replication' => 'High Availability',
24 'Monitoring Database Activity' => 'Monitoring',
25 'Monitoring Disk Usage' => 'Monitoring',
26 'Reliability and the Write-Ahead Log' => 'Write-Ahead Log' }
27
28 def base_name
29 @base_name ||= clean_heading_name(at_css('h1').content)
30 end
17 31
18 if %w(Overview Introduction).include?(name)32 def get_name
33 if %w(Overview Introduction).include?(base_name)
19 result[:pg_chapter_name]34 result[:pg_chapter_name]
35 elsif PREPEND_TYPES.include?(type)
36 "#{type}: #{base_name}"
20 else37 else
21 name.remove! ' (Common Table Expressions)'38 REPLACE_NAMES[base_name] || base_name
22 REPLACE_NAMES[name] || name
23 end39 end
24 end40 end
25 41
26 def clean_heading_name(name)
27 name.remove! %r{\A[\d\.\s]+}
28 name.remove! 'Using '
29 name.remove! %r{\AThe }
30 name
31 end
32
33 def get_type42 def get_type
34 return if initial_page?43 return if initial_page?
35 44
36 if result[:pg_up_path] == 'sql-commands.html'45 if result[:pg_up_path] == 'sql-commands.html'
37 'Commands'46 'Commands'
38 elsif result[:pg_up_path].start_with? 'reference-'47 elsif result[:pg_up_path].start_with?('reference-')
39 'Applications'48 'Applications'
40 elsif type = result[:pg_chapter_name]49 elsif type = result[:pg_chapter_name]
41 if type.start_with?('Func') && (match = name.match(/\A(?!Form|Seq|Set|Enum)(.+) Func/))50 if type.start_with?('Func') && (match = base_name.match(/\A(?!Form|Seq|Set|Enum)(.+) Func/))
42 "Functions: #{match[1]}"51 "Functions: #{match[1]}"
43 else52 else
44 type.remove 'SQL '53 type.remove! 'SQL '
54 REPLACE_TYPES[type] || type
45 end55 end
46 end56 end
47 end57 end
48 58
49 def additional_entries59 def additional_entries
50 return [] if skip_additional_entries?60 return [] if skip_additional_entries?
51 return get_config_entries if config_page?61 return config_additional_entries if type && type.include?('Configuration')
62 return data_types_additional_entries if type == 'Data Types'
52 return get_heading_entries('h3[id]') if slug == 'functions-xml'63 return get_heading_entries('h3[id]') if slug == 'functions-xml'
53 64
54 if type == 'Data Types'
55 return get_custom_entries case slug
56 when 'rangetypes' then 'li > p > .TYPE:first-child'
57 when 'datatype-textsearch' then '.SECT2 > .TYPE'
58 else '.CALSTABLE td:first-child > .TYPE' end
59 end
60
61 entries = get_heading_entries('h2[id]')65 entries = get_heading_entries('h2[id]')
62 66
63 if slug == 'queries-union'67 case slug
68 when 'queries-union'
64 entries.concat get_custom_entries('p > .LITERAL:first-child')69 entries.concat get_custom_entries('p > .LITERAL:first-child')
65 elsif slug == 'queries-table-expressions'70 when 'queries-table-expressions'
66 entries.concat get_heading_entries('h3[id]')71 entries.concat get_heading_entries('h3[id]')
67 entries.concat get_custom_entries('dt > .LITERAL:first-child')72 entries.concat get_custom_entries('dt > .LITERAL:first-child')
68 elsif slug == 'functions-logical'73 when 'functions-logical'
69 entries.concat get_custom_entries('> table td:first-child > code')74 entries.concat get_custom_entries('> table td:first-child > code')
70 elsif slug == 'functions-formatting'75 when 'functions-formatting'
71 entries.concat get_custom_entries('#FUNCTIONS-FORMATTING-TABLE td:first-child > code')76 entries.concat get_custom_entries('#FUNCTIONS-FORMATTING-TABLE td:first-child > code')
72 elsif slug == 'functions-admin'77 when 'functions-admin'
73 entries.concat get_custom_entries('.TABLE td:first-child > code')78 entries.concat get_custom_entries('.TABLE td:first-child > code')
74 elsif slug == 'functions-string'79 when 'functions-string'
75 entries.concat get_custom_entries('> div[id^="FUNC"] td:first-child > code')80 entries.concat get_custom_entries('> div[id^="FUNC"] td:first-child > code')
76 elsif type && type.start_with?('Functions')81 else
77 entries.concat get_custom_entries('> .TABLE td:first-child > code:first-child')82 if type && type.start_with?('Functions')
78 entries.concat get_comparison_entries if slug == 'functions-comparison'83 entries.concat get_custom_entries('> .TABLE td:first-child > code:first-child')
84 entries.concat %w(IS NULL BETWEEN DISTINCT\ FROM).map { |name| ["#{self.name}: #{name}"] } if slug == 'functions-comparison'
85 end
79 end86 end
80 87
81 entries88 entries
82 end89 end
83 90
84 def get_config_entries91 def config_additional_entries
85 css('.VARIABLELIST dt[id]').map do |node|92 css('.VARIABLELIST dt[id]').map do |node|
86 name = node.at_css('.VARNAME').content93 name = node.at_css('.VARNAME').content
87 ["Config: #{name}", node['id']]94 ["Config: #{name}", node['id']]
88 end95 end
89 end96 end
90 97
98 def data_types_additional_entries
99 selector = case slug
100 when 'rangetypes'
101 'li > p > .TYPE:first-child'
102 when 'datatype-textsearch'
103 '.SECT2 > .TYPE'
104 else
105 '.CALSTABLE td:first-child > .TYPE'
106 end
107 get_custom_entries(selector)
108 end
109
110 def include_default_entry?
111 !initial_page? && !at_css('.TOC')
112 end
113
114 SKIP_ENTRIES_SLUGS = [
115 'config-setting',
116 'applevel-consistency' ]
117
118 SKIP_ENTRIES_TYPES = [
119 'Localization',
120 'Type Conversion',
121 'Full Text Search',
122 'Performance Tips',
123 'Client Authentication',
124 'Managing Databases',
125 'Maintenance',
126 'Backup and Restore',
127 'High Availability',
128 'Monitoring' ]
129
130 def skip_additional_entries?
131 SKIP_ENTRIES_SLUGS.include?(slug) || SKIP_ENTRIES_TYPES.include?(type)
132 end
133
134 def clean_heading_name(name)
135 name.remove! %r{\A[\d\.\s]+}
136 name.remove! 'Using '
137 name.remove! %r{\AThe }
138 name.remove! ' (Common Table Expressions)'
139 name
140 end
141
91 def get_heading_entries(selector)142 def get_heading_entries(selector)
92 css(selector).inject [] do |entries, node|143 css(selector).each_with_object([]) do |node, entries|
93 name = node.content144 name = node.content
94 clean_heading_name(name)145 clean_heading_name(name)
95 146 entries << ["#{additional_entry_prefix}: #{name}", node['id']] unless skip_heading?(name)
96 unless skip_heading?(name)
97 entries << ["#{additional_entry_prefix}: #{name}", node['id']]
98 end
99
100 entries
101 end147 end
102 end148 end
103 149
104 def get_custom_entries(selector)150 def get_custom_entries(selector)
105 css(selector).inject [] do |entries, node|151 css(selector).each_with_object([]) do |node, entries|
106 name = node.content152 name = node.content
107 name.remove! %r{\(.*?\)}m153 name.remove! %r{\(.*?\)}m
108 name.remove! %r{\[.*?\]}m154 name.remove! %r{\[.*?\]}m
@@ -117,14 +163,6 @@ module Docs · 8 unmodified lines
117 node['id'] = id163 node['id'] = id
118 entries << [name, id]164 entries << [name, id]
119 end165 end
120
121 entries
122 end
123 end
124
125 def get_comparison_entries
126 %w(IS NULL BETWEEN DISTINCT\ FROM).map do |name|
127 ["#{self.name}: #{name}"]
128 end166 end
129 end167 end
130 168
@@ -132,22 +170,10 @@ module Docs · 1 unmodified line
132 type.dup.gsub!('Functions: ', '') || self.name170 type.dup.gsub!('Functions: ', '') || self.name
133 end171 end
134 172
135 def skip_additional_entries?
136 slug == 'config-setting' || %w(Concurrency\ Control Localization).include?(type)
137 end
138
139 def skip_heading?(name)173 def skip_heading?(name)
140 %w(Usage\ Patterns Portability Caveats Overview).include?(name) ||174 %w(Usage\ Patterns Portability Caveats Overview).include?(name) ||
141 (type.start_with?('Functions') && slug != 'functions-xml' && name.split.first.upcase!)175 (type.start_with?('Functions') && slug != 'functions-xml' && name.split.first.upcase!)
142 end176 end
143
144 def include_default_entry?
145 !(initial_page? || at_css('.TOC') || config_page?)
146 end
147
148 def config_page?
149 slug.start_with? 'runtime-config'
150 end
151 end177 end
152 end178 end
153end179end
lib/docs/filters/postgresql/extract_metadata.rbrenamed from lib/docs/filters/postgresql/clean_nav.rb+1−2
@@ -1,10 +1,9 @@
1module Docs1module Docs
2 class Postgresql2 class Postgresql
3 class CleanNavFilter < Filter3 class ExtractMetadataFilter < Filter
4 def call4 def call
5 extract_up_path5 extract_up_path
6 extract_chapter6 extract_chapter
7 css('.NAVHEADER', '.NAVFOOTER').remove
8 doc7 doc
9 end8 end
10 9
lib/docs/scrapers/postgresql.rb+32−13
@@ -1,14 +1,13 @@
1module Docs1module Docs
2 class Postgresql < FileScraper2 class Postgresql < UrlScraper
3 self.name = 'PostgreSQL'3 self.name = 'PostgreSQL'
4 self.type = 'postgres'4 self.type = 'postgres'
5 self.version = 'up to 9.3.2'5 self.version = '9.4'
6 self.dir = '/Users/Thibaut/DevDocs/Docs/PostgreSQL'6 self.base_url = "http://www.postgresql.org/docs/#{version}/static/"
7 self.base_url = 'http://www.postgresql.org/docs/9.3/static/'
8 self.root_path = 'reference.html'7 self.root_path = 'reference.html'
9 self.initial_paths = %w(sql.html runtime-config.html charset.html)8 self.initial_paths = %w(sql.html admin.html)
10 9
11 html_filters.insert_before 'normalize_urls', 'postgresql/clean_nav'10 html_filters.insert_before 'normalize_urls', 'postgresql/extract_metadata'
12 html_filters.push 'postgresql/clean_html', 'postgresql/entries', 'title'11 html_filters.push 'postgresql/clean_html', 'postgresql/entries', 'title'
13 12
14 options[:title] = false13 options[:title] = false
@@ -19,7 +18,6 @@ module Docs · 4 unmodified lines
19 arrays.html18 arrays.html
20 rowtypes.html19 rowtypes.html
21 rangetypes.html20 rangetypes.html
22 mvcc-intro.html
23 transaction-iso.html21 transaction-iso.html
24 explicit-locking.html22 explicit-locking.html
25 applevel-consistency.html23 applevel-consistency.html
@@ -27,7 +25,15 @@ module Docs · 1 unmodified line
27 config-setting.html25 config-setting.html
28 locale.html26 locale.html
29 collation.html27 collation.html
30 multibyte.html)28 multibyte.html
29 using-explain.html
30 planner-stats.html
31 explicit-joins.html
32 populate.html
33 non-durability.html
34 logfile-maintenance.html
35 continuous-archiving.html
36 dynamic-trace.html)
31 37
32 options[:only_patterns] = [38 options[:only_patterns] = [
33 /\Asql\-/,39 /\Asql\-/,
@@ -37,18 +43,31 @@ module Docs · 3 unmodified lines
37 /\Aqueries\-/,43 /\Aqueries\-/,
38 /\Adatatype\-/,44 /\Adatatype\-/,
39 /\Afunctions\-/,45 /\Afunctions\-/,
46 /\Atypeconv\-/,
47 /\Atextsearch\-/,
48 /\Amvcc\-/,
40 /\Aindexes\-/,49 /\Aindexes\-/,
41 /\Aruntime\-config\-/]50 /\Aruntime\-config\-/,
51 /\Aauth\-/,
52 /\Aclient\-authentication/,
53 /\Amanage\-ag/,
54 /\Aroutine/,
55 /\Abackup\-/,
56 /\Amonitoring\-/,
57 /\Awal\-/,
58 /\Adisk/,
59 /role/,
60 /recovery/,
61 /standby/]
42 62
43 options[:skip] = %w(63 options[:skip] = %w(
44 ddl-others.html64 ddl-others.html
45 runtime-config-custom.html
46 runtime-config-short.html
47 functions-event-triggers.html65 functions-event-triggers.html
48 functions-trigger.html)66 functions-trigger.html
67 textsearch-migration.html)
49 68
50 options[:attribution] = <<-HTML69 options[:attribution] = <<-HTML
51 &copy; 1996&ndash;2013 The PostgreSQL Global Development Group<br>70 &copy; 1996&ndash;2014 The PostgreSQL Global Development Group<br>
52 Licensed under the PostgreSQL License.71 Licensed under the PostgreSQL License.
53 HTML72 HTML
54 end73 end