Update and improve PostgreSQL documentation (9.4)

Thibaut committed 11 years ago

Commit df1f2031 parent 38b16ed4 files · +125 −77Browse files

Unified · Split

4 files

lib/docs/filters/postgresql/clean_html.rb +4 −0
lib/docs/filters/postgresql/entries.rb +88 −62
lib/docs/filters/postgresql/extract_metadata.rb +1 −2
lib/docs/scrapers/postgresql.rb +32 −13

lib/docs/filters/postgresql/clean_html.rb+4−0

@@ -11,6 +11,10 @@ module Docs · 10 unmodified lines
11	11	end
12	12
13	13	def other
	14	+ @doc = at_css('#docContent')
	15	+
	16	+ css('.NAVHEADER', '.NAVFOOTER').remove
	17	+
14	18	css('a[name]').each do \|node\|
15	19	node.parent['id'] = node['name']
16	20	node.before(node.children).remove

lib/docs/filters/postgresql/entries.rb+88−62

@@ -11,98 +11,144 @@ module Docs · 10 unmodified lines
11	11	'System Administration Functions' => 'Administration Functions',
12	12	'System Information Functions' => 'Information Functions' }
13	13
14		− def get_name
15		− name = at_css('h1').content
16		− clean_heading_name(name)
	14	+ PREPEND_TYPES = [
	15	+ 'Type Conversion',
	16	+ 'Full Text Search',
	17	+ 'Performance Tips',
	18	+ 'Server Configuration',
	19	+ 'Monitoring' ]
	20	+
	21	+ REPLACE_TYPES = {
	22	+ 'Routine Database Maintenance Tasks' => 'Maintenance',
	23	+ 'High Availability, Load Balancing, and Replication' => 'High Availability',
	24	+ 'Monitoring Database Activity' => 'Monitoring',
	25	+ 'Monitoring Disk Usage' => 'Monitoring',
	26	+ 'Reliability and the Write-Ahead Log' => 'Write-Ahead Log' }
	27	+
	28	+ def base_name
	29	+ @base_name \|\|= clean_heading_name(at_css('h1').content)
	30	+ end
17	31
18		− if %w(Overview Introduction).include?(name)
	32	+ def get_name
	33	+ if %w(Overview Introduction).include?(base_name)
19	34	result[:pg_chapter_name]
	35	+ elsif PREPEND_TYPES.include?(type)
	36	+ "#{type}: #{base_name}"
20	37	else
21		− name.remove! ' (Common Table Expressions)'
22		− REPLACE_NAMES[name] \|\| name
	38	+ REPLACE_NAMES[base_name] \|\| base_name
23	39	end
24	40	end
25	41
26		− def clean_heading_name(name)
27		− name.remove! %r{\A[\d\.\s]+}
28		− name.remove! 'Using '
29		− name.remove! %r{\AThe }
30		− name
31		− end
32		−
33	42	def get_type
34	43	return if initial_page?
35	44
36	45	if result[:pg_up_path] == 'sql-commands.html'
37	46	'Commands'
38		− elsif result[:pg_up_path].start_with? 'reference-'
	47	+ elsif result[:pg_up_path].start_with?('reference-')
39	48	'Applications'
40	49	elsif type = result[:pg_chapter_name]
41		− if type.start_with?('Func') && (match = name.match(/\A(?!Form\|Seq\|Set\|Enum)(.+) Func/))
	50	+ if type.start_with?('Func') && (match = base_name.match(/\A(?!Form\|Seq\|Set\|Enum)(.+) Func/))
42	51	"Functions: #{match[1]}"
43	52	else
44		− type.remove 'SQL '
	53	+ type.remove! 'SQL '
	54	+ REPLACE_TYPES[type] \|\| type
45	55	end
46	56	end
47	57	end
48	58
49	59	def additional_entries
50	60	return [] if skip_additional_entries?
51		− return get_config_entries if config_page?
	61	+ return config_additional_entries if type && type.include?('Configuration')
	62	+ return data_types_additional_entries if type == 'Data Types'
52	63	return get_heading_entries('h3[id]') if slug == 'functions-xml'
53	64
54		− if type == 'Data Types'
55		− return get_custom_entries case slug
56		− when 'rangetypes' then 'li > p > .TYPE:first-child'
57		− when 'datatype-textsearch' then '.SECT2 > .TYPE'
58		− else '.CALSTABLE td:first-child > .TYPE' end
59		− end
60		−
61	65	entries = get_heading_entries('h2[id]')
62	66
63		− if slug == 'queries-union'
	67	+ case slug
	68	+ when 'queries-union'
64	69	entries.concat get_custom_entries('p > .LITERAL:first-child')
65		− elsif slug == 'queries-table-expressions'
	70	+ when 'queries-table-expressions'
66	71	entries.concat get_heading_entries('h3[id]')
67	72	entries.concat get_custom_entries('dt > .LITERAL:first-child')
68		− elsif slug == 'functions-logical'
	73	+ when 'functions-logical'
69	74	entries.concat get_custom_entries('> table td:first-child > code')
70		− elsif slug == 'functions-formatting'
	75	+ when 'functions-formatting'
71	76	entries.concat get_custom_entries('#FUNCTIONS-FORMATTING-TABLE td:first-child > code')
72		− elsif slug == 'functions-admin'
	77	+ when 'functions-admin'
73	78	entries.concat get_custom_entries('.TABLE td:first-child > code')
74		− elsif slug == 'functions-string'
	79	+ when 'functions-string'
75	80	entries.concat get_custom_entries('> div[id^="FUNC"] td:first-child > code')
76		− elsif type && type.start_with?('Functions')
77		− entries.concat get_custom_entries('> .TABLE td:first-child > code:first-child')
78		− entries.concat get_comparison_entries if slug == 'functions-comparison'
	81	+ else
	82	+ if type && type.start_with?('Functions')
	83	+ entries.concat get_custom_entries('> .TABLE td:first-child > code:first-child')
	84	+ entries.concat %w(IS NULL BETWEEN DISTINCT\ FROM).map { \|name\| ["#{self.name}: #{name}"] } if slug == 'functions-comparison'
	85	+ end
79	86	end
80	87
81	88	entries
82	89	end
83	90
84		− def get_config_entries
	91	+ def config_additional_entries
85	92	css('.VARIABLELIST dt[id]').map do \|node\|
86	93	name = node.at_css('.VARNAME').content
87	94	["Config: #{name}", node['id']]
88	95	end
89	96	end
90	97
	98	+ def data_types_additional_entries
	99	+ selector = case slug
	100	+ when 'rangetypes'
	101	+ 'li > p > .TYPE:first-child'
	102	+ when 'datatype-textsearch'
	103	+ '.SECT2 > .TYPE'
	104	+ else
	105	+ '.CALSTABLE td:first-child > .TYPE'
	106	+ end
	107	+ get_custom_entries(selector)
	108	+ end
	109	+
	110	+ def include_default_entry?
	111	+ !initial_page? && !at_css('.TOC')
	112	+ end
	113	+
	114	+ SKIP_ENTRIES_SLUGS = [
	115	+ 'config-setting',
	116	+ 'applevel-consistency' ]
	117	+
	118	+ SKIP_ENTRIES_TYPES = [
	119	+ 'Localization',
	120	+ 'Type Conversion',
	121	+ 'Full Text Search',
	122	+ 'Performance Tips',
	123	+ 'Client Authentication',
	124	+ 'Managing Databases',
	125	+ 'Maintenance',
	126	+ 'Backup and Restore',
	127	+ 'High Availability',
	128	+ 'Monitoring' ]
	129	+
	130	+ def skip_additional_entries?
	131	+ SKIP_ENTRIES_SLUGS.include?(slug) \|\| SKIP_ENTRIES_TYPES.include?(type)
	132	+ end
	133	+
	134	+ def clean_heading_name(name)
	135	+ name.remove! %r{\A[\d\.\s]+}
	136	+ name.remove! 'Using '
	137	+ name.remove! %r{\AThe }
	138	+ name.remove! ' (Common Table Expressions)'
	139	+ name
	140	+ end
	141	+
91	142	def get_heading_entries(selector)
92		− css(selector).inject [] do \|entries, node\|
	143	+ css(selector).each_with_object([]) do \|node, entries\|
93	144	name = node.content
94	145	clean_heading_name(name)
95		−
96		− unless skip_heading?(name)
97		− entries << ["#{additional_entry_prefix}: #{name}", node['id']]
98		− end
99		−
100		− entries
	146	+ entries << ["#{additional_entry_prefix}: #{name}", node['id']] unless skip_heading?(name)
101	147	end
102	148	end
103	149
104	150	def get_custom_entries(selector)
105		− css(selector).inject [] do \|entries, node\|
	151	+ css(selector).each_with_object([]) do \|node, entries\|
106	152	name = node.content
107	153	name.remove! %r{\(.*?\)}m
108	154	name.remove! %r{\[.*?\]}m
@@ -117,14 +163,6 @@ module Docs · 8 unmodified lines
117	163	node['id'] = id
118	164	entries << [name, id]
119	165	end
120		−
121		− entries
122		− end
123		− end
124		−
125		− def get_comparison_entries
126		− %w(IS NULL BETWEEN DISTINCT\ FROM).map do \|name\|
127		− ["#{self.name}: #{name}"]
128	166	end
129	167	end
130	168
@@ -132,22 +170,10 @@ module Docs · 1 unmodified line
132	170	type.dup.gsub!('Functions: ', '') \|\| self.name
133	171	end
134	172
135		− def skip_additional_entries?
136		− slug == 'config-setting' \|\| %w(Concurrency\ Control Localization).include?(type)
137		− end
138		−
139	173	def skip_heading?(name)
140	174	%w(Usage\ Patterns Portability Caveats Overview).include?(name) \|\|
141	175	(type.start_with?('Functions') && slug != 'functions-xml' && name.split.first.upcase!)
142	176	end
143		−
144		− def include_default_entry?
145		− !(initial_page? \|\| at_css('.TOC') \|\| config_page?)
146		− end
147		−
148		− def config_page?
149		− slug.start_with? 'runtime-config'
150		− end
151	177	end
152	178	end
153	179	end

lib/docs/filters/postgresql/extract_metadata.rbrenamed from lib/docs/filters/postgresql/clean_nav.rb+1−2

@@ -1,10 +1,9 @@
1	1	module Docs
2	2	class Postgresql
3		− class CleanNavFilter < Filter
	3	+ class ExtractMetadataFilter < Filter
4	4	def call
5	5	extract_up_path
6	6	extract_chapter
7		− css('.NAVHEADER', '.NAVFOOTER').remove
8	7	doc
9	8	end
10	9

lib/docs/scrapers/postgresql.rb+32−13

@@ -1,14 +1,13 @@
1	1	module Docs
2		− class Postgresql < FileScraper
	2	+ class Postgresql < UrlScraper
3	3	self.name = 'PostgreSQL'
4	4	self.type = 'postgres'
5		− self.version = 'up to 9.3.2'
6		− self.dir = '/Users/Thibaut/DevDocs/Docs/PostgreSQL'
7		− self.base_url = 'http://www.postgresql.org/docs/9.3/static/'
	5	+ self.version = '9.4'
	6	+ self.base_url = "http://www.postgresql.org/docs/#{version}/static/"
8	7	self.root_path = 'reference.html'
9		− self.initial_paths = %w(sql.html runtime-config.html charset.html)
	8	+ self.initial_paths = %w(sql.html admin.html)
10	9
11		− html_filters.insert_before 'normalize_urls', 'postgresql/clean_nav'
	10	+ html_filters.insert_before 'normalize_urls', 'postgresql/extract_metadata'
12	11	html_filters.push 'postgresql/clean_html', 'postgresql/entries', 'title'
13	12
14	13	options[:title] = false
@@ -19,7 +18,6 @@ module Docs · 4 unmodified lines
19	18	arrays.html
20	19	rowtypes.html
21	20	rangetypes.html
22		− mvcc-intro.html
23	21	transaction-iso.html
24	22	explicit-locking.html
25	23	applevel-consistency.html
@@ -27,7 +25,15 @@ module Docs · 1 unmodified line
27	25	config-setting.html
28	26	locale.html
29	27	collation.html
30		− multibyte.html)
	28	+ multibyte.html
	29	+ using-explain.html
	30	+ planner-stats.html
	31	+ explicit-joins.html
	32	+ populate.html
	33	+ non-durability.html
	34	+ logfile-maintenance.html
	35	+ continuous-archiving.html
	36	+ dynamic-trace.html)
31	37
32	38	options[:only_patterns] = [
33	39	/\Asql\-/,
@@ -37,18 +43,31 @@ module Docs · 3 unmodified lines
37	43	/\Aqueries\-/,
38	44	/\Adatatype\-/,
39	45	/\Afunctions\-/,
	46	+ /\Atypeconv\-/,
	47	+ /\Atextsearch\-/,
	48	+ /\Amvcc\-/,
40	49	/\Aindexes\-/,
41		− /\Aruntime\-config\-/]
	50	+ /\Aruntime\-config\-/,
	51	+ /\Aauth\-/,
	52	+ /\Aclient\-authentication/,
	53	+ /\Amanage\-ag/,
	54	+ /\Aroutine/,
	55	+ /\Abackup\-/,
	56	+ /\Amonitoring\-/,
	57	+ /\Awal\-/,
	58	+ /\Adisk/,
	59	+ /role/,
	60	+ /recovery/,
	61	+ /standby/]
42	62
43	63	options[:skip] = %w(
44	64	ddl-others.html
45		− runtime-config-custom.html
46		− runtime-config-short.html
47	65	functions-event-triggers.html
48		− functions-trigger.html)
	66	+ functions-trigger.html
	67	+ textsearch-migration.html)
49	68
50	69	options[:attribution] = <<-HTML
51		− © 1996–2013 The PostgreSQL Global Development Group<br>
	70	+ © 1996–2014 The PostgreSQL Global Development Group<br>
52	71	Licensed under the PostgreSQL License.
53	72	HTML
54	73	end