History

Finish CodeIgniter scraper

Thibaut Courouble committed 10 years ago

Commit 0efa4c31 parent 8232a0517 files · +59 −44Browse files

Showing the changes to lib (3 of 17 files). Show all

Unified · Split

3 files
  1. lib/docs/filters/codeigniter/clean_html.rb +10 −4
  2. lib/docs/filters/codeigniter/entries.rb +22 −17
  3. lib/docs/scrapers/codeigniter.rb +19 −12
lib/docs/filters/codeigniter/clean_html.rb+10−4
@@ -8,16 +8,22 @@ module Docs · 7 unmodified lines
88 node.content = node.content
99 end
1010
11+ css('div[class^="highlight-"]').each do |node|
12+ node.content = node.content.strip
13+ node.name = 'pre'
14+ node['class'] = 'php' if node['class'].include?('highlight-ci')
15+ end
16+
1117 css('table').each do |node|
1218 node.remove_attribute 'border'
1319 end
1420
15− css('.section > h2', '.section > h3', '.section > h4', '.section > h5').each do |node|
16− node['id'] = node.parent['id']
17− node.parent.remove_attribute 'id'
21+ css('.section').each do |node|
22+ node.first_element_child['id'] = node['id'] if node['id']
23+ node.before(node.children).remove
1824 end
1925
20− doc.children
26+ doc
2127 end
2228 end
2329 end
lib/docs/filters/codeigniter/entries.rb+22−17
@@ -1,38 +1,43 @@
11 module Docs
22 class Codeigniter
33 class EntriesFilter < Docs::EntriesFilter
4− def include_default_entry?
5− not slug.ends_with? 'index'
6− end
7−
84 def get_name
95 at_css('h1').content.strip
106 end
117
128 def get_type
13− slug.split('/')[0].capitalize
9+ type = slug.split('/').first.capitalize
10+
11+ if type.in? %w(Tutorial Installation Overview General)
12+ type.prepend 'User guide: '
13+ elsif type.in? %w(Libraries Helpers)
14+ type = name.remove %r{\ (?:Helper|Class|Classes|Library|Driver)\z}
15+ end
16+
17+ type
1418 end
1519
1620 def additional_entries
1721 entries = []
1822
19− css('.class').each do |c_node|
20− c_name = c_node.at_css('dt > .descname').content
21− c_id = c_node.at_css('dt')['id']
22− entries << [c_name, c_id, get_type]
23+ css('.class').each do |node|
24+ class_name = node.at_css('dt > .descname').content
25+ class_id = node.at_css('dt[id]')['id']
26+ entries << [class_name, class_id]
2327
24− c_node.css('.method').each do |node|
25− m_name = node.at_css('.descname').content
26− name = c_name + '::' + m_name + '()'
27− id = node.at_css('dt')['id']
28− entries << [name, id, get_type]
28+ node.css('.method').each do |n|
29+ name = n.at_css('.descname').content
30+ name = "#{class_name}::#{name}()"
31+ id = node.at_css('dt[id]')['id']
32+ entries << [name, id]
2933 end
3034 end
3135
3236 css('.function').each do |node|
33− name = node.at_css('.descname').content + '()'
34− id = node.at_css('dt')['id']
35− entries << [name, id, get_type]
37+ name = "#{node.at_css('.descname').content}()"
38+ id = node.at_css('dt[id]')['id']
39+ type = self.type.start_with?('User guide') ? 'Functions' : self.type
40+ entries << [name, id, type]
3641 end
3742
3843 entries
lib/docs/scrapers/codeigniter.rb+19−12
@@ -1,12 +1,11 @@
11 module Docs
22 class Codeigniter < UrlScraper
33 self.name = 'CodeIgniter'
4− self.type = 'codeigniter'
5− self.release = '3.0'
6− self.base_url = 'http://www.codeigniter.com/user_guide/'
4+ self.type = 'sphinx'
5+ self.base_url = 'https://www.codeigniter.com/user_guide/'
76 self.root_path = 'index.html'
87 self.links = {
9− home: 'http://codeigniter.com/',
8+ home: 'https://codeigniter.com/',
109 code: 'https://github.com/bcit-ci/CodeIgniter'
1110 }
1211
@@ -14,22 +13,30 @@ module Docs · 1 unmodified line
1413
1514 options[:container] = '.document'
1615
17− options[:only_patterns] = [
18− /\Ageneral/,
19− /\Alibraries/,
20− /\Adatabase/,
21− /\Ahelpers/
22− ]
23−
2416 options[:skip] = %w(
17+ license.html
18+ changelog.html
19+ DCO.html
2520 general/welcome.html
2621 general/requirements.html
2722 general/credits.html
23+ libraries/index.html
24+ helpers/index.html
2825 )
2926
27+ options[:skip_patterns] = [
28+ /\Acontributing/,
29+ /\Adocumentation/,
30+ /\Ainstallation\/upgrade/
31+ ]
32+
3033 options[:attribution] = <<-HTML
31− &copy; British Columbia Institute of Technology<br>
34+ &copy; 2014&ndash;2016 British Columbia Institute of Technology<br>
3235 Licensed under the MIT License.
3336 HTML
37+
38+ version '3.0' do
39+ self.release = '3.0.4'
40+ end
3441 end
3542 end