History

Finish CodeIgniter scraper

Thibaut Courouble committed 10 years ago

Commit 0efa4c31 parent 8232a0517 files · +59 −44Browse files

Showing the changes to lib (3 of 17 files). Show all

Unified · Split

3 files
  1. lib/docs/filters/codeigniter/clean_html.rb +10 −4
  2. lib/docs/filters/codeigniter/entries.rb +22 −17
  3. lib/docs/scrapers/codeigniter.rb +19 −12
lib/docs/filters/codeigniter/clean_html.rb+10−4
@@ -8,16 +8,22 @@ module Docs · 7 unmodified lines
8 node.content = node.content8 node.content = node.content
9 end9 end
10 10
11 css('div[class^="highlight-"]').each do |node|
12 node.content = node.content.strip
13 node.name = 'pre'
14 node['class'] = 'php' if node['class'].include?('highlight-ci')
15 end
16
11 css('table').each do |node|17 css('table').each do |node|
12 node.remove_attribute 'border'18 node.remove_attribute 'border'
13 end19 end
14 20
15 css('.section > h2', '.section > h3', '.section > h4', '.section > h5').each do |node|21 css('.section').each do |node|
16 node['id'] = node.parent['id']22 node.first_element_child['id'] = node['id'] if node['id']
17 node.parent.remove_attribute 'id'23 node.before(node.children).remove
18 end24 end
19 25
20 doc.children26 doc
21 end27 end
22 end28 end
23 end29 end
lib/docs/filters/codeigniter/entries.rb+22−17
@@ -1,38 +1,43 @@
1module Docs1module Docs
2 class Codeigniter2 class Codeigniter
3 class EntriesFilter < Docs::EntriesFilter3 class EntriesFilter < Docs::EntriesFilter
4 def include_default_entry?
5 not slug.ends_with? 'index'
6 end
7
8 def get_name4 def get_name
9 at_css('h1').content.strip5 at_css('h1').content.strip
10 end6 end
11 7
12 def get_type8 def get_type
13 slug.split('/')[0].capitalize9 type = slug.split('/').first.capitalize
10
11 if type.in? %w(Tutorial Installation Overview General)
12 type.prepend 'User guide: '
13 elsif type.in? %w(Libraries Helpers)
14 type = name.remove %r{\ (?:Helper|Class|Classes|Library|Driver)\z}
15 end
16
17 type
14 end18 end
15 19
16 def additional_entries20 def additional_entries
17 entries = []21 entries = []
18 22
19 css('.class').each do |c_node|23 css('.class').each do |node|
20 c_name = c_node.at_css('dt > .descname').content24 class_name = node.at_css('dt > .descname').content
21 c_id = c_node.at_css('dt')['id']25 class_id = node.at_css('dt[id]')['id']
22 entries << [c_name, c_id, get_type]26 entries << [class_name, class_id]
23 27
24 c_node.css('.method').each do |node|28 node.css('.method').each do |n|
25 m_name = node.at_css('.descname').content29 name = n.at_css('.descname').content
26 name = c_name + '::' + m_name + '()'30 name = "#{class_name}::#{name}()"
27 id = node.at_css('dt')['id']31 id = node.at_css('dt[id]')['id']
28 entries << [name, id, get_type]32 entries << [name, id]
29 end33 end
30 end34 end
31 35
32 css('.function').each do |node|36 css('.function').each do |node|
33 name = node.at_css('.descname').content + '()'37 name = "#{node.at_css('.descname').content}()"
34 id = node.at_css('dt')['id']38 id = node.at_css('dt[id]')['id']
35 entries << [name, id, get_type]39 type = self.type.start_with?('User guide') ? 'Functions' : self.type
40 entries << [name, id, type]
36 end41 end
37 42
38 entries43 entries
lib/docs/scrapers/codeigniter.rb+19−12
@@ -1,12 +1,11 @@
1module Docs1module Docs
2 class Codeigniter < UrlScraper2 class Codeigniter < UrlScraper
3 self.name = 'CodeIgniter'3 self.name = 'CodeIgniter'
4 self.type = 'codeigniter'4 self.type = 'sphinx'
5 self.release = '3.0'5 self.base_url = 'https://www.codeigniter.com/user_guide/'
6 self.base_url = 'http://www.codeigniter.com/user_guide/'
7 self.root_path = 'index.html'6 self.root_path = 'index.html'
8 self.links = {7 self.links = {
9 home: 'http://codeigniter.com/',8 home: 'https://codeigniter.com/',
10 code: 'https://github.com/bcit-ci/CodeIgniter'9 code: 'https://github.com/bcit-ci/CodeIgniter'
11 }10 }
12 11
@@ -14,22 +13,30 @@ module Docs · 1 unmodified line
14 13
15 options[:container] = '.document'14 options[:container] = '.document'
16 15
17 options[:only_patterns] = [
18 /\Ageneral/,
19 /\Alibraries/,
20 /\Adatabase/,
21 /\Ahelpers/
22 ]
23
24 options[:skip] = %w(16 options[:skip] = %w(
17 license.html
18 changelog.html
19 DCO.html
25 general/welcome.html20 general/welcome.html
26 general/requirements.html21 general/requirements.html
27 general/credits.html22 general/credits.html
23 libraries/index.html
24 helpers/index.html
28 )25 )
29 26
27 options[:skip_patterns] = [
28 /\Acontributing/,
29 /\Adocumentation/,
30 /\Ainstallation\/upgrade/
31 ]
32
30 options[:attribution] = <<-HTML33 options[:attribution] = <<-HTML
31 &copy; British Columbia Institute of Technology<br>34 &copy; 2014&ndash;2016 British Columbia Institute of Technology<br>
32 Licensed under the MIT License.35 Licensed under the MIT License.
33 HTML36 HTML
37
38 version '3.0' do
39 self.release = '3.0.4'
40 end
34 end41 end
35end42end