History

Finish scikit-image scraper

Thibaut Courouble committed 10 years ago

Commit 50f58321 parent 7f06c9c11 files · +30 −20Browse files

Showing the changes to lib/docs (4 of 11 files). Show all

Unified · Split

4 files
  1. lib/docs/filters/scikit_image/clean_html.rb +0 −11
  2. lib/docs/filters/scikit_image/entries.rb +19 −4
  3. lib/docs/filters/sphinx/clean_html.rb +1 −1
  4. lib/docs/scrapers/scikit_image.rb +7 −7
lib/docs/filters/scikit_image/clean_html.rb+0−11
@@ -1,11 +0,0 @@
1module Docs
2 class ScikitImage
3 class CleanHtmlFilter < Filter
4 def call
5 css('h2').remove
6 css('h1 + table').remove
7 doc
8 end
9 end
10 end
11end
lib/docs/filters/scikit_image/entries.rb+19−4
@@ -4,17 +4,27 @@ module Docs · 3 unmodified lines
4 def get_name4 def get_name
5 name = at_css('h1').content.strip5 name = at_css('h1').content.strip
6 name.remove! "\u{00b6}"6 name.remove! "\u{00b6}"
7 name.remove! 'Module: '7
8 name.remove! %r{ \(.*\)}8 if slug.start_with?('api')
9 name.downcase!9 name.remove! 'Module: '
10 name.remove! %r{ \(.*\)}
11 name.downcase!
12 end
13
10 name14 name
11 end15 end
12 16
13 def get_type17 def get_type
14 name.split('.').first18 if slug.start_with?('api')
19 name.split('.').first
20 else
21 'Guide'
22 end
15 end23 end
16 24
17 def additional_entries25 def additional_entries
26 return [] unless slug.start_with?('api')
27
18 entries = []28 entries = []
19 29
20 css('.class > dt[id]', '.exception > dt[id]', '.attribute > dt[id]').each do |node|30 css('.class > dt[id]', '.exception > dt[id]', '.attribute > dt[id]').each do |node|
@@ -30,8 +40,13 @@ module Docs · 9 unmodified lines
30 css('.function > dt[id]', '.method > dt[id]', '.classmethod > dt[id]').each do |node|40 css('.function > dt[id]', '.method > dt[id]', '.classmethod > dt[id]').each do |node|
31 entries << [node['id'].remove('skimage.') + '()', node['id']]41 entries << [node['id'].remove('skimage.') + '()', node['id']]
32 end42 end
43
33 entries44 entries
34 end45 end
46
47 def include_default_entry?
48 slug != 'api/api'
49 end
35 end50 end
36 end51 end
37end52end
lib/docs/filters/sphinx/clean_html.rb+1−1
@@ -9,7 +9,7 @@ module Docs · 8 unmodified lines
9 node.remove9 node.remove
10 end10 end
11 11
12 css('em.xref', 'tt').each do |node|12 css('em.xref', 'tt', 'cite').each do |node|
13 node.name = 'code'13 node.name = 'code'
14 end14 end
15 15
lib/docs/scrapers/scikit_image.rb+7−7
@@ -3,22 +3,22 @@ module Docs · 2 unmodified lines
3 self.name = 'scikit-image'3 self.name = 'scikit-image'
4 self.slug = 'scikit_image'4 self.slug = 'scikit_image'
5 self.type = 'sphinx'5 self.type = 'sphinx'
6 self.release = '0.12.2'6 self.release = '0.12.3'
7 self.base_url = 'http://scikit-image.org/docs/0.12.x/api/'7 self.base_url = 'http://scikit-image.org/docs/0.12.x/'
8 self.root_path = 'api.html'
9
10 self.links = {8 self.links = {
11 home: 'http://scikit-image.org/',9 home: 'http://scikit-image.org/',
12 code: 'https://github.com/scikit-image/scikit-image'10 code: 'https://github.com/scikit-image/scikit-image'
13 }11 }
14 12
15 html_filters.push 'scikit_image/entries', 'scikit_image/clean_html', 'sphinx/clean_html'13 html_filters.push 'scikit_image/entries', 'sphinx/clean_html'
16 14
17 options[:container] = '.span9'15 options[:container] = '.span9'
16 options[:skip] = %w(api_changes.html)
17 options[:only_patterns] = [/\Aapi/, /\Auser_guide/]
18 18
19 options[:attribution] = <<-HTML19 options[:attribution] = <<-HTML
20 &copy; 2011 the scikit-image development team<br>20 &copy; 2011 the scikit-image team<br>
21 Licensed under the scikit-image License.21 Licensed under the BSD 3-clause License.
22 HTML22 HTML
23 end23 end
24end24end