History

Finish Drupal scraper

Thibaut committed 11 years ago

Commit 79822f81 parent dac9b9b14 files · +56 −52Browse files

Showing the changes to lib/docs (4 of 14 files). Show all

Unified · Split

4 files
  1. lib/docs/filters/core/normalize_paths.rb +5 −0
  2. lib/docs/filters/drupal/clean_html.rb +15 −3
  3. lib/docs/filters/drupal/entries.rb +4 −9
  4. lib/docs/scrapers/drupal.rb +18 −31
lib/docs/filters/core/normalize_paths.rb+5−0
@@ -35,6 +35,11 @@ module Docs · 34 unmodified lines
3535 def normalize_path(path)
3636 path = path.downcase
3737
38+ if context[:decode_and_clean_paths]
39+ path = URI.unescape(path)
40+ path.gsub! %r{[!;:]+}, '-'
41+ end
42+
3843 if path == '.'
3944 'index'
4045 elsif path.end_with? '/'
lib/docs/filters/drupal/clean_html.rb+15−3
@@ -11,11 +11,23 @@ module Docs · 10 unmodified lines
1111 end
1212
1313 def other
14− css('#page-title-tools', '.element-invisible', '.breadcrumb', '#sidebar-first', '#api-alternatives').remove
15− css('#aside', '#api-function-signature tr:not(.active)', '.comments').remove
14+ css('.element-invisible', '#sidebar-first', '#api-alternatives', '#aside', '.comments', '.view-filters',
15+ '#api-function-signature tr:not(.active)', '.ctools-collapsible-container', 'img[width="13"]').remove
16+
17+ at_css('#main').replace(at_css('.content'))
18+ at_css('#page-heading').replace(at_css('#page-subtitle'))
19+
20+ css('th.views-field > a', '.content').each do |node|
21+ node.before(node.children).remove
22+ end
23+
24+ css('pre').each do |node|
25+ node.content = node.content
26+ end
27+
1628 # Replaces the signature table from api.drupal.org with a simple pre tag
1729 css('#api-function-signature').each do |table|
18− signature = table.css('.signature').first.inner_html
30+ signature = table.css('.signature').first.at_css('code').inner_html
1931 table.replace '<pre class="signature">' + signature + '</pre>'
2032 end
2133 end
lib/docs/filters/drupal/entries.rb+4−9
@@ -1,20 +1,15 @@
11 module Docs
22 class Drupal
33 class EntriesFilter < Docs::EntriesFilter
4−
54 def get_name
6− name = css('#page-subtitle').first.content
7− name.remove! 'function '
5+ name = at_css('#page-subtitle').content
6+ name.remove! %r{(abstract|public|static|protected|final|function|class)\s+}
87 name
98 end
109
11− def path
12− Drupal::fixUri(result[:path])
13− end
14−
1510 def get_type
16− type = css('dl[api-related-topics] dt')
17− type.first ? type.first.content : nil
11+ type = css('.breadcrumb > a')[1].content.strip
12+ type.split('.').first
1813 end
1914
2015 def include_default_entry?
lib/docs/scrapers/drupal.rb+18−31
@@ -1,38 +1,26 @@
11 module Docs
22 class Drupal < UrlScraper
3− self.name = 'Drupal'
43 self.type = 'drupal'
5− self.version = '7.37'
4+ self.version = '7.38'
65 self.base_url = 'https://api.drupal.org/api/drupal/'
7− self.initial_paths = %w(
8− groups
9− groups?page=1)
10−
11− html_filters.replace 'normalize_paths', 'drupal/normalize_paths'
12− html_filters.replace 'internal_urls', 'drupal/internal_urls'
6+ self.initial_paths = %w(groups groups?page=1)
7+ self.links = {
8+ home: 'https://www.drupal.org/'
9+ }
1310
1411 html_filters.push 'drupal/entries', 'drupal/clean_html', 'title'
1512
16− options[:container] = '#page'
13+ options[:decode_and_clean_paths] = true
14+ options[:container] = '#page-inner'
1715 options[:title] = false
18− options[:root_title] = 'Drupal - Open Source CMS | Drupal.org'
16+ options[:root_title] = 'Drupal'
1917
2018 options[:only_patterns] = [
19+ /\/class\/[^\/]+/,
2120 /\/group\/[^\/]+/,
2221 /\/function\/[^\/]+/]
2322
24− options[:skip_link] = ->(link) {
25− begin
26− return unless q = URL.parse(link['href']).query
27− Hash[URI.decode_www_form(q)].has_key? "order"
28− rescue URI::InvalidURIError
29− false
30− end
31− }
32−
33− options[:skip] = %w(
34− 'modules-system-system.install/group/updates-7.x-extra/7',
35− 'modules-system-system.install/group/updates-6.x-to-7.x/7')
23+ options[:skip_link] = ->(link) { link['href'] =~ /[\?&]order/ }
3624
3725 options[:skip_patterns] = [
3826 /\/group\/updates\-7/,
@@ -40,26 +28,25 @@ module Docs · 1 unmodified line
4028 /_update_[0-9]{4}/, # Skip update functions
4129 /\/[4-6](\.[0-9])*$/, # Skip previous versions
4230 /\/[8-9](\.[0-9])*$/, # Skip future versions
31+ /\/class\/hierarchy\//, # Skip class hierarchy listings
4332 /\/function\/calls\//, # Skip function calls listings
33+ /\/function\/invokes\//, # Skip function invokations listings
34+ /\/function\/overrides\//, # Skip function overrides listings
35+ /\/function\/references\//, # Skip function references listings
4436 /\/function\/implementations\//, # Skip hook implementation listings
37+ /\/function\/theme_references\//, # Skip hook references listings
4538 /\.test\/function\// # Skip test files
4639 ]
4740
4841 options[:fix_urls] = ->(url) do
49− url.sub! /\/7$/, '' # Remove the version indicator from the current version
42+ url.remove! %r{/7$}
5043 url
5144 end
5245
5346 options[:attribution] = <<-HTML
54− &copy; 2000&ndash;2015 by the individual contributors.<br>
55− Licensed under the Creative Commons License, Attribution-ShareAlike2.0.<br>
47+ &copy; 2001&ndash;2015 by the original authors<br>
48+ Licensed under the GNU General Public License, version 2 and later.<br>
5649 Drupal is a registered trademark of Dries Buytaert.
5750 HTML
58−
59− # Method used at several places to fix special characters at urls from api.drupal.org
60− def self.fixUri(path)
61− p = path.gsub /%21|!|%2b|%3b|%3a/i, '-' # !+;:
62− end
63−
6451 end
6552 end

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t