History

Finish Drupal scraper

Thibaut committed 11 years ago

Commit 79822f81 parent dac9b9b14 files · +56 −52Browse files

Unified · Split

14 files
  1. assets/images/icons.png +0 −0
  2. assets/images/icons@2x.png +0 −0
  3. assets/javascripts/news.json +3 −0
  4. assets/javascripts/templates/pages/about_tmpl.coffee +5 −5
  5. assets/javascripts/views/pages/drupal.coffee +6 −0
  6. assets/stylesheets/application-dark.css.scss +1 −0
  7. assets/stylesheets/global/_icons.scss +2 −1
  8. assets/stylesheets/pages/_drupal.scss +1 −7
  9. lib/docs/filters/core/normalize_paths.rb +5 −0
  10. lib/docs/filters/drupal/clean_html.rb +15 −3
  11. lib/docs/filters/drupal/entries.rb +4 −9
  12. lib/docs/scrapers/drupal.rb +18 −31
  13. public/icons/docs/drupal/16.png +0 −0
  14. public/icons/docs/drupal/16@2x.png +0 −0
assets/images/icons.png+0−0

Binary file not shown.

assets/images/icons@2x.png+0−0

Binary file not shown.

assets/javascripts/news.json+3−0
@@ -1,5 +1,8 @@
11 [
22 [
3+ "2015-07-05",
4+ "New documentation: <a href=\"/drupal/\">Drupal</a>"
5+ ], [
36 "2015-05-24",
47 "New <a href=\"/rust/\">Rust</a> documentation"
58 ], [
assets/javascripts/templates/pages/about_tmpl.coffee+5−5
@@ -135,6 +135,11 @@ credits = [ · 134 unmodified lines
135135 'Django Software Foundation and individual contributors',
136136 'BSD',
137137 'https://raw.githubusercontent.com/django/django/master/LICENSE'
138+ ], [
139+ 'Drupal',
140+ '2001-2015 by the original authors<br>Drupal is a registered trademark of Dries Buytaert.',
141+ 'GPLv2',
142+ 'https://api.drupal.org/api/drupal/LICENSE.txt'
138143 ], [
139144 'Ember.js',
140145 '2015 Yehuda Katz, Tom Dale and Ember.js contributors',
@@ -220,11 +225,6 @@ credits = [ · 79 unmodified lines
220225 '2004 John Gruber',
221226 'BSD',
222227 'http://daringfireball.net/projects/markdown/license'
223− ], [
224− 'MaxCDN',
225− '2015 MaxCDN',
226− 'MIT',
227− 'https://raw.githubusercontent.com/MaxCDN/api-docs/master/LICENSE'
228228 ], [
229229 'Meteor',
230230 '2011-2015 Meteor Development Group',
assets/javascripts/views/pages/drupal.coffee+6−0
@@ -0,0 +1,6 @@
1+#= require views/pages/base
2+
3+class app.views.DrupalPage extends app.views.BasePage
4+ afterRender: ->
5+ @highlightCode @findAll('pre.php'), 'php'
6+ return
assets/stylesheets/application-dark.css.scss+1−0
@@ -37,6 +37,7 @@ · 36 unmodified lines
3737 'pages/clojure',
3838 'pages/coffeescript',
3939 'pages/d3',
40+ 'pages/drupal',
4041 'pages/ember',
4142 'pages/express',
4243 'pages/git',
assets/stylesheets/global/_icons.scss+2−1
@@ -4,7 +4,7 @@ · 3 unmodified lines
44 width: 1rem;
55 height: 1rem;
66 background-image: image-url('icons.png');
7− background-size: 10rem 8rem;
7+ background-size: 10rem 9rem;
88 }
99
1010 @media (-webkit-min-device-pixel-ratio: 1.5), (min-resolution: 144dpi) {
@@ -101,3 +101,4 @@ · 90 unmodified lines
101101 ._icon-meteor:before { background-position: -7rem -7rem; @extend %darkIconFix !optional; }
102102 ._icon-npm:before { background-position: -8rem -7rem; }
103103 ._icon-apache_http_server:before { background-position: -9rem -7rem; }
104+._icon-drupal:before { background-position: 0 -8rem; }
assets/stylesheets/pages/_drupal.scss+1−7
@@ -1,10 +1,4 @@
11 ._drupal {
2− h1#page-subtitle {
3− margin-top: 0;
4− @extend %lined-heading;
5− }
6−
72 h3 { @extend %block-heading; }
8−
93 .signature { @extend %note, %note-blue; }
10−}
4+}
lib/docs/filters/core/normalize_paths.rb+5−0
@@ -35,6 +35,11 @@ module Docs · 34 unmodified lines
3535 def normalize_path(path)
3636 path = path.downcase
3737
38+ if context[:decode_and_clean_paths]
39+ path = URI.unescape(path)
40+ path.gsub! %r{[!;:]+}, '-'
41+ end
42+
3843 if path == '.'
3944 'index'
4045 elsif path.end_with? '/'
lib/docs/filters/drupal/clean_html.rb+15−3
@@ -11,11 +11,23 @@ module Docs · 10 unmodified lines
1111 end
1212
1313 def other
14− css('#page-title-tools', '.element-invisible', '.breadcrumb', '#sidebar-first', '#api-alternatives').remove
15− css('#aside', '#api-function-signature tr:not(.active)', '.comments').remove
14+ css('.element-invisible', '#sidebar-first', '#api-alternatives', '#aside', '.comments', '.view-filters',
15+ '#api-function-signature tr:not(.active)', '.ctools-collapsible-container', 'img[width="13"]').remove
16+
17+ at_css('#main').replace(at_css('.content'))
18+ at_css('#page-heading').replace(at_css('#page-subtitle'))
19+
20+ css('th.views-field > a', '.content').each do |node|
21+ node.before(node.children).remove
22+ end
23+
24+ css('pre').each do |node|
25+ node.content = node.content
26+ end
27+
1628 # Replaces the signature table from api.drupal.org with a simple pre tag
1729 css('#api-function-signature').each do |table|
18− signature = table.css('.signature').first.inner_html
30+ signature = table.css('.signature').first.at_css('code').inner_html
1931 table.replace '<pre class="signature">' + signature + '</pre>'
2032 end
2133 end
lib/docs/filters/drupal/entries.rb+4−9
@@ -1,20 +1,15 @@
11 module Docs
22 class Drupal
33 class EntriesFilter < Docs::EntriesFilter
4−
54 def get_name
6− name = css('#page-subtitle').first.content
7− name.remove! 'function '
5+ name = at_css('#page-subtitle').content
6+ name.remove! %r{(abstract|public|static|protected|final|function|class)\s+}
87 name
98 end
109
11− def path
12− Drupal::fixUri(result[:path])
13− end
14−
1510 def get_type
16− type = css('dl[api-related-topics] dt')
17− type.first ? type.first.content : nil
11+ type = css('.breadcrumb > a')[1].content.strip
12+ type.split('.').first
1813 end
1914
2015 def include_default_entry?
lib/docs/scrapers/drupal.rb+18−31
@@ -1,38 +1,26 @@
11 module Docs
22 class Drupal < UrlScraper
3− self.name = 'Drupal'
43 self.type = 'drupal'
5− self.version = '7.37'
4+ self.version = '7.38'
65 self.base_url = 'https://api.drupal.org/api/drupal/'
7− self.initial_paths = %w(
8− groups
9− groups?page=1)
10−
11− html_filters.replace 'normalize_paths', 'drupal/normalize_paths'
12− html_filters.replace 'internal_urls', 'drupal/internal_urls'
6+ self.initial_paths = %w(groups groups?page=1)
7+ self.links = {
8+ home: 'https://www.drupal.org/'
9+ }
1310
1411 html_filters.push 'drupal/entries', 'drupal/clean_html', 'title'
1512
16− options[:container] = '#page'
13+ options[:decode_and_clean_paths] = true
14+ options[:container] = '#page-inner'
1715 options[:title] = false
18− options[:root_title] = 'Drupal - Open Source CMS | Drupal.org'
16+ options[:root_title] = 'Drupal'
1917
2018 options[:only_patterns] = [
19+ /\/class\/[^\/]+/,
2120 /\/group\/[^\/]+/,
2221 /\/function\/[^\/]+/]
2322
24− options[:skip_link] = ->(link) {
25− begin
26− return unless q = URL.parse(link['href']).query
27− Hash[URI.decode_www_form(q)].has_key? "order"
28− rescue URI::InvalidURIError
29− false
30− end
31− }
32−
33− options[:skip] = %w(
34− 'modules-system-system.install/group/updates-7.x-extra/7',
35− 'modules-system-system.install/group/updates-6.x-to-7.x/7')
23+ options[:skip_link] = ->(link) { link['href'] =~ /[\?&]order/ }
3624
3725 options[:skip_patterns] = [
3826 /\/group\/updates\-7/,
@@ -40,26 +28,25 @@ module Docs · 1 unmodified line
4028 /_update_[0-9]{4}/, # Skip update functions
4129 /\/[4-6](\.[0-9])*$/, # Skip previous versions
4230 /\/[8-9](\.[0-9])*$/, # Skip future versions
31+ /\/class\/hierarchy\//, # Skip class hierarchy listings
4332 /\/function\/calls\//, # Skip function calls listings
33+ /\/function\/invokes\//, # Skip function invokations listings
34+ /\/function\/overrides\//, # Skip function overrides listings
35+ /\/function\/references\//, # Skip function references listings
4436 /\/function\/implementations\//, # Skip hook implementation listings
37+ /\/function\/theme_references\//, # Skip hook references listings
4538 /\.test\/function\// # Skip test files
4639 ]
4740
4841 options[:fix_urls] = ->(url) do
49− url.sub! /\/7$/, '' # Remove the version indicator from the current version
42+ url.remove! %r{/7$}
5043 url
5144 end
5245
5346 options[:attribution] = <<-HTML
54− &copy; 2000&ndash;2015 by the individual contributors.<br>
55− Licensed under the Creative Commons License, Attribution-ShareAlike2.0.<br>
47+ &copy; 2001&ndash;2015 by the original authors<br>
48+ Licensed under the GNU General Public License, version 2 and later.<br>
5649 Drupal is a registered trademark of Dries Buytaert.
5750 HTML
58−
59− # Method used at several places to fix special characters at urls from api.drupal.org
60− def self.fixUri(path)
61− p = path.gsub /%21|!|%2b|%3b|%3a/i, '-' # !+;:
62− end
63−
6451 end
6552 end
public/icons/docs/drupal/16.png+0−0

Binary file not shown.

public/icons/docs/drupal/16@2x.png+0−0

Binary file not shown.