History

Finish Drupal scraper

Thibaut committed 11 years ago

Commit 79822f81 parent dac9b9b14 files · +56 −52Browse files

Unified · Split

14 files
  1. assets/images/icons.png +0 −0
  2. assets/images/icons@2x.png +0 −0
  3. assets/javascripts/news.json +3 −0
  4. assets/javascripts/templates/pages/about_tmpl.coffee +5 −5
  5. assets/javascripts/views/pages/drupal.coffee +6 −0
  6. assets/stylesheets/application-dark.css.scss +1 −0
  7. assets/stylesheets/global/_icons.scss +2 −1
  8. assets/stylesheets/pages/_drupal.scss +1 −7
  9. lib/docs/filters/core/normalize_paths.rb +5 −0
  10. lib/docs/filters/drupal/clean_html.rb +15 −3
  11. lib/docs/filters/drupal/entries.rb +4 −9
  12. lib/docs/scrapers/drupal.rb +18 −31
  13. public/icons/docs/drupal/16.png +0 −0
  14. public/icons/docs/drupal/16@2x.png +0 −0
assets/images/icons.png+0−0

Binary file not shown.

assets/images/icons@2x.png+0−0

Binary file not shown.

assets/javascripts/news.json+3−0
@@ -1,5 +1,8 @@
1[1[
2 [2 [
3 "2015-07-05",
4 "New documentation: <a href=\"/drupal/\">Drupal</a>"
5 ], [
3 "2015-05-24",6 "2015-05-24",
4 "New <a href=\"/rust/\">Rust</a> documentation"7 "New <a href=\"/rust/\">Rust</a> documentation"
5 ], [8 ], [
assets/javascripts/templates/pages/about_tmpl.coffee+5−5
@@ -135,6 +135,11 @@ credits = [ · 134 unmodified lines
135 'Django Software Foundation and individual contributors',135 'Django Software Foundation and individual contributors',
136 'BSD',136 'BSD',
137 'https://raw.githubusercontent.com/django/django/master/LICENSE'137 'https://raw.githubusercontent.com/django/django/master/LICENSE'
138 ], [
139 'Drupal',
140 '2001-2015 by the original authors<br>Drupal is a registered trademark of Dries Buytaert.',
141 'GPLv2',
142 'https://api.drupal.org/api/drupal/LICENSE.txt'
138 ], [143 ], [
139 'Ember.js',144 'Ember.js',
140 '2015 Yehuda Katz, Tom Dale and Ember.js contributors',145 '2015 Yehuda Katz, Tom Dale and Ember.js contributors',
@@ -220,11 +225,6 @@ credits = [ · 79 unmodified lines
220 '2004 John Gruber',225 '2004 John Gruber',
221 'BSD',226 'BSD',
222 'http://daringfireball.net/projects/markdown/license'227 'http://daringfireball.net/projects/markdown/license'
223 ], [
224 'MaxCDN',
225 '2015 MaxCDN',
226 'MIT',
227 'https://raw.githubusercontent.com/MaxCDN/api-docs/master/LICENSE'
228 ], [228 ], [
229 'Meteor',229 'Meteor',
230 '2011-2015 Meteor Development Group',230 '2011-2015 Meteor Development Group',
assets/javascripts/views/pages/drupal.coffee+6−0
@@ -0,0 +1,6 @@
1#= require views/pages/base
2
3class app.views.DrupalPage extends app.views.BasePage
4 afterRender: ->
5 @highlightCode @findAll('pre.php'), 'php'
6 return
assets/stylesheets/application-dark.css.scss+1−0
@@ -37,6 +37,7 @@ · 36 unmodified lines
37 'pages/clojure',37 'pages/clojure',
38 'pages/coffeescript',38 'pages/coffeescript',
39 'pages/d3',39 'pages/d3',
40 'pages/drupal',
40 'pages/ember',41 'pages/ember',
41 'pages/express',42 'pages/express',
42 'pages/git',43 'pages/git',
assets/stylesheets/global/_icons.scss+2−1
@@ -4,7 +4,7 @@ · 3 unmodified lines
4 width: 1rem;4 width: 1rem;
5 height: 1rem;5 height: 1rem;
6 background-image: image-url('icons.png');6 background-image: image-url('icons.png');
7 background-size: 10rem 8rem;7 background-size: 10rem 9rem;
8}8}
9 9
10@media (-webkit-min-device-pixel-ratio: 1.5), (min-resolution: 144dpi) {10@media (-webkit-min-device-pixel-ratio: 1.5), (min-resolution: 144dpi) {
@@ -101,3 +101,4 @@ · 90 unmodified lines
101._icon-meteor:before { background-position: -7rem -7rem; @extend %darkIconFix !optional; }101._icon-meteor:before { background-position: -7rem -7rem; @extend %darkIconFix !optional; }
102._icon-npm:before { background-position: -8rem -7rem; }102._icon-npm:before { background-position: -8rem -7rem; }
103._icon-apache_http_server:before { background-position: -9rem -7rem; }103._icon-apache_http_server:before { background-position: -9rem -7rem; }
104._icon-drupal:before { background-position: 0 -8rem; }
assets/stylesheets/pages/_drupal.scss+1−7
@@ -1,10 +1,4 @@
1._drupal {1._drupal {
2 h1#page-subtitle {
3 margin-top: 0;
4 @extend %lined-heading;
5 }
6
7 h3 { @extend %block-heading; }2 h3 { @extend %block-heading; }
8
9 .signature { @extend %note, %note-blue; }3 .signature { @extend %note, %note-blue; }
10}4}
lib/docs/filters/core/normalize_paths.rb+5−0
@@ -35,6 +35,11 @@ module Docs · 34 unmodified lines
35 def normalize_path(path)35 def normalize_path(path)
36 path = path.downcase36 path = path.downcase
37 37
38 if context[:decode_and_clean_paths]
39 path = URI.unescape(path)
40 path.gsub! %r{[!;:]+}, '-'
41 end
42
38 if path == '.'43 if path == '.'
39 'index'44 'index'
40 elsif path.end_with? '/'45 elsif path.end_with? '/'
lib/docs/filters/drupal/clean_html.rb+15−3
@@ -11,11 +11,23 @@ module Docs · 10 unmodified lines
11 end11 end
12 12
13 def other13 def other
14 css('#page-title-tools', '.element-invisible', '.breadcrumb', '#sidebar-first', '#api-alternatives').remove14 css('.element-invisible', '#sidebar-first', '#api-alternatives', '#aside', '.comments', '.view-filters',
15 css('#aside', '#api-function-signature tr:not(.active)', '.comments').remove15 '#api-function-signature tr:not(.active)', '.ctools-collapsible-container', 'img[width="13"]').remove
16
17 at_css('#main').replace(at_css('.content'))
18 at_css('#page-heading').replace(at_css('#page-subtitle'))
19
20 css('th.views-field > a', '.content').each do |node|
21 node.before(node.children).remove
22 end
23
24 css('pre').each do |node|
25 node.content = node.content
26 end
27
16 # Replaces the signature table from api.drupal.org with a simple pre tag28 # Replaces the signature table from api.drupal.org with a simple pre tag
17 css('#api-function-signature').each do |table|29 css('#api-function-signature').each do |table|
18 signature = table.css('.signature').first.inner_html30 signature = table.css('.signature').first.at_css('code').inner_html
19 table.replace '<pre class="signature">' + signature + '</pre>'31 table.replace '<pre class="signature">' + signature + '</pre>'
20 end32 end
21 end33 end
lib/docs/filters/drupal/entries.rb+4−9
@@ -1,20 +1,15 @@
1module Docs1module Docs
2 class Drupal2 class Drupal
3 class EntriesFilter < Docs::EntriesFilter3 class EntriesFilter < Docs::EntriesFilter
4
5 def get_name4 def get_name
6 name = css('#page-subtitle').first.content5 name = at_css('#page-subtitle').content
7 name.remove! 'function '6 name.remove! %r{(abstract|public|static|protected|final|function|class)\s+}
8 name7 name
9 end8 end
10 9
11 def path
12 Drupal::fixUri(result[:path])
13 end
14
15 def get_type10 def get_type
16 type = css('dl[api-related-topics] dt')11 type = css('.breadcrumb > a')[1].content.strip
17 type.first ? type.first.content : nil12 type.split('.').first
18 end13 end
19 14
20 def include_default_entry?15 def include_default_entry?
lib/docs/scrapers/drupal.rb+18−31
@@ -1,38 +1,26 @@
1module Docs1module Docs
2 class Drupal < UrlScraper2 class Drupal < UrlScraper
3 self.name = 'Drupal'
4 self.type = 'drupal'3 self.type = 'drupal'
5 self.version = '7.37'4 self.version = '7.38'
6 self.base_url = 'https://api.drupal.org/api/drupal/'5 self.base_url = 'https://api.drupal.org/api/drupal/'
7 self.initial_paths = %w(6 self.initial_paths = %w(groups groups?page=1)
8 groups7 self.links = {
9 groups?page=1)8 home: 'https://www.drupal.org/'
10 9 }
11 html_filters.replace 'normalize_paths', 'drupal/normalize_paths'
12 html_filters.replace 'internal_urls', 'drupal/internal_urls'
13 10
14 html_filters.push 'drupal/entries', 'drupal/clean_html', 'title'11 html_filters.push 'drupal/entries', 'drupal/clean_html', 'title'
15 12
16 options[:container] = '#page'13 options[:decode_and_clean_paths] = true
14 options[:container] = '#page-inner'
17 options[:title] = false15 options[:title] = false
18 options[:root_title] = 'Drupal - Open Source CMS | Drupal.org'16 options[:root_title] = 'Drupal'
19 17
20 options[:only_patterns] = [18 options[:only_patterns] = [
19 /\/class\/[^\/]+/,
21 /\/group\/[^\/]+/,20 /\/group\/[^\/]+/,
22 /\/function\/[^\/]+/]21 /\/function\/[^\/]+/]
23 22
24 options[:skip_link] = ->(link) {23 options[:skip_link] = ->(link) { link['href'] =~ /[\?&]order/ }
25 begin
26 return unless q = URL.parse(link['href']).query
27 Hash[URI.decode_www_form(q)].has_key? "order"
28 rescue URI::InvalidURIError
29 false
30 end
31 }
32
33 options[:skip] = %w(
34 'modules-system-system.install/group/updates-7.x-extra/7',
35 'modules-system-system.install/group/updates-6.x-to-7.x/7')
36 24
37 options[:skip_patterns] = [25 options[:skip_patterns] = [
38 /\/group\/updates\-7/,26 /\/group\/updates\-7/,
@@ -40,26 +28,25 @@ module Docs · 1 unmodified line
40 /_update_[0-9]{4}/, # Skip update functions28 /_update_[0-9]{4}/, # Skip update functions
41 /\/[4-6](\.[0-9])*$/, # Skip previous versions29 /\/[4-6](\.[0-9])*$/, # Skip previous versions
42 /\/[8-9](\.[0-9])*$/, # Skip future versions30 /\/[8-9](\.[0-9])*$/, # Skip future versions
31 /\/class\/hierarchy\//, # Skip class hierarchy listings
43 /\/function\/calls\//, # Skip function calls listings32 /\/function\/calls\//, # Skip function calls listings
33 /\/function\/invokes\//, # Skip function invokations listings
34 /\/function\/overrides\//, # Skip function overrides listings
35 /\/function\/references\//, # Skip function references listings
44 /\/function\/implementations\//, # Skip hook implementation listings36 /\/function\/implementations\//, # Skip hook implementation listings
37 /\/function\/theme_references\//, # Skip hook references listings
45 /\.test\/function\// # Skip test files38 /\.test\/function\// # Skip test files
46 ]39 ]
47 40
48 options[:fix_urls] = ->(url) do41 options[:fix_urls] = ->(url) do
49 url.sub! /\/7$/, '' # Remove the version indicator from the current version42 url.remove! %r{/7$}
50 url43 url
51 end44 end
52 45
53 options[:attribution] = <<-HTML46 options[:attribution] = <<-HTML
54 &copy; 2000&ndash;2015 by the individual contributors.<br>47 &copy; 2001&ndash;2015 by the original authors<br>
55 Licensed under the Creative Commons License, Attribution-ShareAlike2.0.<br>48 Licensed under the GNU General Public License, version 2 and later.<br>
56 Drupal is a registered trademark of Dries Buytaert.49 Drupal is a registered trademark of Dries Buytaert.
57 HTML50 HTML
58
59 # Method used at several places to fix special characters at urls from api.drupal.org
60 def self.fixUri(path)
61 p = path.gsub /%21|!|%2b|%3b|%3a/i, '-' # !+;:
62 end
63
64 end51 end
65end52end
public/icons/docs/drupal/16.png+0−0

Binary file not shown.

public/icons/docs/drupal/16@2x.png+0−0

Binary file not shown.