History

Finish Matplotlib scraper

Thibaut Courouble committed 10 years ago

Commit 8d038301 parent 8f8815012 files · +102 −17Browse files

Unified · Split

12 files
  1. assets/images/docs.png +0 −0
  2. assets/images/docs@2x.png +0 −0
  3. assets/javascripts/news.json +2 −2
  4. assets/javascripts/templates/pages/about_tmpl.coffee +5 −0
  5. assets/stylesheets/global/_icons.scss +1 −0
  6. assets/stylesheets/pages/_sphinx.scss +6 −3
  7. lib/docs/filters/matplotlib/clean_html.rb +56 −1
  8. lib/docs/filters/matplotlib/entries.rb +27 −9
  9. lib/docs/scrapers/matplotlib.rb +8 −6
  10. public/icons/docs/matplotlib/16.png +0 −0
  11. public/icons/docs/matplotlib/16@2x.png +0 −0
  12. public/icons/docs/matplotlib/SOURCE +1 −0
assets/images/docs.png+0−0

Binary file not shown.

assets/images/docs@2x.png+0−0

Binary file not shown.

assets/javascripts/news.json+2−2
@@ -1,7 +1,7 @@
1[1[
2 [2 [
3 "2016-07-02",3 "2016-07-03",
4 "New documentation: <a href=\"/cmake/\">CMake</a>"4 "New documentations: <a href=\"/cmake/\">CMake</a> and <a href=\"/matplotlib/\">Matplotlib</a>"
5 ], [5 ], [
6 "2016-06-19",6 "2016-06-19",
7 "New documentation: <a href=\"/love/\">L&Ouml;VE</a>"7 "New documentation: <a href=\"/love/\">L&Ouml;VE</a>"
assets/javascripts/templates/pages/about_tmpl.coffee+5−0
@@ -295,6 +295,11 @@ credits = [ · 294 unmodified lines
295 '2004 John Gruber',295 '2004 John Gruber',
296 'BSD',296 'BSD',
297 'https://daringfireball.net/projects/markdown/license'297 'https://daringfireball.net/projects/markdown/license'
298 ], [
299 'Matplotlib',
300 '2012-2016 Matplotlib Development Team. All rights reserved.',
301 'Custom',
302 'https://raw.githubusercontent.com/matplotlib/matplotlib/master/LICENSE/LICENSE'
298 ], [303 ], [
299 'Meteor',304 'Meteor',
300 '2011-2016 Meteor Development Group',305 '2011-2016 Meteor Development Group',
assets/stylesheets/global/_icons.scss+1−0
@@ -132,6 +132,7 @@ · 131 unmodified lines
132._icon-q:before { background-position: -5rem -8rem; }132._icon-q:before { background-position: -5rem -8rem; }
133._icon-react_native:before { background-position: 0 -9rem; }133._icon-react_native:before { background-position: 0 -9rem; }
134._icon-phalcon:before { background-position: -1rem -9rem; }134._icon-phalcon:before { background-position: -1rem -9rem; }
135._icon-matplotlib:before { background-position: -2rem -9rem; }
135._icon-cmake:before { background-position: -3rem -9rem; }136._icon-cmake:before { background-position: -3rem -9rem; }
136._icon-elixir:before { background-position: -4rem -9rem; @extend %darkIconFix !optional; }137._icon-elixir:before { background-position: -4rem -9rem; @extend %darkIconFix !optional; }
137._icon-vagrant:before { background-position: -5rem -9rem; }138._icon-vagrant:before { background-position: -5rem -9rem; }
assets/stylesheets/pages/_sphinx.scss+6−3
@@ -1,6 +1,8 @@
1%sphinx {1%sphinx {
2 h2, h3 { @extend %block-heading; }2 h2, h3 { @extend %block-heading; }
3 dl:not(.docutils) > dt { @extend %block-label, %label-blue; }3 h4 { font-size: 1em; }
4 > dl:not(.docutils) > dt { @extend %block-label, %label-blue; }
5 dl > dl > dt { @extend %block-label; }
4 dt + dt { margin-top: -.5em; }6 dt + dt { margin-top: -.5em; }
5 7
6 .note, .admonition, .versionadded, .versionchanged, .deprecated-removed { @extend %note; }8 .note, .admonition, .versionadded, .versionchanged, .deprecated-removed { @extend %note; }
@@ -8,7 +10,7 @@ · 1 unmodified line
8 .warning, .deprecated-removed { @extend %note-red; }10 .warning, .deprecated-removed { @extend %note-red; }
9 .versionmodified { font-weight: bold; }11 .versionmodified { font-weight: bold; }
10 12
11 p > code, li > code { @extend %label; }13 p > code, li > code, dd > code { @extend %label; }
12 14
13 .admonition-title {15 .admonition-title {
14 float: left;16 float: left;
@@ -22,10 +24,11 @@ · 7 unmodified lines
22 clear: left;24 clear: left;
23 margin: 0;25 margin: 0;
24 }26 }
27 .admonition-title + dl { padding-top: .5em; }
25 28
26 ul.simple { margin: 1em 0; }29 ul.simple { margin: 1em 0; }
27 30
28 h2 > a, h3 > a, dt > a { float: right; }31 h2 > a, h3 > a, dt[id] > a.external { float: right; }
29}32}
30 33
31._sphinx {34._sphinx {
lib/docs/filters/matplotlib/clean_html.rb+56−1
@@ -2,9 +2,64 @@ module Docs · 1 unmodified line
2 class Matplotlib2 class Matplotlib
3 class CleanHtmlFilter < Filter3 class CleanHtmlFilter < Filter
4 def call4 def call
5 css('.headerlink').each do |node|5 css('.headerlink', 'hr').remove
6
7 css('.toc-backref', '.toctree-wrapper', '.contents', 'span.pre', 'pre a > code').each do |node|
8 node.before(node.children).remove
9 end
10
11 css('div[class*="highlight-"]').each do |node|
12 pre = node.at_css('pre')
13 pre.content = pre.content
14 pre['data-language'] = node['class'][/highlight\-(\w+)/, 1]
15 node.replace(pre)
16 end
17
18 css('span[id]:empty').each do |node|
19 node.next_element['id'] = node['id']
6 node.remove20 node.remove
7 end21 end
22
23 css('.section').each do |node|
24 if node['id']
25 if node.first_element_child['id']
26 node.element_children[1]['id'] = node['id']
27 else
28 node.first_element_child['id'] = node['id']
29 end
30 end
31
32 node.before(node.children).remove
33 end
34
35 css('h2 > a > code').each do |node|
36 node.parent.before(node.content).remove
37 end
38
39 css('dt[id]').each do |node|
40 node.inner_html = "<code>#{node.content.strip}</code>"
41 end
42
43 css('li > p:first-child:last-child').each do |node|
44 node.before(node.children).remove
45 end
46
47 css('table[border]').each do |node|
48 node.remove_attribute 'border'
49 end
50
51 css('code[class]').each do |node|
52 node.remove_attribute 'class'
53 end
54
55 css('h1').each do |node|
56 node.content = node.content
57 end
58
59 css('p.rubric').each do |node|
60 node.name = 'h4'
61 end
62
8 doc63 doc
9 end64 end
10 end65 end
lib/docs/filters/matplotlib/entries.rb+27−9
@@ -1,29 +1,47 @@
1module Docs1module Docs
2 class Matplotlib2 class Matplotlib
3 class EntriesFilter < Docs::EntriesFilter3 class EntriesFilter < Docs::EntriesFilter
4 NAME_BY_SLUG = {
5 'matplotlib_configuration_api' => 'matplotlib',
6 'tri_api' => 'tri'
7 }
8
9 TYPE_BY_SLUG = {
10 'pyplot_summary' => 'pyplot'
11 }
12
4 def get_name13 def get_name
14 return NAME_BY_SLUG[slug] if NAME_BY_SLUG.key?(slug)
5 name = at_css('h1').content.strip15 name = at_css('h1').content.strip
6 name.remove! "\u{00b6}"16 name.remove! "\u{00b6}"
17 name.remove! 'matplotlib.'
18 name.remove! %r{ \(.*\)}
19 name.downcase!
7 name20 name
8 end21 end
9 22
10 def get_type23 def get_type
11 name = at_css('h1').content.strip24 return TYPE_BY_SLUG[slug] if TYPE_BY_SLUG.key?(slug)
12 name.remove! "\u{00b6}"25 name.split('.').first
13 name
14 end26 end
15 27
16 def additional_entries28 def additional_entries
17 entries = []29 entries = []
18 ents = css('dt .descname')
19 30
20 if ents31 css('.class > dt[id]', '.exception > dt[id]', '.attribute > dt[id]').each do |node|
21 ents.each do |node|32 entries << [node['id'].remove('matplotlib.'), node['id']]
22 name = node.content.sub(/\(.*\)/, '()')33 end
23 id = node.parent['id']34
24 entries << [name, id, get_name]35 css('.data > dt[id]').each do |node|
36 if node['id'].split('.').last.upcase! # skip constants
37 entries << [node['id'].remove('matplotlib.'), node['id']]
25 end38 end
26 end39 end
40
41 css('.function > dt[id]', '.method > dt[id]', '.classmethod > dt[id]').each do |node|
42 entries << [node['id'].remove('matplotlib.') + '()', node['id']]
43 end
44
27 entries45 entries
28 end46 end
29 end47 end
lib/docs/scrapers/matplotlib.rb+8−6
@@ -1,9 +1,8 @@
1module Docs1module Docs
2 class Matplotlib < FileScraper2 class Matplotlib < UrlScraper
3 self.name = 'Matplotlib'3 self.name = 'Matplotlib'
4 self.type = 'sphinx'4 self.type = 'sphinx'
5 self.root_path = 'index.html'5 self.root_path = 'index.html'
6 self.release = "1.5.1"
7 self.links = {6 self.links = {
8 home: 'http://matplotlib.org/',7 home: 'http://matplotlib.org/',
9 code: 'https://github.com/matplotlib/matplotlib'8 code: 'https://github.com/matplotlib/matplotlib'
@@ -12,13 +11,16 @@ module Docs · 2 unmodified lines
12 html_filters.push 'matplotlib/entries', 'matplotlib/clean_html'11 html_filters.push 'matplotlib/entries', 'matplotlib/clean_html'
13 12
14 options[:container] = '.body'13 options[:container] = '.body'
14 options[:skip] = %w(api_changes.html)
15 15
16 options[:attribution] = <<-HTML16 options[:attribution] = <<-HTML
17 &copy; Matplotlib Development Team <br>17 &copy; 2012&ndash;2016 Matplotlib Development Team. All rights reserved.<br>
18 Licensed under the BSD License.18 Licensed under the Matplotlib License Agreement.
19 HTML19 HTML
20 20
21 self.dir = '~/workspace/tmp/matplotlib/matplotlib.github.com-master/1.5.1/api/'21 version '1.5' do
22 # self.base_url = 'http://matplotlib.org/api/'22 self.release = '1.5.1'
23 self.base_url = 'http://matplotlib.org/1.5.1/api/'
24 end
23 end25 end
24end26end
public/icons/docs/matplotlib/16.png+0−0

Binary file not shown.

public/icons/docs/matplotlib/16@2x.png+0−0

Binary file not shown.

public/icons/docs/matplotlib/SOURCE+1−0
@@ -0,0 +1 @@
1https://upload.wikimedia.org/wikipedia/commons/thumb/8/84/Matplotlib_icon.svg/1024px-Matplotlib_icon.svg.png