History

Finish Docker scraper

Thibaut Courouble committed 10 years ago

Commit e92d0e11 parent 457439f11 files · +65 −21Browse files

Showing the changes to lib (3 of 11 files). Show all

Unified · Split

3 files
  1. lib/docs/filters/docker/clean_html.rb +5 −6
  2. lib/docs/filters/docker/entries.rb +44 −8
  3. lib/docs/scrapers/docker.rb +10 −10
lib/docs/filters/docker/clean_html.rb+5−6
@@ -3,14 +3,13 @@ module Docs · 2 unmodified lines
3 class CleanHtmlFilter < Filter3 class CleanHtmlFilter < Filter
4 def call4 def call
5 if root_page?5 if root_page?
6 @doc = at_css('.tabs-content')6 doc.inner_html = "<h1>Docker Documentation</h1>"
7 else7 return doc
8 @doc = at_css('#content')
9 end8 end
10 9
11 if not at_css('h1')10 @doc = at_css('#content')
12 at_css('h2').name = 'h1'11
13 end12 at_css('h2').name = 'h1' unless at_css('h1')
14 13
15 css('pre').each do |node|14 css('pre').each do |node|
16 node.content = node.content15 node.content = node.content
lib/docs/filters/docker/entries.rb+44−8
@@ -2,20 +2,56 @@ module Docs · 1 unmodified line
2 class Docker2 class Docker
3 class EntriesFilter < Docs::EntriesFilter3 class EntriesFilter < Docs::EntriesFilter
4 def get_name4 def get_name
5 title = at_css('h1').content5 name = nav_link ? nav_link.content.strip : at_css('#content h1').content.strip
6 name.capitalize! if name == 'exoscale'
6 7
7 title = title.split(' — ').first8 if name =~ /\A[a-z\-\s]+\z/
8 title = title.split(' - ').first9 name.prepend 'docker ' if subpath =~ /engine\/reference\/commandline\/./
10 name.prepend 'docker-compose ' if subpath =~ /compose\/reference\/./
11 name.prepend 'docker-machine ' if subpath =~ /machine\/reference\/./
12 name.prepend 'swarm ' if subpath =~ /swarm\/reference\/./ && name != 'swarm'
13 else
14 name << " (#{product})" if name !~ /#{product}/i
15 end
9 16
10 title17 name
11 end18 end
12 19
13 def get_type20 def get_type
14 if slug.count('/') == 1 or slug.count('/') == 221 unless nav_link
15 slug.split('/').first.capitalize22 return 'Engine: User guide' if subpath.start_with?('engine/userguide')
16 else23 return 'Engine: Secure' if subpath.start_with?('engine/security')
17 slug.split('/')[0...-1].map(&:capitalize).join('/').gsub(/\-|\_/, ' ')24 return 'Engine' if subpath == 'engine/'
25 end
26
27 type = nav_link.ancestors('article').to_a.reverse.to_a[0..1].map do |node|
28 node.at_css('> button').content.strip
29 end.join(': ')
30
31 type.remove! %r{\ADocker }
32 type.remove! %r{ Engine}
33 type.sub! %r{Command[\-\s]line reference}i, 'CLI'
34 type = 'Engine: Reference' if type == 'Engine: reference'
35 type
36 end
37
38 def nav_link
39 return @nav_link if defined?(@nav_link)
40 @nav_link = at_css('#multiple .active')
41
42 unless @nav_link
43 link = at_css('#content li a')
44 return unless link
45 link = at_css("#multiple a[href='#{link['href']}']")
46 return unless link
47 @nav_link = link.ancestors('article').first.at_css('button')
18 end48 end
49
50 @nav_link
51 end
52
53 def product
54 @product ||= subpath.split('/').first.capitalize
19 end55 end
20 end56 end
21 end57 end
lib/docs/scrapers/docker.rb+10−10
@@ -1,16 +1,14 @@
1module Docs1module Docs
2 class Docker < UrlScraper2 class Docker < UrlScraper
3 self.name = 'Docker'
4 self.slug = 'docker'
5 self.type = 'docker'3 self.type = 'docker'
6
7 self.links = {4 self.links = {
8 home: 'https://docker.com/',5 home: 'https://docker.com/',
9 code: 'https://github.com/docker/docker'6 code: 'https://github.com/docker/docker'
10 }7 }
11 8
12 html_filters.push 'docker/clean_html', 'docker/entries'9 html_filters.push 'docker/entries', 'docker/clean_html'
13 10
11 options[:container] = '#docs'
14 options[:trailing_slash] = true12 options[:trailing_slash] = true
15 13
16 options[:only_patterns] = [14 options[:only_patterns] = [
@@ -19,14 +17,16 @@ module Docs · 2 unmodified lines
19 /\Amachine\//,17 /\Amachine\//,
20 /\Aswarm\//18 /\Aswarm\//
21 ]19 ]
22 20 options[:skip] = %w(swarm/scheduler/)
23 options[:skip] = [21 options[:replace_paths] = {
24 "engine/installation/ubuntulinux/" # invalid document22 'engine/installation/ubuntulinux/' => 'engine/installation/linux/ubuntulinux/'
25 ]23 }
26 24
27 options[:attribution] = <<-HTML25 options[:attribution] = <<-HTML
28 &copy; 2016 Docker Inc.<br>26 &copy; 2013&ndash;2016 Docker, Inc.<br>
29 Licensed under the Apache 2 License.27 Licensed under the Apache License, Version 2.0.<br>
28 Docker and the Docker logo are trademarks or registered trademarks of Docker, Inc. in the United States and/or other countries.<br>
29 Docker, Inc. and other parties may also have trademark rights in other terms used herein.
30 HTML30 HTML
31 31
32 version '1.11' do32 version '1.11' do

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t