History

Merge pull request #2607 from denilsonsa/graphviz

New scraper: graphviz (also known as DOT language)

Simon Legner committed 1 year ago

Commit 96a10592 parents 7bbcb5a cbaedce8 files · +213 −-1Browse files

Unified · Split

8 files
  1. assets/javascripts/vendor/prism.js +78 −1
  2. lib/docs/core/requester.rb +6 −0
  3. lib/docs/filters/graphviz/clean_html.rb +47 −0
  4. lib/docs/filters/graphviz/entries.rb +28 −0
  5. lib/docs/scrapers/graphviz.rb +55 −0
  6. public/icons/docs/graphviz/16.png +0 −0
  7. public/icons/docs/graphviz/16@2x.png +0 −0
  8. public/icons/docs/graphviz/SOURCE +1 −0
assets/javascripts/vendor/prism.js+78−1
@@ -1,5 +1,5 @@
1/* PrismJS 1.30.01/* PrismJS 1.30.0
2https://prismjs.com/download.html#themes=prism&languages=markup+css+clike+javascript+bash+c+cpp+cmake+coffeescript+crystal+d+dart+diff+django+elixir+erlang+go+groovy+java+json+julia+kotlin+latex+lua+markdown+markup-templating+matlab+nginx+nim+nix+ocaml+perl+php+python+qml+r+jsx+ruby+rust+scss+scala+shell-session+sql+tcl+typescript+yaml+zig */2https://prismjs.com/download.html#themes=prism&languages=markup+css+clike+javascript+bash+c+cpp+cmake+coffeescript+crystal+d+dart+diff+django+dot+elixir+erlang+go+groovy+java+json+julia+kotlin+latex+lua+markdown+markup-templating+matlab+nginx+nim+nix+ocaml+perl+php+python+qml+r+jsx+ruby+rust+scss+scala+shell-session+sql+tcl+typescript+yaml+zig */
3/// <reference lib="WebWorker"/>3/// <reference lib="WebWorker"/>
4 4
5var _self = (typeof window !== 'undefined')5var _self = (typeof window !== 'undefined')
@@ -2929,6 +2929,83 @@ Prism.languages.insertBefore('d', 'function', { · 2923 unmodified lines
2929 2929
2930}(Prism));2930}(Prism));
2931 2931
2932// https://www.graphviz.org/doc/info/lang.html
2933
2934(function (Prism) {
2935
2936 var ID = '(?:' + [
2937 // an identifier
2938 /[a-zA-Z_\x80-\uFFFF][\w\x80-\uFFFF]*/.source,
2939 // a number
2940 /-?(?:\.\d+|\d+(?:\.\d*)?)/.source,
2941 // a double-quoted string
2942 /"[^"\\]*(?:\\[\s\S][^"\\]*)*"/.source,
2943 // HTML-like string
2944 /<(?:[^<>]|(?!<!--)<(?:[^<>"']|"[^"]*"|'[^']*')+>|<!--(?:[^-]|-(?!->))*-->)*>/.source
2945 ].join('|') + ')';
2946
2947 var IDInside = {
2948 'markup': {
2949 pattern: /(^<)[\s\S]+(?=>$)/,
2950 lookbehind: true,
2951 alias: ['language-markup', 'language-html', 'language-xml'],
2952 inside: Prism.languages.markup
2953 }
2954 };
2955
2956 /**
2957 * @param {string} source
2958 * @param {string} flags
2959 * @returns {RegExp}
2960 */
2961 function withID(source, flags) {
2962 return RegExp(source.replace(/<ID>/g, function () { return ID; }), flags);
2963 }
2964
2965 Prism.languages.dot = {
2966 'comment': {
2967 pattern: /\/\/.*|\/\*[\s\S]*?\*\/|^#.*/m,
2968 greedy: true
2969 },
2970 'graph-name': {
2971 pattern: withID(/(\b(?:digraph|graph|subgraph)[ \t\r\n]+)<ID>/.source, 'i'),
2972 lookbehind: true,
2973 greedy: true,
2974 alias: 'class-name',
2975 inside: IDInside
2976 },
2977 'attr-value': {
2978 pattern: withID(/(=[ \t\r\n]*)<ID>/.source),
2979 lookbehind: true,
2980 greedy: true,
2981 inside: IDInside
2982 },
2983 'attr-name': {
2984 pattern: withID(/([\[;, \t\r\n])<ID>(?=[ \t\r\n]*=)/.source),
2985 lookbehind: true,
2986 greedy: true,
2987 inside: IDInside
2988 },
2989 'keyword': /\b(?:digraph|edge|graph|node|strict|subgraph)\b/i,
2990 'compass-point': {
2991 pattern: /(:[ \t\r\n]*)(?:[ewc_]|[ns][ew]?)(?![\w\x80-\uFFFF])/,
2992 lookbehind: true,
2993 alias: 'builtin'
2994 },
2995 'node': {
2996 pattern: withID(/(^|[^-.\w\x80-\uFFFF\\])<ID>/.source),
2997 lookbehind: true,
2998 greedy: true,
2999 inside: IDInside
3000 },
3001 'operator': /[=:]|-[->]/,
3002 'punctuation': /[\[\]{};,]/
3003 };
3004
3005 Prism.languages.gv = Prism.languages.dot;
3006
3007}(Prism));
3008
2932Prism.languages.elixir = {3009Prism.languages.elixir = {
2933 'doc': {3010 'doc': {
2934 pattern: /@(?:doc|moduledoc)\s+(?:("""|''')[\s\S]*?\1|("|')(?:\\(?:\r\n|[\s\S])|(?!\2)[^\\\r\n])*\2)/,3011 pattern: /@(?:doc|moduledoc)\s+(?:("""|''')[\s\S]*?\1|("|')(?:\\(?:\r\n|[\s\S])|(?!\2)[^\\\r\n])*\2)/,
lib/docs/core/requester.rb+6−0
@@ -54,6 +54,12 @@ module Docs · 53 unmodified lines
54 end54 end
55 55
56 def handle_response(response)56 def handle_response(response)
57 if ENV['RETRY'] == '1' && [0, 500, 501, 502, 503, 504].include?(response.code.to_i)
58 instrument 'handle_response.retry', url: response.url do
59 build_and_queue_request(response.url)
60 end
61 return
62 end
57 instrument 'handle_response.requester', url: response.url do63 instrument 'handle_response.requester', url: response.url do
58 on_response.each do |callback|64 on_response.each do |callback|
59 result = callback.call(response)65 result = callback.call(response)
lib/docs/filters/graphviz/clean_html.rb+47−0
@@ -0,0 +1,47 @@
1module Docs
2 class Graphviz
3 class CleanHtmlFilter < Filter
4 def call
5 css('[tabindex]').remove_attribute('tabindex')
6
7 content = at_css('.td-content')
8 @doc = content if content
9
10 css('a:contains("Search the Graphviz codebase")').remove
11 css('.td-page-meta__lastmod').remove
12
13 css('pre:has(code)').each do |node|
14 pre = Nokogiri::XML::Node.new('pre', @doc)
15 code = node.at_css('code')
16
17 if code['data-lang']
18 # Syntax highlighting is embedded into this HTML markup.
19 pre['data-language'] = code['data-lang']
20 else
21 # Plain example source-code without highlighting.
22 # Let's guess the language.
23 sourcecode = code.content.strip
24 if sourcecode =~ /^\$/
25 # Starts with '$'? Probably a shell session.
26 pre['data-language'] = 'shell-session'
27 elsif sourcecode =~ /^cmd /
28 # Command line example. No highlighting needed.
29 pre['data-language'] = ''
30 elsif sourcecode =~ /^void /
31 # C language.
32 pre['data-language'] = 'c'
33 else
34 # Nothing else? Let's guess DOT.
35 pre['data-language'] = 'dot'
36 end
37 end
38 pre.content = code.content
39
40 node.replace(pre)
41 end
42
43 doc
44 end
45 end
46 end
47end
lib/docs/filters/graphviz/entries.rb+28−0
@@ -0,0 +1,28 @@
1module Docs
2 class Graphviz
3 class EntriesFilter < Docs::EntriesFilter
4
5 def get_name
6 name = at_css('h1').content.strip
7 end
8
9 def get_type
10 breadcrumbs = css('nav ol.breadcrumb li.breadcrumb-item')
11 category = breadcrumbs[1]&.content&.strip
12
13 # These categories have several sub-pages.
14 return category if [
15 'Attribute Types',
16 'Attributes',
17 'Command Line',
18 'Layout Engines',
19 'Output Formats',
20 ].include?(category)
21
22 # Several categories have only one page each. Let's group them together.
23 return 'Documentation'
24 end
25
26 end
27 end
28end
lib/docs/scrapers/graphviz.rb+55−0
@@ -0,0 +1,55 @@
1module Docs
2 class Graphviz < UrlScraper
3 self.name = 'Graphviz'
4 self.slug = 'graphviz'
5 self.type = 'simple'
6
7 self.links = {
8 home: 'https://www.graphviz.org/',
9 code: 'https://gitlab.com/graphviz/graphviz'
10 }
11
12 options[:container] = 'main'
13
14 # These images are too large:
15 # 980KB https://www.graphviz.org/doc/info/plugins.png
16 # 650KB https://www.graphviz.org/Gallery/twopi/twopi2.svg
17 # All other files are under 100KB
18 options[:max_image_size] = 100_000
19
20 # TODO: the UrlScraper is very unreliable on this website.
21 # I often get several errors:
22 # - SSL connect error
23 # - Failure when receiving data from the peer
24 # - was slow to process (30s)
25 # Setting a :rate_limit doesn't help.
26 # We have to figure out a more reliable solution.
27 #options[:rate_limit] = 100
28
29 options[:attribution] = <<-HTML
30 &copy; 2025 The Graphviz Authors<br>
31 Licensed under the Eclipse Public License 1.0.
32 HTML
33
34 html_filters.push 'graphviz/entries', 'graphviz/clean_html'
35
36 self.release = '14.01'
37 self.base_url = 'https://www.graphviz.org/'
38 self.root_path = 'documentation/'
39 options[:only_patterns] = [
40 /^documentation\//,
41 /^doc\//,
42 /^docs\//,
43 ]
44 options[:replace_paths] = {
45 # Redirections:
46 'docs/outputs/cmap/' => 'docs/outputs/imap/',
47 'doc/info/output.html' => 'docs/outputs/',
48 }
49
50 def get_latest_version(opts)
51 tags = get_gitlab_tags('gitlab.com', 'graphviz', 'graphviz', opts)
52 tags[0]['name']
53 end
54 end
55end
public/icons/docs/graphviz/16.png+0−0

Binary file not shown.

public/icons/docs/graphviz/16@2x.png+0−0

Binary file not shown.

public/icons/docs/graphviz/SOURCE+1−0
@@ -0,0 +1 @@
1https://gitlab.com/graphviz/graphviz.gitlab.io/-/blob/main/static/Resources/favicon.png