Update Godot docs to include v4.2 and fix older version scraping

Godot 4.2 was released on Nov 2023. Since v3.5, godot upstream docs have used some new HTML layouts and file structures. Since older versions still need to be scrapable, this introduces another set of versioned filters for godot. This change also makes small changes to the filters to handle the current website markup for the previous godot doc versions. The @gdscript and @globalscope entries can't currently be browsed, because of an encoding mismatch between the frontend and backend; I've identified a possible fix for that but will PR that separately.

Jesse van Herk committed 2 years ago

Commit 70267061 parent 62e27236 files · +126 −28Browse files

Unified · Split

6 files

lib/docs/filters/godot/clean_html.rb +20 −3
lib/docs/filters/godot/clean_html_v2.rb +1 −1
lib/docs/filters/godot/clean_html_v3.rb +27 −0
lib/docs/filters/godot/entries.rb +4 −2
lib/docs/filters/godot/entries_v3.rb +39 −0
lib/docs/scrapers/godot.rb +35 −22

lib/docs/filters/godot/clean_html.rb+20−3

           at_css('h1').content = 'Godot Engine'
           at_css('.admonition.note').remove
         end
+        css('.admonition-grid').remove
-        css('ul[id].simple li:first-child:last-child').each do |node|
+        css('p[id]').each do |node|
           heading = Nokogiri::XML::Node.new 'h3', doc.document
-          heading['id'] = node.parent['id']
+          heading['id'] = node['id']
           heading.children = node.children
-          node.parent.before(heading).remove
+          node.before(heading).remove
         end
         css('h3 strong').each do |node|
         css('a.reference').remove_attr('class')
+        # flatten gdscript+C# example blocks and add language name.
+        css('div[role="tabpanel"]').each do |node|
+          language_label = Nokogiri::XML::Node.new 'strong', doc.document
+          language_name = 'GDScript' if node.at_css('div.highlight-gdscript')
+          language_name = 'C#' if node.at_css('div.highlight-csharp')
+          language_label.content = language_name.to_s
+          node.before(language_label)
+          node.before(node.children).remove
+        end
+        css('div.sphinx-tabs [role="tablist"]').remove
+        # remove the remotely hosted "percent-translated" badge
+        css('a[href^="https://hosted.weblate"]').remove if root_page?
         doc
       end
     end

lib/docs/filters/godot/clean_html_v2.rb+1−1

@@ -4,7 +4,7 @@ module Docs · 3 unmodified lines
4	def call	4	def call
5	if root_page?	5	if root_page?
6	at_css('h1').content = 'Godot Engine'	6	at_css('h1').content = 'Godot Engine'
7	at_css('.admonition.tip').remove	7	at_css('.admonition.caution').remove
8	end	8	end
9		9
10	css('ul[id].simple li:first-child:last-child').each do \|node\|	10	css('ul[id].simple li:first-child:last-child').each do \|node\|

lib/docs/filters/godot/clean_html_v3.rb+27−0

@@ -0,0 +1,27 @@
	1	module Docs
	2	class Godot
	3	class CleanHtmlV3Filter < Filter
	4	def call
	5	if root_page?
	6	at_css('h1').content = 'Godot Engine'
	7	at_css('.admonition.caution').remove
	8	end
	9
	10	css('ul[id].simple li:first-child:last-child').each do \|node\|
	11	heading = Nokogiri::XML::Node.new 'h3', doc.document
	12	heading['id'] = node.parent['id']
	13	heading.children = node.children
	14	node.parent.before(heading).remove
	15	end
	16
	17	css('h3 strong').each do \|node\|
	18	node.before(node.children).remove
	19	end
	20
	21	css('a.reference').remove_attr('class')
	22
	23	doc
	24	end
	25	end
	26	end
	27	end

lib/docs/filters/godot/entries.rb+4−2

@@ -11,7 +11,7 @@ module Docs · 10 unmodified lines
11	if slug.start_with?('getting_started')	11	if slug.start_with?('getting_started')
12	# Getting started sections are different even between different minor	12	# Getting started sections are different even between different minor
13	# versions from v3 so we're programmatically generating them instead.	13	# versions from v3 so we're programmatically generating them instead.
14	"Getting started: " + slug.split('/')[1].tr_s('_', ' ').capitalize	14	'Getting started: ' + slug.split('/')[1].tr_s('_', ' ').capitalize
15	else	15	else
16	name	16	name
17	end	17	end
@@ -20,9 +20,10 @@ module Docs · 2 unmodified lines
20	def additional_entries	20	def additional_entries
21	return [] unless slug.start_with?('classes')	21	return [] unless slug.start_with?('classes')
22		22
23	css('.simple[id]').each_with_object [] do \|node, entries\|	23	css('p[id]').each_with_object [] do \|node, entries\|
24	name = node.at_css('strong').content	24	name = node.at_css('strong').content
25	next if name == self.name	25	next if name == self.name
		26
26	name.prepend "#{self.name}."	27	name.prepend "#{self.name}."
27	name << '()'	28	name << '()'
28	entries << [name, node['id']] unless entries.any? { \|entry\| entry[0] == name }	29	entries << [name, node['id']] unless entries.any? { \|entry\| entry[0] == name }
@@ -32,6 +33,7 @@ module Docs · 3 unmodified lines
32	def include_default_entry?	33	def include_default_entry?
33	return false if subpath.start_with?('getting_started') && subpath.end_with?('index.html')	34	return false if subpath.start_with?('getting_started') && subpath.end_with?('index.html')
34	return false if subpath == 'classes/index.html'	35	return false if subpath == 'classes/index.html'
		36
35	true	37	true
36	end	38	end
37	end	39	end

lib/docs/filters/godot/entries_v3.rb+39−0

@@ -0,0 +1,39 @@
	1	module Docs
	2	class Godot
	3	class EntriesV3Filter < Docs::EntriesFilter
	4	def get_name
	5	name = at_css('h1').content
	6	name.remove! "\u{00B6}" # Remove the pilcrow
	7	name
	8	end
	9
	10	def get_type
	11	if slug.start_with?('getting_started')
	12	# Getting started sections are different even between different minor
	13	# versions from v3 so we're programmatically generating them instead.
	14	"Getting started: " + slug.split('/')[1].tr_s('_', ' ').capitalize
	15	else
	16	name
	17	end
	18	end
	19
	20	def additional_entries
	21	return [] unless slug.start_with?('classes')
	22
	23	css('.simple[id]').each_with_object [] do \|node, entries\|
	24	name = node.at_css('strong').content
	25	next if name == self.name
	26	name.prepend "#{self.name}."
	27	name << '()'
	28	entries << [name, node['id']] unless entries.any? { \|entry\| entry[0] == name }
	29	end
	30	end
	31
	32	def include_default_entry?
	33	return false if subpath.start_with?('getting_started') && subpath.end_with?('index.html')
	34	return false if subpath == 'classes/index.html'
	35	true
	36	end
	37	end
	38	end
	39	end

lib/docs/scrapers/godot.rb+35−22

@@ -5,59 +5,72 @@ module Docs · 4 unmodified lines
5	home: 'https://godotengine.org/',	5	home: 'https://godotengine.org/',
6	code: 'https://github.com/godotengine/godot'	6	code: 'https://github.com/godotengine/godot'
7	}	7	}
		8	# godot docs since 3.5 don't link everything from the index.
		9	self.initial_paths = %w[
		10	getting_started/introduction/index.html
		11	getting_started/step_by_step/index.html
		12	classes/index.html
		13	]
8		14
9	options[:container] = '.document .section'	15	options[:container] = '.document > [itemprop="articleBody"]'
10
11	options[:download_images] = false	16	options[:download_images] = false
12	options[:only_patterns] = [/\Agetting_started\//, /\Aclasses\//]	17	options[:only_patterns] = [%r{\Agetting_started/}, %r{\Aclasses/}]
		18
		19	options[:attribution] = <<-HTML
		20	© 2014–present Juan Linietsky, Ariel Manzur and the Godot community<br>
		21	Licensed under the Creative Commons Attribution Unported License v3.0.
		22	HTML
13		23
14	options[:attribution] = ->(filter) do	24	version '4.2' do
15	if filter.subpath.start_with?('classes')	25	self.release = '4.2.2'
16	<<-HTML	26	self.base_url = "https://docs.godotengine.org/en/#{self.version}/"
17	© 2014–2022 Juan Linietsky, Ariel Manzur, Godot Engine contributors<br>	27	html_filters.push 'godot/entries', 'godot/clean_html', 'sphinx/clean_html'
18	Licensed under the MIT License.
19	HTML
20	else
21	<<-HTML
22	© 2014–2022 Juan Linietsky, Ariel Manzur and the Godot community<br>
23	Licensed under the Creative Commons Attribution Unported License v3.0.
24	HTML
25	end
26	end	28	end
27		29
28	version '3.5' do	30	version '3.5' do
29	self.release = '3.5.1'	31	self.release = '3.5.3'
30	self.base_url = "https://docs.godotengine.org/en/#{self.version}/"	32	self.base_url = "https://docs.godotengine.org/en/#{self.version}/"
31	options[:container] = '.document > [itemprop="articleBody"] > section[id]'	33
		34	# godot 3.5 upstream docs are formatted like godot4
32	html_filters.push 'godot/entries', 'godot/clean_html', 'sphinx/clean_html'	35	html_filters.push 'godot/entries', 'godot/clean_html', 'sphinx/clean_html'
33	end	36	end
34		37
35	version '3.4' do	38	version '3.4' do
36	self.release = '3.4.5'	39	self.release = '3.4.5'
37	self.base_url = "https://docs.godotengine.org/en/#{self.version}/"	40	self.base_url = "https://docs.godotengine.org/en/#{self.version}/"
		41
38	options[:container] = '.document > [itemprop="articleBody"] > section[id]'	42	options[:container] = '.document > [itemprop="articleBody"] > section[id]'
39	html_filters.push 'godot/entries', 'godot/clean_html', 'sphinx/clean_html'	43	html_filters.push 'godot/entries_v3', 'godot/clean_html_v3', 'sphinx/clean_html'
40	end	44	end
41		45
42	version '3.3' do	46	version '3.3' do
43	self.release = '3.3.0'	47	self.release = '3.3.0'
44	self.base_url = "https://docs.godotengine.org/en/#{self.version}/"	48	self.base_url = "https://docs.godotengine.org/en/#{self.version}/"
45	html_filters.push 'godot/entries', 'godot/clean_html', 'sphinx/clean_html'	49	self.initial_paths = %w[/index.html]
		50
		51	options[:only_patterns] = [%r{\Aclasses/}]
		52	options[:container] = '.document .section'
		53	html_filters.push 'godot/entries_v3', 'godot/clean_html_v3', 'sphinx/clean_html'
46	end	54	end
47		55
48	version '3.2' do	56	version '3.2' do
49	self.release = '3.2.3'	57	self.release = '3.2.3'
50	self.base_url = "https://docs.godotengine.org/en/#{self.version}/"	58	self.base_url = "https://docs.godotengine.org/en/#{self.version}/"
51	html_filters.push 'godot/entries', 'godot/clean_html', 'sphinx/clean_html'	59	self.initial_paths = %w[/index.html]
		60
		61	options[:only_patterns] = [%r{\Aclasses/}]
		62	options[:container] = '.document .section'
		63	html_filters.push 'godot/entries_v3', 'godot/clean_html_v3', 'sphinx/clean_html'
52	end	64	end
53		65
54	version '2.1' do	66	version '2.1' do
55	self.release = '2.1.6'	67	self.release = '2.1.6'
56	self.base_url = "https://docs.godotengine.org/en/#{self.version}/"	68	self.base_url = "https://docs.godotengine.org/en/#{self.version}/"
		69	self.initial_paths = %w[/index.html]
57		70
58	options[:skip] = %w(classes/class_@global\ scope.html)	71	options[:skip] = %w(classes/class_@global\ scope.html)
59	options[:only_patterns] = [/\Alearning\//, /\Aclasses\//]	72	options[:only_patterns] = [%r{\Alearning/}, %r{\Aclasses/}]
60		73	options[:container] = '.document .section'
61	html_filters.push 'godot/entries_v2', 'godot/clean_html_v2', 'sphinx/clean_html'	74	html_filters.push 'godot/entries_v2', 'godot/clean_html_v2', 'sphinx/clean_html'
62	end	75	end
63		76