krisyotam/devdocs

.gitignore+1−0

@@ -8,6 +8,7 @@ public/docs/*/ · 7 unmodified lines
8	docs/*/	8	docs/*/
9	!docs/*.md	9	!docs/*.md
10	/vendor	10	/vendor
		11	/coverage
11	*.tar	12	*.tar
12	*.tar.bz2	13	*.tar.bz2
13	*.tar.gz	14	*.tar.gz

Gemfile+1−0

@@ -48,6 +48,7 @@ group :test do · 47 unmodified lines
48	gem 'minitest'	48	gem 'minitest'
49	gem 'rack-test', require: false	49	gem 'rack-test', require: false
50	gem 'rr', require: false	50	gem 'rr', require: false
		51	gem 'simplecov', require: false
51	end	52	end
52		53
53	if ENV['SELENIUM'] == '1'	54	if ENV['SELENIUM'] == '1'

Gemfile.lock+12−1

       sprockets-rails
       tilt
     date (3.5.1)
+    docile (1.4.1)
     drb (2.2.3)
     erb (6.0.1)
     erubi (1.13.1)
       nokogiri (>= 1.12.0)
     method_source (1.1.0)
     mini_portile2 (2.8.9)
-    minitest (5.27.0)
+    minitest (6.0.2)
+      drb (~> 2.0)
+      prism (~> 1.5)
     multi_json (1.15.0)
     mustermann (3.0.3)
       ruby2_keywords (~> 0.0.1)
     pp (0.6.3)
       prettyprint
     prettyprint (0.2.0)
+    prism (1.9.0)
     progress (3.6.0)
     progress_bar (1.3.4)
       highline (>= 1.6)
     sassc-embedded (1.80.8)
       sass-embedded (~> 1.80)
     securerandom (0.3.2)
+    simplecov (0.22.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.13.2)
+    simplecov_json_formatter (0.1.4)
     sinatra (3.2.0)
       mustermann (~> 3.0)
       rack (~> 2.2, >= 2.2.4)
   redcarpet
   rr
   rss
+  simplecov
   sinatra
   sinatra-contrib
   sprockets

README.md+52−51

lib/docs/filters/core/clean_text.rb+6−3

@@ -6,9 +6,12 @@ module Docs · 5 unmodified lines
6		6
7	def call	7	def call
8	return html if context[:clean_text] == false	8	return html if context[:clean_text] == false
9	html.strip!	9
10	while html.gsub!(EMPTY_NODES_RGX, ''); end	10	# Clone frozen literal.
11	html	11	result = html.dup
		12	result.strip!
		13	while result.gsub!(EMPTY_NODES_RGX, ''); end
		14	result
12	end	15	end
13	end	16	end
14	end	17	end

lib/tasks/test.thor+7−0

@@ -26,4 +26,11 @@ class TestCLI < Thor · 25 unmodified lines
26	def app	26	def app
27	require 'app_test'	27	require 'app_test'
28	end	28	end
		29
		30	desc 'coverage', 'Coverage report for core tests'
		31	def coverage
		32	ENV['COVERAGE'] = 'true'
		33	require_relative '../../test/test_helper'
		34	self.all
		35	end
29	end	36	end

test/lib/docs/core/doc_test.rb+393−0

@@ -1,5 +1,6 @@
1	require_relative '../../../test_helper'	1	require_relative '../../../test_helper'
2	require_relative '../../../../lib/docs'	2	require_relative '../../../../lib/docs'
		3	require 'base64'
3		4
4	class DocsDocTest < Minitest::Spec	5	class DocsDocTest < Minitest::Spec
5	let :doc do	6	let :doc do
@@ -237,6 +238,18 @@ class DocsDocTest < Minitest::Spec · 231 unmodified lines
237	doc.store_page(store, 'id')	238	doc.store_page(store, 'id')
238	end	239	end
239	end	240	end
		241
		242	context "when the store can't be opened" do
		243	before do
		244	stub(store).open { raise Docs::SetupError, "oh no" }
		245	end
		246
		247	it "logs an error" do
		248	output = capture_io { doc.store_page(store, 'id') }.first
		249
		250	assert_match "ERROR: oh no", output
		251	end
		252	end
240	end	253	end
241		254
242	describe ".store_pages" do	255	describe ".store_pages" do
@@ -353,6 +366,18 @@ class DocsDocTest < Minitest::Spec · 110 unmodified lines
353	doc.store_pages(store)	366	doc.store_pages(store)
354	end	367	end
355	end	368	end
		369
		370	context "when the store can't be opened" do
		371	before do
		372	stub(store).replace { raise Docs::SetupError, "oh no" }
		373	end
		374
		375	it "logs an error" do
		376	output = capture_io { doc.store_pages(store) }.first
		377
		378	assert_match "ERROR: oh no", output
		379	end
		380	end
356	end	381	end
357		382
358	describe ".versions" do	383	describe ".versions" do
@@ -406,4 +431,372 @@ class DocsDocTest < Minitest::Spec · 47 unmodified lines
406	assert_equal "Outdated minor version", instance.outdated_state('2006-01-02', '2006-02-03')	431	assert_equal "Outdated minor version", instance.outdated_state('2006-01-02', '2006-02-03')
407	end	432	end
408	end	433	end
		434
		435	# PRIVATE CLASS METHODS
		436
		437	describe ".default_slug" do
		438	it "returns name.downcase when name is alphanumeric" do
		439	doc.name = 'FooBar'
		440	assert_equal 'foobar', doc.send(:default_slug)
		441	end
		442
		443	it "returns name.downcase when name contains underscores" do
		444	doc.name = 'Foo_Bar'
		445	assert_equal 'foo_bar', doc.send(:default_slug)
		446	end
		447
		448	it "returns nil when name contains a hyphen" do
		449	doc.name = 'Foo-Bar'
		450	assert_nil doc.send(:default_slug)
		451	end
		452
		453	it "returns nil when name contains a space" do
		454	doc.name = 'Foo Bar'
		455	assert_nil doc.send(:default_slug)
		456	end
		457
		458	it "returns nil when name contains a dot" do
		459	doc.name = 'Foo.Bar'
		460	assert_nil doc.send(:default_slug)
		461	end
		462	end
		463
		464	describe ".store_page?" do
		465	it "returns true when page has non-empty :entries" do
		466	assert doc.send(:store_page?, page)
		467	end
		468
		469	it "returns false when page has empty :entries" do
		470	refute doc.send(:store_page?, page.merge(entries: []))
		471	end
		472
		473	it "returns false when page has nil :entries" do
		474	refute doc.send(:store_page?, page.merge(entries: nil))
		475	end
		476	end
		477
		478	describe ".store_index" do
		479	let :index do
		480	obj = Object.new
		481	stub(obj).to_json { '{"entries":[]}' }
		482	obj
		483	end
		484
		485	context "when read_write is true (default)" do
		486	it "reads old JSON from store" do
		487	mock(store).read('index.json') { '{}' }
		488	stub(store).write
		489	doc.send(:store_index, store, 'index.json', index)
		490	end
		491
		492	it "writes new JSON to store" do
		493	stub(store).read { '{}' }
		494	mock(store).write('index.json', '{"entries":[]}')
		495	doc.send(:store_index, store, 'index.json', index)
		496	end
		497	end
		498
		499	context "when read_write is false" do
		500	it "doesn't read from store" do
		501	dont_allow(store).read
		502	doc.send(:store_index, store, 'index.json', index, false)
		503	end
		504
		505	it "doesn't write to store" do
		506	dont_allow(store).write
		507	doc.send(:store_index, store, 'index.json', index, false)
		508	end
		509	end
		510	end
		511
		512	describe ".store_meta" do
		513	it "writes to META_FILENAME" do
		514	stub(store).size { 0 }
		515	mock(store).write(Docs::Doc::META_FILENAME, anything)
		516	doc.send(:store_meta, store)
		517	end
		518
		519	it "includes :mtime as an integer in the written JSON" do
		520	stub(store).size { 0 }
		521	stub(store).write(Docs::Doc::META_FILENAME, anything) do \|_, json\|
		522	@json = JSON.parse(json)
		523	end
		524	doc.send(:store_meta, store)
		525	assert @json.key?('mtime')
		526	assert_instance_of Integer, @json['mtime']
		527	end
		528
		529	it "includes :db_size from the store" do
		530	stub(store).size(Docs::Doc::DB_FILENAME) { 42 }
		531	stub(store).write(Docs::Doc::META_FILENAME, anything) do \|_, json\|
		532	@json = JSON.parse(json)
		533	end
		534	doc.send(:store_meta, store)
		535	assert_equal 42, @json['db_size']
		536	end
		537	end
		538
		539	# PRIVATE INSTANCE METHODS
		540
		541	describe "#fetch" do
		542	let :instance do
		543	doc.new
		544	end
		545
		546	let :logger do
		547	l = Object.new
		548	stub(l).debug
		549	stub(l).error
		550	l
		551	end
		552
		553	let :opts do
		554	{ logger: logger }
		555	end
		556
		557	let :response do
		558	r = Object.new
		559	stub(r).success? { true }
		560	stub(r).body { 'body' }
		561	r
		562	end
		563
		564	before do
		565	stub(Docs::Request).run { response }
		566	end
		567
		568	it "returns the response body on success" do
		569	assert_equal 'body', instance.send(:fetch, 'http://example.com/', opts)
		570	end
		571
		572	it "calls Request.run with the given url" do
		573	mock(Docs::Request).run('http://example.com/', anything) { response }
		574	instance.send(:fetch, 'http://example.com/', opts)
		575	end
		576
		577	it "adds Authorization header when opts has :github_token and url is GitHub API" do
		578	opts[:github_token] = 'mytoken'
		579	mock(Docs::Request).run(anything, anything) do \|_, request_opts\|
		580	assert_equal 'token mytoken', request_opts[:headers]['Authorization']
		581	response
		582	end
		583	instance.send(:fetch, 'https://api.github.com/repos', opts)
		584	end
		585
		586	it "doesn't add Authorization header for non-GitHub API urls even with :github_token" do
		587	opts[:github_token] = 'mytoken'
		588	mock(Docs::Request).run(anything, anything) do \|_, request_opts\|
		589	refute request_opts[:headers].key?('Authorization')
		590	response
		591	end
		592	instance.send(:fetch, 'http://example.com/', opts)
		593	end
		594
		595	context "when response is unsuccessful" do
		596	before do
		597	stub(response).success? { false }
		598	stub(response).timed_out? { false }
		599	stub(response).code { 404 }
		600	end
		601
		602	it "raises an error" do
		603	assert_raises(RuntimeError) do
		604	instance.send(:fetch, 'http://example.com/', opts)
		605	end
		606	end
		607
		608	it "includes the response code in the error message" do
		609	err = assert_raises(RuntimeError) { instance.send(:fetch, 'http://example.com/', opts) }
		610	assert_match '404', err.message
		611	end
		612	end
		613
		614	context "when response times out" do
		615	before do
		616	stub(response).success? { false }
		617	stub(response).timed_out? { true }
		618	end
		619
		620	it "raises an error with a timeout message" do
		621	err = assert_raises(RuntimeError) { instance.send(:fetch, 'http://example.com/', opts) }
		622	assert_match 'Timed out', err.message
		623	end
		624	end
		625	end
		626
		627	describe "#fetch_doc" do
		628	let :instance do
		629	doc.new
		630	end
		631
		632	let :opts do
		633	{ logger: nil }
		634	end
		635
		636	it "returns a Nokogiri::HTML::Document" do
		637	stub(instance).fetch { '<html><body></body></html>' }
		638	assert_instance_of Nokogiri::HTML::Document, instance.send(:fetch_doc, 'http://example.com/', opts)
		639	end
		640
		641	it "fetches the given url" do
		642	mock(instance).fetch('http://example.com/', opts) { '<html></html>' }
		643	instance.send(:fetch_doc, 'http://example.com/', opts)
		644	end
		645	end
		646
		647	describe "#fetch_json" do
		648	let :instance do
		649	doc.new
		650	end
		651
		652	let :opts do
		653	{ logger: nil }
		654	end
		655
		656	it "returns parsed JSON" do
		657	stub(instance).fetch { '{"key":"value"}' }
		658	assert_equal({ 'key' => 'value' }, instance.send(:fetch_json, 'http://example.com/', opts))
		659	end
		660
		661	it "fetches the given url" do
		662	mock(instance).fetch('http://example.com/', opts) { '{}' }
		663	instance.send(:fetch_json, 'http://example.com/', opts)
		664	end
		665	end
		666
		667	describe "#get_npm_version" do
		668	let :instance do
		669	doc.new
		670	end
		671
		672	let :opts do
		673	{ logger: nil }
		674	end
		675
		676	it "returns the 'latest' dist-tag by default" do
		677	stub(instance).fetch_json { { 'dist-tags' => { 'latest' => '1.2.3' } } }
		678	assert_equal '1.2.3', instance.send(:get_npm_version, 'some-package', opts)
		679	end
		680
		681	it "returns a specific dist-tag when given" do
		682	stub(instance).fetch_json { { 'dist-tags' => { 'next' => '2.0.0-beta' } } }
		683	assert_equal '2.0.0-beta', instance.send(:get_npm_version, 'some-package', opts, 'next')
		684	end
		685
		686	it "fetches from the npm registry URL" do
		687	mock(instance).fetch_json('https://registry.npmjs.com/some-package', opts) { { 'dist-tags' => { 'latest' => '1.0.0' } } }
		688	instance.send(:get_npm_version, 'some-package', opts)
		689	end
		690	end
		691
		692	describe "#get_latest_github_release" do
		693	let :instance do
		694	doc.new
		695	end
		696
		697	let :opts do
		698	{ logger: nil }
		699	end
		700
		701	it "strips a leading 'v' from the tag name" do
		702	stub(instance).fetch_json { { 'tag_name' => 'v1.2.3' } }
		703	assert_equal '1.2.3', instance.send(:get_latest_github_release, 'owner', 'repo', opts)
		704	end
		705
		706	it "returns the tag name unchanged when it doesn't start with 'v'" do
		707	stub(instance).fetch_json { { 'tag_name' => '1.2.3' } }
		708	assert_equal '1.2.3', instance.send(:get_latest_github_release, 'owner', 'repo', opts)
		709	end
		710
		711	it "fetches from the correct GitHub releases URL" do
		712	mock(instance).fetch_json('https://api.github.com/repos/owner/repo/releases/latest', opts) { { 'tag_name' => '1.0.0' } }
		713	instance.send(:get_latest_github_release, 'owner', 'repo', opts)
		714	end
		715	end
		716
		717	describe "#get_github_tags" do
		718	let :instance do
		719	doc.new
		720	end
		721
		722	let :opts do
		723	{ logger: nil }
		724	end
		725
		726	it "fetches from the correct GitHub tags URL" do
		727	mock(instance).fetch_json('https://api.github.com/repos/owner/repo/tags', opts) { [] }
		728	instance.send(:get_github_tags, 'owner', 'repo', opts)
		729	end
		730
		731	it "returns the tags list" do
		732	tags = [{ 'name' => 'v1.0' }]
		733	stub(instance).fetch_json { tags }
		734	assert_equal tags, instance.send(:get_github_tags, 'owner', 'repo', opts)
		735	end
		736	end
		737
		738	describe "#get_github_file_contents" do
		739	let :instance do
		740	doc.new
		741	end
		742
		743	let :opts do
		744	{ logger: nil }
		745	end
		746
		747	it "returns Base64-decoded file contents" do
		748	stub(instance).fetch_json { { 'content' => ::Base64.encode64('hello world') } }
		749	assert_equal 'hello world', instance.send(:get_github_file_contents, 'owner', 'repo', 'README.md', opts)
		750	end
		751
		752	it "fetches from the correct GitHub contents URL" do
		753	mock(instance).fetch_json('https://api.github.com/repos/owner/repo/contents/README.md', opts) { { 'content' => ::Base64.encode64('') } }
		754	instance.send(:get_github_file_contents, 'owner', 'repo', 'README.md', opts)
		755	end
		756	end
		757
		758	describe "#get_latest_github_commit_date" do
		759	let :instance do
		760	doc.new
		761	end
		762
		763	let :opts do
		764	{ logger: nil }
		765	end
		766
		767	it "returns the epoch timestamp of the latest commit" do
		768	date_str = '2023-06-15T12:00:00Z'
		769	commits = [{ 'commit' => { 'author' => { 'date' => date_str } } }]
		770	stub(instance).fetch_json { commits }
		771	assert_equal Date.iso8601(date_str).to_time.to_i, instance.send(:get_latest_github_commit_date, 'owner', 'repo', opts)
		772	end
		773
		774	it "fetches from the correct GitHub commits URL" do
		775	date_str = '2023-06-15T12:00:00Z'
		776	commits = [{ 'commit' => { 'author' => { 'date' => date_str } } }]
		777	mock(instance).fetch_json('https://api.github.com/repos/owner/repo/commits', opts) { commits }
		778	instance.send(:get_latest_github_commit_date, 'owner', 'repo', opts)
		779	end
		780	end
		781
		782	describe "#get_gitlab_tags" do
		783	let :instance do
		784	doc.new
		785	end
		786
		787	let :opts do
		788	{ logger: nil }
		789	end
		790
		791	it "fetches from the correct GitLab tags URL" do
		792	mock(instance).fetch_json('https://gitlab.example.com/api/v4/projects/group%2Fproject/repository/tags', opts) { [] }
		793	instance.send(:get_gitlab_tags, 'gitlab.example.com', 'group', 'project', opts)
		794	end
		795
		796	it "returns the tags list" do
		797	tags = [{ 'name' => 'v1.0' }]
		798	stub(instance).fetch_json { tags }
		799	assert_equal tags, instance.send(:get_gitlab_tags, 'gitlab.example.com', 'group', 'project', opts)
		800	end
		801	end
409	end	802	end

test/lib/docs/core/entry_index_test.rb+66−0

@@ -154,6 +154,72 @@ class DocsEntryIndexTest < Minitest::Spec · 153 unmodified lines
154	end	154	end
155	end	155	end
156		156
		157	describe "#sort_fn" do
		158	def sort(a, b)
		159	index.send(:sort_fn, a, b)
		160	end
		161
		162	context "when neither string starts with a digit 1-9" do
		163	it "uses case-insensitive string comparison" do
		164	assert_operator sort('apple', 'banana'), :<, 0
		165	assert_operator sort('banana', 'apple'), :>, 0
		166	end
		167
		168	it "returns 0 for case-insensitive equal strings" do
		169	assert_equal 0, sort('Apple', 'apple')
		170	end
		171
		172	it "treats strings starting with '0' as non-numeric" do
		173	assert_equal '0apple'.casecmp('banana'), sort('0apple', 'banana')
		174	end
		175	end
		176
		177	context "when at least one string starts with 1-9 but neither contains version dots" do
		178	it "uses case-insensitive string comparison" do
		179	# lexicographic: '1' < '2', so '10 Test' < '2 Test'
		180	assert_operator sort('10 Test', '2 Test'), :<, 0
		181	end
		182
		183	it "returns 0 for case-insensitive equal strings" do
		184	assert_equal 0, sort('2 Test', '2 test')
		185	end
		186	end
		187
		188	context "when one string has version dots and the other does not" do
		189	it "sorts the dotted string before the undotted one" do
		190	assert_operator sort('1.2. Test', '2 Test'), :<, 0
		191	end
		192
		193	it "sorts the undotted string after the dotted one" do
		194	assert_operator sort('2 Test', '1.2. Test'), :>, 0
		195	end
		196
		197	it "sorts the dotted string before a non-numeric string" do
		198	assert_operator sort('1.2. Test', 'abc'), :<, 0
		199	end
		200	end
		201
		202	context "when both strings have version dots" do
		203	it "sorts numerically rather than lexicographically" do
		204	assert_operator sort('1.9. Test', '1.10. Test'), :<, 0
		205	assert_operator sort('4.20. Test', '4.3. Test'), :>, 0
		206	end
		207
		208	it "pads shorter version depths with zeros before comparing" do
		209	assert_operator sort('4. Test', '4.2. Test'), :<, 0
		210	assert_operator sort('4.2. Test', '4. Test'), :>, 0
		211	end
		212
		213	it "compares multi-level versions against shorter ones correctly" do
		214	assert_operator sort('4.2.2. Test', '4.3. Test'), :<, 0
		215	end
		216
		217	it "returns 0 for identical version strings" do
		218	assert_equal 0, sort('1.2. Test', '1.2. Test')
		219	end
		220	end
		221	end
		222
157	describe "#to_json" do	223	describe "#to_json" do
158	it "returns the JSON string for #as_json" do	224	it "returns the JSON string for #as_json" do
159	stub(index).as_json { { entries: [1], types: [2] } }	225	stub(index).as_json { { entries: [1], types: [2] } }

test/lib/docs/core/filter_test.rb+38−0

@@ -8,6 +8,12 @@ class DocsFilterTest < Minitest::Spec · 7 unmodified lines
8	before do	8	before do
9	context[:base_url] = 'http://example.com/path'	9	context[:base_url] = 'http://example.com/path'
10	context[:url] = 'http://example.com/path/file'	10	context[:url] = 'http://example.com/path/file'
		11	context[:version] = '6.1'
		12	context[:release] = '6.1.1'
		13	context[:links] = {
		14	home: 'homunculus',
		15	code: 'codified'
		16	}
11	end	17	end
12		18
13	describe "#subpath" do	19	describe "#subpath" do
@@ -55,6 +61,24 @@ class DocsFilterTest < Minitest::Spec · 41 unmodified lines
55	end	61	end
56	end	62	end
57		63
		64	describe "#version" do
		65	it "returns the version" do
		66	assert_equal filter.version, "6.1"
		67	end
		68	end
		69
		70	describe "#release" do
		71	it "returns the release" do
		72	assert_equal filter.release, "6.1.1"
		73	end
		74	end
		75
		76	describe "#links" do
		77	it "returns the links hash" do
		78	assert_equal filter.links, { home: 'homunculus', code: 'codified' }
		79	end
		80	end
		81
58	describe "#root_page?" do	82	describe "#root_page?" do
59	it "returns true when #subpath is blank" do	83	it "returns true when #subpath is blank" do
60	stub(filter).subpath { '' }	84	stub(filter).subpath { '' }
@@ -176,4 +200,18 @@ class DocsFilterTest < Minitest::Spec · 115 unmodified lines
176	refute filter.absolute_url_string?('http')	200	refute filter.absolute_url_string?('http')
177	end	201	end
178	end	202	end
		203
		204	describe "#clean_path" do
		205	it "replaces some special characters with dashes" do
		206	bad_path = "/horrible!url:design;huh".freeze
		207	good_path = "/horrible-url-design-huh".freeze
		208	assert_equal filter.clean_path(bad_path), good_path
		209	end
		210
		211	it "replaces + with the word 'plus'" do
		212	bad_path = "/c++".freeze
		213	good_path = "/c_plus__plus_".freeze
		214	assert_equal filter.clean_path(bad_path), good_path
		215	end
		216	end
179	end	217	end

test/lib/docs/core/scraper_test.rb+162−0

@@ -510,4 +510,166 @@ class DocsScraperTest < Minitest::Spec · 509 unmodified lines
510	assert_equal Docs, pipeline.instrumentation_service	510	assert_equal Docs, pipeline.instrumentation_service
511	end	511	end
512	end	512	end
		513
		514	# PRIVATE INSTANCE METHODS
		515
		516	describe "#url_for" do
		517	it "returns the root url when path is empty" do
		518	assert_equal scraper.root_url.to_s, scraper.send(:url_for, '')
		519	end
		520
		521	it "returns the root url when path is '/'" do
		522	assert_equal scraper.root_url.to_s, scraper.send(:url_for, '/')
		523	end
		524
		525	it "joins base_url and path for a non-root path" do
		526	stub(Scraper).base_url { 'http://example.com/' }
		527	assert_equal 'http://example.com/page', scraper.send(:url_for, '/page')
		528	end
		529	end
		530
		531	describe "#parse" do
		532	it "returns a two-element array" do
		533	response.body = '<html><head><title>T</title></head><body></body></html>'
		534	assert_equal 2, scraper.send(:parse, response).length
		535	end
		536
		537	it "extracts the title from an HTML document" do
		538	response.body = '<html><head><title>My Title</title></head><body></body></html>'
		539	_, title = scraper.send(:parse, response)
		540	assert_equal 'My Title', title
		541	end
		542
		543	it "returns nil title for an HTML fragment" do
		544	response.body = '<div>just a fragment</div>'
		545	_, title = scraper.send(:parse, response)
		546	assert_nil title
		547	end
		548	end
		549
		550	describe "#pipeline_context" do
		551	it "returns options merged with the response url" do
		552	stub(scraper).options { { base_url: 'http://example.com/' } }
		553	response.url = 'http://example.com/page'
		554	context = scraper.send(:pipeline_context, response)
		555	assert_equal 'http://example.com/page', context[:url]
		556	assert_equal 'http://example.com/', context[:base_url]
		557	end
		558
		559	it "does not mutate the options hash" do
		560	opts = { base_url: 'http://example.com/' }
		561	stub(scraper).options { opts }
		562	response.url = 'url'
		563	scraper.send(:pipeline_context, response)
		564	refute opts.key?(:url)
		565	end
		566	end
		567
		568	describe "#process_response" do
		569	before do
		570	stub(scraper).parse(response) { ['<div></div>', 'My Title'] }
		571	response.url = 'http://example.com/'
		572	end
		573
		574	it "returns a hash" do
		575	assert_instance_of Hash, scraper.send(:process_response, response)
		576	end
		577
		578	it "passes :html_title from parse into the pipeline context" do
		579	mock(scraper.pipeline).call(anything, anything, anything) do \|_, context, _\|
		580	assert_equal 'My Title', context[:html_title]
		581	end
		582	scraper.send(:process_response, response)
		583	end
		584	end
		585
		586	describe "#with_filters" do
		587	it "uses only the given filters inside the block" do
		588	scraper.send(:with_filters) do
		589	@filters = scraper.pipeline.filters
		590	end
		591	assert_equal [], @filters
		592	end
		593
		594	it "resets the pipeline after the block" do
		595	pipeline_before = scraper.pipeline
		596	scraper.send(:with_filters) {}
		597	refute_same pipeline_before, scraper.pipeline
		598	end
		599	end
		600
		601	describe "#additional_options" do
		602	it "returns an empty hash" do
		603	assert_equal({}, scraper.send(:additional_options))
		604	end
		605	end
		606
		607	class FixedScraper < Scraper
		608	include Docs::Scraper::FixInternalUrlsBehavior
		609	end
		610
		611	describe "FixInternalUrlsBehavior" do
		612	let :fixed_scraper do
		613	FixedScraper.new.tap { \|s\| s.extend FakeInstrumentation }
		614	end
		615
		616	describe ".with_internal_urls (private)" do
		617	it "sets .internal_urls to the result of fetch_internal_urls during the block" do
		618	any_instance_of(FixedScraper) do \|instance\|
		619	stub(instance).fetch_internal_urls { ['url1', 'url2'] }
		620	end
		621	FixedScraper.send(:with_internal_urls) do
		622	@urls = FixedScraper.internal_urls
		623	end
		624	assert_equal ['url1', 'url2'], @urls
		625	end
		626
		627	it "clears .internal_urls after the block" do
		628	any_instance_of(FixedScraper) do \|instance\|
		629	stub(instance).fetch_internal_urls { [] }
		630	end
		631	FixedScraper.send(:with_internal_urls) {}
		632	assert_nil FixedScraper.internal_urls
		633	end
		634	end
		635
		636	describe "#additional_options (private)" do
		637	it "returns an empty hash when .internal_urls is nil" do
		638	stub(FixedScraper).internal_urls { nil }
		639	assert_equal({}, fixed_scraper.send(:additional_options))
		640	end
		641
		642	it "returns fixed_internal_urls: true when .internal_urls is set" do
		643	stub(FixedScraper).internal_urls { ['url'] }
		644	assert fixed_scraper.send(:additional_options)[:fixed_internal_urls]
		645	end
		646
		647	it "sets :only to a Set of .internal_urls" do
		648	stub(FixedScraper).internal_urls { ['url1', 'url2'] }
		649	assert_equal Set.new(['url1', 'url2']), fixed_scraper.send(:additional_options)[:only]
		650	end
		651
		652	it "sets :only_patterns to nil" do
		653	stub(FixedScraper).internal_urls { ['url'] }
		654	assert_nil fixed_scraper.send(:additional_options)[:only_patterns]
		655	end
		656
		657	it "sets :skip to nil" do
		658	stub(FixedScraper).internal_urls { ['url'] }
		659	assert_nil fixed_scraper.send(:additional_options)[:skip]
		660	end
		661	end
		662
		663	describe "#process_response (private)" do
		664	before do
		665	stub(fixed_scraper).parse(response) { ['<div></div>', nil] }
		666	response.url = 'http://example.com/page'
		667	end
		668
		669	it "merges :response_url from the response into the result" do
		670	result = fixed_scraper.send(:process_response, response)
		671	assert_equal 'http://example.com/page', result[:response_url]
		672	end
		673	end
		674	end
513	end	675	end

test/lib/docs/core/scrapers/url_scraper_test.rb+62−0

@@ -85,6 +85,38 @@ class DocsUrlScraperTest < Minitest::Spec · 84 unmodified lines
85	stub(Docs::Requester).run { 'response' }	85	stub(Docs::Requester).run { 'response' }
86	assert_equal 'response', result	86	assert_equal 'response', result
87	end	87	end
		88
		89	context "when options[:rate_limit] is set" do
		90	before do
		91	@saved_limiter = Docs::UrlScraper.class_variable_get(:@@rate_limiter)
		92	Docs::UrlScraper.class_variable_set(:@@rate_limiter, nil)
		93	stub(Scraper).options { { rate_limit: 5 } }
		94	stub(Docs::Requester).run
		95	end
		96
		97	after do
		98	Docs::UrlScraper.class_variable_set(:@@rate_limiter, @saved_limiter)
		99	end
		100
		101	it "creates a RateLimiter with the given limit" do
		102	stub(Typhoeus).before
		103	scraper.send :request_all, 'urls'
		104	assert_equal 5, Docs::UrlScraper.class_variable_get(:@@rate_limiter).limit
		105	end
		106
		107	it "registers the RateLimiter with Typhoeus.before" do
		108	mock(Typhoeus).before
		109	scraper.send :request_all, 'urls'
		110	end
		111
		112	it "updates the existing RateLimiter's limit instead of creating a new one" do
		113	existing = Docs::UrlScraper::RateLimiter.new(3)
		114	Docs::UrlScraper.class_variable_set(:@@rate_limiter, existing)
		115	dont_allow(Typhoeus).before
		116	scraper.send :request_all, 'urls'
		117	assert_equal 5, existing.limit
		118	end
		119	end
88	end	120	end
89		121
90	describe "#process_response?" do	122	describe "#process_response?" do
@@ -120,4 +152,34 @@ class DocsUrlScraperTest < Minitest::Spec · 29 unmodified lines
120	assert result	152	assert result
121	end	153	end
122	end	154	end
		155
		156	describe "RateLimiter" do
		157	let :limiter do
		158	Docs::UrlScraper::RateLimiter.new(3)
		159	end
		160
		161	describe "#initialize" do
		162	it "sets the limit" do
		163	assert_equal 3, limiter.limit
		164	end
		165	end
		166
		167	describe "#limit=" do
		168	it "updates the limit" do
		169	limiter.limit = 10
		170	assert_equal 10, limiter.limit
		171	end
		172	end
		173
		174	describe "#to_proc" do
		175	it "returns a Proc" do
		176	assert_instance_of Proc, limiter.to_proc
		177	end
		178
		179	it "returns a proc that calls #call" do
		180	mock(limiter).call
		181	limiter.to_proc.call
		182	end
		183	end
		184	end
123	end	185	end

test/lib/docs/core/subscriber_test.rb+98−0

@@ -0,0 +1,98 @@
	1	# frozen_string_literal: true
	2
	3	require_relative '../../../test_helper'
	4	require_relative '../../../../lib/docs'
	5
	6	class DocsSubscriberTest < Minitest::Spec
	7	let(:subscriber) { Docs::Subscriber.new }
	8
	9	describe "#format_url" do
	10	it "removes http:// scheme" do
	11	assert_equal 'devdocs.test.co/path', subscriber.send(:format_url, 'http://devdocs.test.co/path')
	12	end
	13
	14	it "removes https:// scheme" do
	15	assert_equal 'devdocs.test.co/path', subscriber.send(:format_url, 'https://devdocs.test.co/path')
	16	end
	17
	18	it "returns non-http urls unchanged" do
	19	assert_equal 'devdocs.test.co/path', subscriber.send(:format_url, 'devdocs.test.co/path')
	20	end
	21	end
	22
	23	describe "#format_path" do
	24	it "removes the current working directory prefix" do
	25	full_path = File.join(Dir.pwd, 'some/file.rb')
	26	assert_equal 'some/file.rb', subscriber.send(:format_path, full_path)
	27	end
	28
	29	it "returns paths outside the cwd unchanged" do
	30	assert_equal '/other/path', subscriber.send(:format_path, '/other/path')
	31	end
	32
	33	it "calls to_s on the argument" do
	34	path = Object.new
	35	def path.to_s; '/other/path'; end
	36	assert_equal '/other/path', subscriber.send(:format_path, path)
	37	end
	38	end
	39
	40	describe "#justify" do
	41	describe "when terminal_width is nil" do
	42	before { subscriber.instance_variable_set(:@terminal_width, nil) }
	43
	44	it "returns the string unchanged" do
	45	assert_equal 'hello', subscriber.send(:justify, 'hello')
	46	end
	47	end
	48
	49	describe "when terminal_width is set" do
	50	before { subscriber.instance_variable_set(:@terminal_width, 20) }
	51
	52	it "pads a short string to terminal width" do
	53	result = subscriber.send(:justify, 'hi')
	54	assert_equal 20, result.length
	55	assert result.start_with?('hi')
	56	end
	57
	58	it "truncates a long string to terminal width" do
	59	result = subscriber.send(:justify, 'a' * 30)
	60	assert_equal 20, result.length
	61	end
	62
	63	it "reserves space for a trailing tag and pads the rest" do
	64	result = subscriber.send(:justify, 'hi [tag]')
	65	assert_equal 20, result.length
	66	assert result.end_with?('[tag]')
	67	end
	68
	69	it "truncates the non-tag part when the string is too long" do
	70	result = subscriber.send(:justify, ('a' * 30) + ' [tag]')
	71	assert_equal 20, result.length
	72	assert result.end_with?('[tag]')
	73	end
	74	end
	75	end
	76
	77	describe "#terminal_width" do
	78	it "returns nil when stdout is not a tty" do
	79	old_stdout = $stdout
	80	$stdout = StringIO.new
	81	sub = Docs::Subscriber.new
	82	assert_nil sub.send(:terminal_width)
	83	ensure
	84	$stdout = old_stdout
	85	end
	86
	87	it "reads COLUMNS env var when stdout is a tty" do
	88	sub = Docs::Subscriber.new
	89	# Override tty? on the instance to avoid needing a real tty
	90	def sub.tty?; true; end
	91	old_columns = ENV.delete('COLUMNS')
	92	ENV['COLUMNS'] = '666'
	93	assert_equal 666, sub.send(:terminal_width)
	94	ensure
	95	ENV['COLUMNS'] = old_columns
	96	end
	97	end
	98	end

test/lib/docs/filters/core/attribution_test.rb+39−0

@@ -0,0 +1,39 @@
	1	require_relative '../../../../test_helper'
	2	require_relative '../../../../../lib/docs'
	3
	4	class AttributionFilterTest < Minitest::Spec
	5	include FilterTestHelper
	6	self.filter_class = Docs::AttributionFilter
	7	self.filter_type = 'html'
	8
	9	before do
	10	@body = "<div class=\"doc\">Hello!</div>"
	11	end
	12
	13	context "when attribution is a string" do
	14	before do
	15	context[:attribution] = "<span id=\"mtwain\">Copyright 2033 Mark Twain</span>"
	16	context[:base_url] = "q"
	17	end
	18
	19	it "adds it to the document" do
	20	assert_equal filter_output.at_css('#mtwain').inner_html, "Copyright 2033 Mark Twain"
	21	refute_nil filter_output.at_css('p._attribution-p'), "Attribution node should exist"
	22	refute_nil filter_output.at_css('.doc'), "Existing doc should still be present"
	23	refute_nil filter_output.at_css('a._attribution-link'), "Attribution link should exist"
	24	end
	25	end
	26
	27	context "when attribution is a Proc" do
	28	before do
	29	context[:attribution] = ->(filter) {
	30	"<span id=\"mtwain\">Copyright 2034 Mark Twain</span>"
	31	}
	32	context[:base_url] = "q"
	33	end
	34
	35	it "adds its output to the document" do
	36	assert_equal filter_output.at_css('#mtwain').inner_html, "Copyright 2034 Mark Twain"
	37	end
	38	end
	39	end

test/lib/docs/filters/core/clean_local_urls_test.rb+22−0

@@ -0,0 +1,22 @@
	1	require_relative '../../../../test_helper'
	2	require_relative '../../../../../lib/docs'
	3
	4	class CleanLocalUrlsFilterTest < Minitest::Spec
	5	include FilterTestHelper
	6	self.filter_class = Docs::CleanLocalUrlsFilter
	7
	8	it "removes images/iframes referencing localhost" do
	9	context[:base_url] = 'http://localhost'
	10	@body = '<img src="http://localhost/img.png"><iframe src="http://localhost/frame"></iframe>'
	11	assert_empty filter_output.css('img, iframe')
	12	end
	13
	14	it "turns links referencing localhost into plain text" do
	15	context[:base_url] = 'http://localhost'
	16	@body = '<a href="http://localhost/page">Link</a>'
	17	node = filter_output.at_css('span')
	18	assert node, 'expected a <span> node'
	19	refute node['href']
	20	assert_equal 'Link', node.text
	21	end
	22	end

test/lib/docs/filters/core/images_test.rb+133−0

@@ -0,0 +1,133 @@
	1	require_relative '../../../../test_helper'
	2	require_relative '../../../../../lib/docs'
	3
	4	class ImagesFilterTest < Minitest::Spec
	5	include FilterTestHelper
	6	self.filter_class = Docs::ImagesFilter
	7
	8	IMG_SRC = 'http://example.com/img.png'
	9	IMG_BODY = '<img src="http://example.com/img.png">'
	10
	11	before do
	12	Docs::ImagesFilter.cache.clear
	13	context[:optimize_images] = false
	14	end
	15
	16	def make_response(success: true, mime_type: 'image/png', content_length: 100, body: 'imgdata', code: 200)
	17	OpenStruct.new(success?: success, mime_type: mime_type, content_length: content_length, body: body, code: code)
	18	end
	19
	20	def stub_request(response)
	21	stub(Docs::Request).run { \|_url, &block\| block.call(response) }
	22	end
	23
	24	context "when download_images is false" do
	25	it "returns doc unchanged and makes no requests" do
	26	context[:download_images] = false
	27	@body = IMG_BODY
	28	dont_allow(Docs::Request).run
	29	assert_equal IMG_SRC, filter_output.at_css('img')['src']
	30	end
	31	end
	32
	33	context "with a data: URL" do
	34	it "skips the image and makes no request" do
	35	@body = '<img src="data:image/png;base64,aaaaaa2320345bbb234">'
	36	dont_allow(Docs::Request).run
	37	assert_equal 'data:image/png;base64,aaaaaa2320345bbb234', filter_output.at_css('img')['src']
	38	end
	39	end
	40
	41	context "with a non-http(s) URL" do
	42	it "skips the image and makes no request" do
	43	@body = '<img src="ftp://example.com/img.png">'
	44	dont_allow(Docs::Request).run
	45	filter_output
	46	end
	47	end
	48
	49	context "with a successful image response" do
	50	it "replaces src with a base64 data URI" do
	51	@body = IMG_BODY
	52	stub_request make_response(body: 'imgdata', mime_type: 'image/png', content_length: 7)
	53	expected = "data:image/png;base64,#{Base64.strict_encode64('imgdata')}"
	54	assert_equal expected, filter_output.at_css('img')['src']
	55	end
	56
	57	it "caches the data URI for subsequent calls" do
	58	@body = IMG_BODY
	59	stub_request make_response(body: 'imgdata', mime_type: 'image/png', content_length: 7)
	60	filter_output
	61	assert Docs::ImagesFilter.cache.key?(IMG_SRC)
	62	assert Docs::ImagesFilter.cache[IMG_SRC].start_with?('data:')
	63	end
	64	end
	65
	66	context "when response is not successful" do
	67	it "instruments broken.image" do
	68	@body = IMG_BODY
	69	stub_request make_response(success: false, code: 404)
	70	@called = false
	71	filter.subscribe('broken.image') { @called = true }
	72	filter_output
	73	assert @called
	74	end
	75
	76	it "does not alter the src" do
	77	@body = IMG_BODY
	78	stub_request make_response(success: false, code: 404)
	79	assert_equal IMG_SRC, filter_output.at_css('img')['src']
	80	end
	81	end
	82
	83	context "when response mime type is not image/" do
	84	it "instruments invalid.image" do
	85	@body = IMG_BODY
	86	stub_request make_response(mime_type: 'text/html')
	87	@called = false
	88	filter.subscribe('invalid.image') { @called = true }
	89	filter_output
	90	assert @called
	91	end
	92	end
	93
	94	context "when an exception is raised during the request" do
	95	it "instruments error.image" do
	96	@body = IMG_BODY
	97	stub(Docs::Request).run { raise 'connection error' }
	98	@called = false
	99	filter.subscribe('error.image') { @called = true }
	100	filter_output
	101	assert @called
	102	end
	103	end
	104
	105	context "with a cached src" do
	106	it "uses the cached data URI without making a request" do
	107	cached = "data:image/png;base64,#{Base64.strict_encode64('cached')}"
	108	Docs::ImagesFilter.cache[IMG_SRC] = cached
	109	@body = IMG_BODY
	110	dont_allow(Docs::Request).run
	111	assert_equal cached, filter_output.at_css('img')['src']
	112	end
	113
	114	it "leaves src unchanged when cache marks the URL as failed" do
	115	Docs::ImagesFilter.cache[IMG_SRC] = false
	116	@body = IMG_BODY
	117	dont_allow(Docs::Request).run
	118	assert_equal IMG_SRC, filter_output.at_css('img')['src']
	119	end
	120	end
	121
	122	context "when optimize_images is not disabled" do
	123	it "passes image data through optimize_image_data" do
	124	@body = IMG_BODY
	125	optimized = 'optimized_imgdata'
	126	stub_request make_response(body: 'imgdata', mime_type: 'image/png', content_length: 7)
	127	stub(Docs::ImagesFilter).optimize_image_data('imgdata') { optimized }
	128	context.delete(:optimize_images)
	129	expected = "data:image/png;base64,#{Base64.strict_encode64(optimized)}"
	130	assert_equal expected, filter_output.at_css('img')['src']
	131	end
	132	end
	133	end

test/test_helper.rb+10−0

@@ -1,3 +1,13 @@
		1	# Flag set by Thor coverage task.
		2	if ENV['COVERAGE'] == 'true'
		3	require 'simplecov'
		4
		5	# Focus only on core app files.
		6	SimpleCov.add_filter {\|src\| !(src.filename =~ /\/core\//)}
		7
		8	SimpleCov.start 'rails'
		9	end
		10
1	ENV['RACK_ENV'] = 'test'	11	ENV['RACK_ENV'] = 'test'
2		12
3	require 'bundler/setup'	13	require 'bundler/setup'

@@ -10,9 +10,9 @@ Please reach out to the community on [Discord](https://discord.gg/PRyKn3Vbay) if · 9 unmodified lines
10		10
11	Keep track of development news:	11	Keep track of development news:
12		12
13	* Join the `#contributors` chat room on [Discord](https://discord.gg/PRyKn3Vbay)	13	- Join the `#contributors` chat room on [Discord](https://discord.gg/PRyKn3Vbay)
14	* Watch the repository on [GitHub](https://github.com/freeCodeCamp/devdocs/subscription)	14	- Watch the repository on [GitHub](https://github.com/freeCodeCamp/devdocs/subscription)
15	* Follow [@DevDocs](https://twitter.com/DevDocs) on Twitter	15	- Follow [@DevDocs](https://twitter.com/DevDocs) on Twitter
16		16
17	Table of Contents: [Quick Start](#quick-start) · [Vision](#vision) · [App](#app) · [Scraper](#scraper) · [Commands](#available-commands) · [Contributing](#contributing) · [Documentation](#documentation) · [Related Projects](#related-projects) · [License](#copyright--license) · [Questions?](#questions)	17	Table of Contents: [Quick Start](#quick-start) · [Vision](#vision) · [App](#app) · [Scraper](#scraper) · [Commands](#available-commands) · [Contributing](#contributing) · [Documentation](#documentation) · [Related Projects](#related-projects) · [License](#copyright--license) · [Questions?](#questions)
18		18
@@ -29,6 +29,7 @@ docker run --name devdocs -d -p 9292:9292 ghcr.io/freecodecamp/devdocs:latest · 10 unmodified lines
29	```	29	```
30		30
31	This will start DevDocs at [localhost:9292](http://localhost:9292). We provide both regular and Alpine-based images:	31	This will start DevDocs at [localhost:9292](http://localhost:9292). We provide both regular and Alpine-based images:
		32
32	- `ghcr.io/freecodecamp/devdocs:latest` - Standard image	33	- `ghcr.io/freecodecamp/devdocs:latest` - Standard image
33	- `ghcr.io/freecodecamp/devdocs:latest-alpine` - Alpine-based (smaller size)	34	- `ghcr.io/freecodecamp/devdocs:latest-alpine` - Alpine-based (smaller size)
34		35
@@ -70,14 +71,14 @@ DevDocs aims to make reading and searching reference documentation fast, easy an · 35 unmodified lines
70		71
71	The app's main goals are to:	72	The app's main goals are to:
72		73
73	* Keep load times as short as possible	74	- Keep load times as short as possible
74	* Improve the quality, speed, and order of search results	75	- Improve the quality, speed, and order of search results
75	* Maximize the use of caching and other performance optimizations	76	- Maximize the use of caching and other performance optimizations
76	* Maintain a clean and readable user interface	77	- Maintain a clean and readable user interface
77	* Be fully functional offline	78	- Be fully functional offline
78	* Support full keyboard navigation	79	- Support full keyboard navigation
79	* Reduce “context switch” by using a consistent typography and design across all documentations	80	- Reduce “context switch” by using a consistent typography and design across all documentations
80	* Reduce clutter by focusing on a specific category of content (API/reference) and indexing only the minimum useful to most developers.	81	- Reduce clutter by focusing on a specific category of content (API/reference) and indexing only the minimum useful to most developers.
81		82
82	Note: DevDocs is neither a programming guide nor a search engine. All our content is pulled from third-party sources and the project doesn't intend to compete with full-text search engines. Its backbone is metadata; each piece of content is identified by a unique, "obvious" and short string. Tutorials, guides and other content that don't meet this requirement are outside the scope of the project.	83	Note: DevDocs is neither a programming guide nor a search engine. All our content is pulled from third-party sources and the project doesn't intend to compete with full-text search engines. Its backbone is metadata; each piece of content is identified by a unique, "obvious" and short string. Tutorials, guides and other content that don't meet this requirement are outside the scope of the project.
83		84
@@ -91,10 +92,10 @@ Another driving factor is performance and the fact that everything happens in th · 7 unmodified lines
91		92
92	DevDocs being a developer tool, the browser requirements are high:	93	DevDocs being a developer tool, the browser requirements are high:
93		94
94	* Recent versions of Firefox, Chrome, or Opera	95	- Recent versions of Firefox, Chrome, or Opera
95	* Safari 11.1+	96	- Safari 11.1+
96	* Edge 17+	97	- Edge 17+