History

Log errors instead of exiting when scraping docs

Thibaut Courouble committed 10 years ago

Commit 64eb1c81 parent e0a211e4 files · +21 −2Browse files

Unified · Split

4 files
  1. lib/docs.rb +3 −0
  2. lib/docs/core/scraper.rb +6 −2
  3. lib/docs/subscribers/doc_subscriber.rb +9 −0
  4. lib/tasks/docs.thor +3 −0
lib/docs.rb+3−0
@@ -25,6 +25,9 @@ module Docs · 24 unmodified lines
25 mattr_accessor :store_path25 mattr_accessor :store_path
26 self.store_path = File.expand_path '../public/docs', @@root_path26 self.store_path = File.expand_path '../public/docs', @@root_path
27 27
28 mattr_accessor :rescue_errors
29 self.rescue_errors = false
30
28 class DocNotFound < NameError; end31 class DocNotFound < NameError; end
29 32
30 def self.all33 def self.all
lib/docs/core/scraper.rb+6−2
@@ -163,8 +163,12 @@ module Docs · 162 unmodified lines
163 instrument 'ignore_response.scraper', response: response163 instrument 'ignore_response.scraper', response: response
164 end164 end
165 rescue => e165 rescue => e
166 puts "URL: #{response.url}"166 if Docs.rescue_errors
167 raise e167 instrument 'error.doc', exception: e, url: response.url
168 nil
169 else
170 raise e
171 end
168 end172 end
169 173
170 def process_response(response)174 def process_response(response)
lib/docs/subscribers/doc_subscriber.rb+9−0
@@ -20,6 +20,15 @@ module Docs · 19 unmodified lines
20 log event.payload[:msg]20 log event.payload[:msg]
21 end21 end
22 22
23 def error(event)
24 exception = event.payload[:exception]
25 log "ERROR:"
26 puts " #{event.payload[:url]}"
27 puts " #{exception.class}: #{exception.message.gsub("\n", "\n ")}"
28 puts exception.backtrace.select { |line| line.start_with?(Docs.root_path) }.join("\n ").prepend("\n ")
29 puts "\n"
30 end
31
23 private32 private
24 33
25 def parse_payload(event)34 def parse_payload(event)
lib/tasks/docs.thor+3−0
@@ -58,6 +58,7 @@ class DocsCLI < Thor · 57 unmodified lines
58 option :force, type: :boolean58 option :force, type: :boolean
59 option :package, type: :boolean59 option :package, type: :boolean
60 def generate(name)60 def generate(name)
61 Docs.rescue_errors = true
61 Docs.install_report :store if options[:verbose]62 Docs.install_report :store if options[:verbose]
62 Docs.install_report :scraper if options[:debug]63 Docs.install_report :scraper if options[:debug]
63 Docs.install_report :progress_bar, :doc if $stdout.tty?64 Docs.install_report :progress_bar, :doc if $stdout.tty?
@@ -93,6 +94,8 @@ class DocsCLI < Thor · 29 unmodified lines
93 generate_manifest if result94 generate_manifest if result
94 rescue Docs::DocNotFound => error95 rescue Docs::DocNotFound => error
95 handle_doc_not_found_error(error)96 handle_doc_not_found_error(error)
97 ensure
98 Docs.rescue_errors = false
96 end99 end
97 100
98 desc 'manifest', 'Create the manifest'101 desc 'manifest', 'Create the manifest'