History

Log errors instead of exiting when scraping docs

Thibaut Courouble committed 10 years ago

Commit 64eb1c81 parent e0a211e4 files · +21 −2Browse files

Unified · Split

4 files
  1. lib/docs.rb +3 −0
  2. lib/docs/core/scraper.rb +6 −2
  3. lib/docs/subscribers/doc_subscriber.rb +9 −0
  4. lib/tasks/docs.thor +3 −0
lib/docs.rb+3−0
@@ -25,6 +25,9 @@ module Docs · 24 unmodified lines
2525 mattr_accessor :store_path
2626 self.store_path = File.expand_path '../public/docs', @@root_path
2727
28+ mattr_accessor :rescue_errors
29+ self.rescue_errors = false
30+
2831 class DocNotFound < NameError; end
2932
3033 def self.all
lib/docs/core/scraper.rb+6−2
@@ -163,8 +163,12 @@ module Docs · 162 unmodified lines
163163 instrument 'ignore_response.scraper', response: response
164164 end
165165 rescue => e
166− puts "URL: #{response.url}"
167− raise e
166+ if Docs.rescue_errors
167+ instrument 'error.doc', exception: e, url: response.url
168+ nil
169+ else
170+ raise e
171+ end
168172 end
169173
170174 def process_response(response)
lib/docs/subscribers/doc_subscriber.rb+9−0
@@ -20,6 +20,15 @@ module Docs · 19 unmodified lines
2020 log event.payload[:msg]
2121 end
2222
23+ def error(event)
24+ exception = event.payload[:exception]
25+ log "ERROR:"
26+ puts " #{event.payload[:url]}"
27+ puts " #{exception.class}: #{exception.message.gsub("\n", "\n ")}"
28+ puts exception.backtrace.select { |line| line.start_with?(Docs.root_path) }.join("\n ").prepend("\n ")
29+ puts "\n"
30+ end
31+
2332 private
2433
2534 def parse_payload(event)
lib/tasks/docs.thor+3−0
@@ -58,6 +58,7 @@ class DocsCLI < Thor · 57 unmodified lines
5858 option :force, type: :boolean
5959 option :package, type: :boolean
6060 def generate(name)
61+ Docs.rescue_errors = true
6162 Docs.install_report :store if options[:verbose]
6263 Docs.install_report :scraper if options[:debug]
6364 Docs.install_report :progress_bar, :doc if $stdout.tty?
@@ -93,6 +94,8 @@ class DocsCLI < Thor · 29 unmodified lines
9394 generate_manifest if result
9495 rescue Docs::DocNotFound => error
9596 handle_doc_not_found_error(error)
97+ ensure
98+ Docs.rescue_errors = false
9699 end
97100
98101 desc 'manifest', 'Create the manifest'