You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

82 line
2.0 KiB

  1. #!/usr/bin/env ruby
  2. require 'httparty'
  3. require 'ruby-progressbar'
  4. require 'uri'
  5. allowed_codes = [200, 302, 403, 429]
  6. allowed_links = ["https://www.yelp.com/developers/documentation/v3"]
  7. args = ARGV
  8. filename = args[0]
  9. contents = File.open(filename, 'rb') { |f| f.read }
  10. raw_links = URI.extract(contents, ['http', 'https'])
  11. # Remove trailing ')' from entry URLs
  12. links = []
  13. raw_links.each do |link|
  14. if link.end_with?(')')
  15. links.push(link[0...-1])
  16. else
  17. links.push(link)
  18. end
  19. end
  20. if links.length <= 0
  21. puts "no links to check"
  22. exit(0)
  23. end
  24. fails = []
  25. # Fail on any duplicate elements
  26. dup = links.select{|element| links.count(element) > 1}
  27. if dup.uniq.length > 0
  28. dup.uniq.each do |e|
  29. fails.push("(DUP): #{e}")
  30. end
  31. end
  32. # Remove any duplicates from array
  33. links = links.uniq
  34. count = 0
  35. total = links.length
  36. progressbar = ProgressBar.create(:total => total,
  37. :format => "%a %P% | Processed: %c from %C")
  38. # GET each link and check for valid response code from allowed_codes
  39. links.each do |link|
  40. begin
  41. count += 1
  42. if allowed_links.include?(link)
  43. next
  44. end
  45. res = HTTParty.get(link, timeout: 10)
  46. if res.code.nil?
  47. fails.push("(NIL): #{link}")
  48. next
  49. end
  50. if !allowed_codes.include?(res.code)
  51. fails.push("(#{res.code}): #{link}")
  52. end
  53. rescue HTTParty::RedirectionTooDeep
  54. fails.push("(RTD): #{link}")
  55. rescue Net::ReadTimeout
  56. fails.push("(TMO): #{link}")
  57. rescue Net::OpenTimeout
  58. fails.push("(TMO): #{link}")
  59. rescue OpenSSL::SSL::SSLError
  60. fails.push("(SSL): #{link}")
  61. rescue SocketError
  62. fails.push("(SOK): #{link}")
  63. rescue Errno::ECONNREFUSED
  64. fails.push("(CON): #{link}")
  65. rescue Errno::ECONNRESET
  66. next
  67. end
  68. progressbar.increment
  69. end
  70. puts "#{count}/#{total} links checked"
  71. if fails.length <= 0
  72. puts "all links valid"
  73. exit(0)
  74. else
  75. puts "-- RESULTS --"
  76. fails.sort!
  77. fails.each do |e|
  78. puts e
  79. end
  80. exit(1)
  81. end