You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

validate_links.rb 2.0 KiB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879
  1. #!/usr/bin/env ruby
  2. require 'httparty'
  3. require 'ruby-progressbar'
  4. require 'uri'
  5. allowed_codes = [200, 302, 403, 429]
  6. allowed_links = ["https://www.yelp.com/developers/documentation/v3"]
  7. args = ARGV
  8. filename = args[0]
  9. contents = File.open(filename, 'rb') { |f| f.read }
  10. raw_links = URI.extract(contents, ['http', 'https'])
  11. # Remove trailing ')' from entry URLs
  12. links = []
  13. raw_links.each do |link|
  14. if link.end_with?(')')
  15. links.push(link[0...-1])
  16. else
  17. links.push(link)
  18. end
  19. end
  20. if links.length <= 0
  21. puts "no links to check"
  22. exit(0)
  23. end
  24. fails = []
  25. # Fail on any duplicate elements
  26. dup = links.select{|element| links.count(element) > 1}
  27. if dup.uniq.length > 0
  28. dup.uniq.each do |e|
  29. fails.push("(DUP): #{e}")
  30. end
  31. end
  32. # Remove any duplicates from array
  33. links = links.uniq
  34. count = 0
  35. total = links.length
  36. progressbar = ProgressBar.create(:total => total,
  37. :format => "%a %P% | Processed: %c from %C")
  38. # GET each link and check for valid response code from allowed_codes
  39. links.each do |link|
  40. begin
  41. count += 1
  42. if allowed_links.include?(link)
  43. next
  44. end
  45. res = HTTParty.get(link, timeout: 10)
  46. if res.code.nil?
  47. fails.push("(NIL): #{link}")
  48. next
  49. end
  50. if !allowed_codes.include?(res.code)
  51. fails.push("(#{res.code}): #{link}")
  52. end
  53. rescue Net::ReadTimeout
  54. fails.push("(TMO): #{link}")
  55. rescue Net::OpenTimeout
  56. fails.push("(TMO): #{link}")
  57. rescue OpenSSL::SSL::SSLError
  58. fails.push("(SSL): #{link}")
  59. rescue SocketError
  60. fails.push("(SOK): #{link}")
  61. rescue Errno::ECONNREFUSED
  62. fails.push("(CON): #{link}")
  63. rescue Errno::ECONNRESET
  64. continue
  65. end
  66. progressbar.increment
  67. end
  68. puts "#{count}/#{total} links checked"
  69. if fails.length <= 0
  70. puts "all links valid"
  71. exit(0)
  72. else
  73. puts "-- RESULTS --"
  74. fails.sort!
  75. fails.each do |e|
  76. puts e
  77. end
  78. exit(1)
  79. end