You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

validate_links.rb 1.9 KiB

пре 7 година
пре 7 година
пре 7 година
пре 7 година
пре 7 година
пре 7 година
пре 7 година
пре 7 година
пре 7 година
пре 7 година
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677
  1. #!/usr/bin/env ruby
  2. require 'httparty'
  3. require 'ruby-progressbar'
  4. require 'uri'
  5. allowed_codes = [200, 302, 403, 429]
  6. allowed_links = ["https://www.yelp.com/developers/documentation/v3"]
  7. args = ARGV
  8. filename = args[0]
  9. contents = File.open(filename, 'rb') { |f| f.read }
  10. raw_links = URI.extract(contents, ['http', 'https'])
  11. # Remove trailing ')' from entry URLs
  12. links = []
  13. raw_links.each do |link|
  14. if link.end_with?(')')
  15. links.push(link[0...-1])
  16. else
  17. links.push(link)
  18. end
  19. end
  20. if links.length <= 0
  21. puts "no links to check"
  22. exit(0)
  23. end
  24. fails = []
  25. # Fail on any duplicate elements
  26. dup = links.select{|element| links.count(element) > 1}
  27. if dup.uniq.length > 0
  28. dup.uniq.each do |e|
  29. fails.push("(DUP): #{e}")
  30. end
  31. end
  32. # Remove any duplicates from array
  33. links = links.uniq
  34. count = 0
  35. total = links.length
  36. progressbar = ProgressBar.create(:total => total,
  37. :format => "%a %P% | Processed: %c from %C")
  38. # GET each link and check for valid response code from allowed_codes
  39. links.each do |link|
  40. begin
  41. count += 1
  42. if allowed_links.include?(link)
  43. next
  44. end
  45. res = HTTParty.get(link, timeout: 10)
  46. if res.code.nil?
  47. fails.push("(NIL): #{link}")
  48. next
  49. end
  50. if !allowed_codes.include?(res.code)
  51. fails.push("(#{res.code}): #{link}")
  52. end
  53. rescue Net::ReadTimeout
  54. fails.push("(TMO): #{link}")
  55. rescue Net::OpenTimeout
  56. fails.push("(TMO): #{link}")
  57. rescue OpenSSL::SSL::SSLError
  58. fails.push("(SSL): #{link}")
  59. rescue SocketError
  60. fails.push("(SOK): #{link}")
  61. rescue Errno::ECONNREFUSED
  62. fails.push("(CON): #{link}")
  63. end
  64. progressbar.increment
  65. end
  66. puts "#{count}/#{total} links checked"
  67. if fails.length <= 0
  68. puts "all links valid"
  69. exit(0)
  70. else
  71. puts "-- RESULTS --"
  72. fails.sort!
  73. fails.each do |e|
  74. puts e
  75. end
  76. exit(1)
  77. end