Não pode escolher mais do que 25 tópicos Os tópicos devem começar com uma letra ou um número, podem incluir traços ('-') e podem ter até 35 caracteres.

validate_links.py 3.0 KiB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293
  1. #!/usr/bin/env python3
  2. import httplib2
  3. import re
  4. import socket
  5. import sys
  6. def parse_links(filename):
  7. """Returns a list of URLs from text file"""
  8. with open(filename, mode='r', encoding='utf-8') as fp:
  9. readme = fp.read()
  10. index_section = readme.find('## Index')
  11. content = readme[index_section:]
  12. raw_links = re.findall(
  13. '((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:\'\".,<>?«»“”‘’]))',
  14. content)
  15. links = [
  16. str(raw_link[0]).rstrip('/') for raw_link in raw_links
  17. ]
  18. return links
  19. def dup_links(links):
  20. """Check for duplicated links"""
  21. print(f'Checking for duplicated links...')
  22. hasError = False
  23. seen = {}
  24. dupes = []
  25. for link in links:
  26. if link not in seen:
  27. seen[link] = 1
  28. else:
  29. if seen[link] == 1:
  30. dupes.append(link)
  31. if not dupes:
  32. print(f"No duplicate links")
  33. else:
  34. print(f"Found duplicate links: {dupes}")
  35. hasError = True
  36. return hasError
  37. def validate_links(links):
  38. """Checks each entry in JSON file for live link"""
  39. print(f'Validating {len(links)} links...')
  40. hasError = False
  41. for link in links:
  42. h = httplib2.Http(disable_ssl_certificate_validation=True, timeout=25)
  43. try:
  44. resp = h.request(link, headers={
  45. # Faking user agent as some hosting services block not-whitelisted UA
  46. 'user-agent': 'Mozilla/5.0'
  47. })
  48. code = int(resp[0]['status'])
  49. # Checking status code errors
  50. if (code >= 300):
  51. hasError = True
  52. print(f"ERR:CLT:{code} : {link}")
  53. except TimeoutError:
  54. hasError = True
  55. print(f"ERR:TMO: {link}")
  56. except socket.error as socketerror:
  57. hasError = True
  58. print(f"ERR:SOC: {socketerror} : {link}")
  59. except Exception as e:
  60. hasError = True
  61. # Ignore some exceptions which are not actually errors.
  62. # The list below should be extended with other exceptions in the future if needed
  63. if (-1 != str(e).find("[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:852)")):
  64. print(f"ERR:SSL: {e} : {link}")
  65. elif (-1 != str(e).find("Content purported to be compressed with gzip but failed to decompress.")):
  66. print(f"ERR:GZP: {e} : {link}")
  67. elif (-1 != str(e).find("Unable to find the server at")):
  68. print(f"ERR:SRV: {e} : {link}")
  69. else:
  70. print(f"ERR:UKN: {e} : {link}")
  71. return hasError
  72. if __name__ == "__main__":
  73. num_args = len(sys.argv)
  74. if num_args < 2:
  75. print("No .md file passed")
  76. sys.exit(1)
  77. links = parse_links(sys.argv[1])
  78. hasError = dup_links(links)
  79. if not hasError:
  80. hasError = validate_links(links)
  81. if hasError:
  82. sys.exit(1)