Check links on PRs and Update main build script

build/main.sh

@@ -1,45 +1,55 @@
 #!/bin/bash
 
 FORMAT_FILE=../README.md
-if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
-  echo "running on $TRAVIS_BRANCH branch"
-  LINK_FILE=../README.md
+echo "running format validation..."
+./validate_format.py $FORMAT_FILE
+if [[ $? != 0 ]]; then
+    echo "format validation failed!"
+    exit 1
+fi
+echo "format validation passed!"
+./build.sh
+if [[ $? != 0 ]]; then
+    echo "JSON build failed!"
 else
-  echo "running on Pull Request #$TRAVIS_PULL_REQUEST"
-  DIFF_URL="https://patch-diff.githubusercontent.com/raw/toddmotto/public-apis/pull/$TRAVIS_PULL_REQUEST.diff"
-  curl $DIFF_URL > diff.txt
-  echo "------- BEGIN DIFF -------"
-  cat diff.txt
-  echo "-------- END DIFF --------"
-  cat diff.txt | egrep "\+" > additions.txt
-  echo "------ BEGIN ADDITIONS -----"
-  cat additions.txt
-  echo "------- END ADDITIONS ------"
-  LINK_FILE=additions.txt
+    echo "JSON build success!"
+fi
+if [ "$TRAVIS_BRANCH" == "master" ]
+then
+    echo "Master build - deploying JSON"
+    ./deploy.sh
+fi
+if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
+    echo "running on $TRAVIS_BRANCH branch - skipping Pull Request logic"
+    exit 0
+fi
+echo "running on Pull Request #$TRAVIS_PULL_REQUEST"
+DIFF_URL="https://patch-diff.githubusercontent.com/raw/toddmotto/public-apis/pull/$TRAVIS_PULL_REQUEST.diff"
+curl $DIFF_URL > diff.txt
+echo "------- BEGIN DIFF -------"
+cat diff.txt
+echo "-------- END DIFF --------"
+cat diff.txt | egrep "\+" > additions.txt
+echo "------ BEGIN ADDITIONS -----"
+cat additions.txt
+echo "------- END ADDITIONS ------"
+LINK_FILE=additions.txt
 
-  echo "checking if /json was changed..."
-  if egrep "\+{3}\s.\/json\/" diff.txt > json.txt; then
+echo "checking if /json was changed..."
+if egrep "\+{3}\s.\/json\/" diff.txt > json.txt; then
     echo "JSON files are auto-generated! Please do not update these files:"
     cat json.txt
     exit 1
-  else
+else
     echo "/json check passed!"
     rm json.txt
-  fi
-
 fi
 
-echo "running format validation..."
-./validate_format.py $FORMAT_FILE
+echo "running link validation..."
+./validate_links.py $LINK_FILE
 if [[ $? != 0 ]]; then
-  echo "format validation failed!"
-  exit 1
+    echo "link validation failed!"
+    exit 1
 else
-  echo "format validation passed!"
-  ./build.sh && ./deploy.sh
-  if [[ $? != 0 ]]; then
-    echo "JSON build and deploy failed!"
-  else
-    echo "JSON build and deploy success!"
-  fi
+    echo "link validation passed!"
 fi

build/validate_links.py

@@ -1,32 +1,26 @@
 #!/usr/bin/env python3
 
 import httplib2
-import json
+import re
 import socket
 import sys
 
 
 def parse_links(filename):
-    """Returns a list of links from JSON object"""
-    data = json.load(open(filename))
-    links = []
-    for entry in data['entries']:
-        link = entry['Link']
-        https = True if link.startswith('https') else False
-        x = {
-            'link': link,
-            'https': https,
-        }
-        links.append(x)
+    """Returns a list of URLs from text file"""
+    with open(filename) as fp:
+        data = fp.read()
+    raw_links = re.findall( \
+            'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', \
+            data)
+    links = [raw_link.replace(')', '') for raw_link in raw_links]
     return links
 
-
 def validate_links(links):
     """Checks each entry in JSON file for live link"""
     print('Validating {} links...'.format(len(links)))
     errors = []
-    for each in links:
-        link = each['link']
+    for link in links:
         h = httplib2.Http(disable_ssl_certificate_validation=True, timeout=5)
         try:
             resp = h.request(link, 'HEAD')
@@ -43,7 +37,7 @@ def validate_links(links):
 if __name__ == "__main__":
     num_args = len(sys.argv)
     if num_args < 2:
-        print("No .json file passed")
+        print("No .md file passed")
         sys.exit(1)
     errors = validate_links(parse_links(sys.argv[1]))
     if len(errors) > 0: