diff --git a/build/main.sh b/build/main.sh index 41c04bdd..ade67324 100755 --- a/build/main.sh +++ b/build/main.sh @@ -1,45 +1,55 @@ #!/bin/bash FORMAT_FILE=../README.md -if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then - echo "running on $TRAVIS_BRANCH branch" - LINK_FILE=../README.md +echo "running format validation..." +./validate_format.py $FORMAT_FILE +if [[ $? != 0 ]]; then + echo "format validation failed!" + exit 1 +fi +echo "format validation passed!" +./build.sh +if [[ $? != 0 ]]; then + echo "JSON build failed!" else - echo "running on Pull Request #$TRAVIS_PULL_REQUEST" - DIFF_URL="https://patch-diff.githubusercontent.com/raw/toddmotto/public-apis/pull/$TRAVIS_PULL_REQUEST.diff" - curl $DIFF_URL > diff.txt - echo "------- BEGIN DIFF -------" - cat diff.txt - echo "-------- END DIFF --------" - cat diff.txt | egrep "\+" > additions.txt - echo "------ BEGIN ADDITIONS -----" - cat additions.txt - echo "------- END ADDITIONS ------" - LINK_FILE=additions.txt + echo "JSON build success!" +fi +if [ "$TRAVIS_BRANCH" == "master" ] +then + echo "Master build - deploying JSON" + ./deploy.sh +fi +if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then + echo "running on $TRAVIS_BRANCH branch - skipping Pull Request logic" + exit 0 +fi +echo "running on Pull Request #$TRAVIS_PULL_REQUEST" +DIFF_URL="https://patch-diff.githubusercontent.com/raw/toddmotto/public-apis/pull/$TRAVIS_PULL_REQUEST.diff" +curl $DIFF_URL > diff.txt +echo "------- BEGIN DIFF -------" +cat diff.txt +echo "-------- END DIFF --------" +cat diff.txt | egrep "\+" > additions.txt +echo "------ BEGIN ADDITIONS -----" +cat additions.txt +echo "------- END ADDITIONS ------" +LINK_FILE=additions.txt - echo "checking if /json was changed..." - if egrep "\+{3}\s.\/json\/" diff.txt > json.txt; then +echo "checking if /json was changed..." +if egrep "\+{3}\s.\/json\/" diff.txt > json.txt; then echo "JSON files are auto-generated! Please do not update these files:" cat json.txt exit 1 - else +else echo "/json check passed!" rm json.txt - fi - fi -echo "running format validation..." -./validate_format.py $FORMAT_FILE +echo "running link validation..." +./validate_links.py $LINK_FILE if [[ $? != 0 ]]; then - echo "format validation failed!" - exit 1 + echo "link validation failed!" + exit 1 else - echo "format validation passed!" - ./build.sh && ./deploy.sh - if [[ $? != 0 ]]; then - echo "JSON build and deploy failed!" - else - echo "JSON build and deploy success!" - fi + echo "link validation passed!" fi diff --git a/build/validate_links.py b/build/validate_links.py index 49bb4f88..15407ba4 100755 --- a/build/validate_links.py +++ b/build/validate_links.py @@ -1,32 +1,26 @@ #!/usr/bin/env python3 import httplib2 -import json +import re import socket import sys def parse_links(filename): - """Returns a list of links from JSON object""" - data = json.load(open(filename)) - links = [] - for entry in data['entries']: - link = entry['Link'] - https = True if link.startswith('https') else False - x = { - 'link': link, - 'https': https, - } - links.append(x) + """Returns a list of URLs from text file""" + with open(filename) as fp: + data = fp.read() + raw_links = re.findall( \ + 'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', \ + data) + links = [raw_link.replace(')', '') for raw_link in raw_links] return links - def validate_links(links): """Checks each entry in JSON file for live link""" print('Validating {} links...'.format(len(links))) errors = [] - for each in links: - link = each['link'] + for link in links: h = httplib2.Http(disable_ssl_certificate_validation=True, timeout=5) try: resp = h.request(link, 'HEAD') @@ -43,7 +37,7 @@ def validate_links(links): if __name__ == "__main__": num_args = len(sys.argv) if num_args < 2: - print("No .json file passed") + print("No .md file passed") sys.exit(1) errors = validate_links(parse_links(sys.argv[1])) if len(errors) > 0: