From f5123d246ad9ad7537fd002e2668157a56aa264a Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Tue, 11 Jan 2022 02:33:41 -0300 Subject: [PATCH 01/79] Create .gitignore file to Python projects --- .gitignore | 130 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..bdca81ea --- /dev/null +++ b/.gitignore @@ -0,0 +1,130 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST +.pypirc + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ \ No newline at end of file From f59cb8290404898b1c5296a4f9c8d91a121badef Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Tue, 11 Jan 2022 02:38:26 -0300 Subject: [PATCH 02/79] Create validate package --- scripts/validate/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 scripts/validate/__init__.py diff --git a/scripts/validate/__init__.py b/scripts/validate/__init__.py new file mode 100644 index 00000000..40a96afc --- /dev/null +++ b/scripts/validate/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- From e526f867d8b17c5f7e275afa4da0efd031b79e1c Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Tue, 11 Jan 2022 02:42:28 -0300 Subject: [PATCH 03/79] Create new requirements file --- scripts/requirements.txt | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 scripts/requirements.txt diff --git a/scripts/requirements.txt b/scripts/requirements.txt new file mode 100644 index 00000000..9b0cb453 --- /dev/null +++ b/scripts/requirements.txt @@ -0,0 +1,5 @@ +certifi==2021.10.8 +charset-normalizer==2.0.10 +idna==3.3 +requests==2.27.1 +urllib3==1.26.8 From 4808d633a170a7732a45016da47390a9d5e3638d Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Tue, 11 Jan 2022 03:59:18 -0300 Subject: [PATCH 04/79] Implement functions to find links in a text/file --- scripts/validate/links.py | 42 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 scripts/validate/links.py diff --git a/scripts/validate/links.py b/scripts/validate/links.py new file mode 100644 index 00000000..b7895097 --- /dev/null +++ b/scripts/validate/links.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- + +import sys +import re +from typing import List + + +def find_links_in_text(text: str) -> List[str]: + """Find links in a text and return a list of URLs.""" + + link_pattern = re.compile(r'((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:\'\".,<>?«»“”‘’]))') + + raw_links = re.findall(link_pattern, text) + + links = [ + str(raw_link[0]).rstrip('/') for raw_link in raw_links + ] + + return links + + +def find_links_in_file(filename: str) -> List[str]: + """Find links in a file and return a list of URLs from text file.""" + + with open(filename, mode='r', encoding='utf-8') as file: + readme = file.read() + index_section = readme.find('## Index') + content = readme[index_section:] + + links = find_links_in_text(content) + + return links + + +if __name__ == '__main__': + num_args = len(sys.argv) + + if num_args < 2: + print('No .md file passed') + sys.exit(1) + + links = find_links_in_file(sys.argv[1]) From 2eb6d2010097041178ce88a98353507c6eaadcbb Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Tue, 11 Jan 2022 04:33:18 -0300 Subject: [PATCH 05/79] Implement functions to check duplicate links --- scripts/validate/links.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/scripts/validate/links.py b/scripts/validate/links.py index b7895097..45bd174e 100644 --- a/scripts/validate/links.py +++ b/scripts/validate/links.py @@ -32,6 +32,31 @@ def find_links_in_file(filename: str) -> List[str]: return links +def check_duplicate_links(links: List[str]) -> bool: + """Check for duplicated links and return True or False.""" + + print('Checking for duplicated links...') + + seen = {} + duplicates = [] + has_duplicate = False + + for link in links: + if link not in seen: + seen[link] = 1 + else: + if seen[link] == 1: + duplicates.append(link) + + if not duplicates: + print(f'No duplicate links.') + else: + print(f'Found duplicate links: {duplicates}') + has_duplicate = True + + return has_duplicate + + if __name__ == '__main__': num_args = len(sys.argv) @@ -40,3 +65,5 @@ if __name__ == '__main__': sys.exit(1) links = find_links_in_file(sys.argv[1]) + + has_duplicate = check_duplicate_links(links) From d728eeddbce61e5b84d3f29efe4f4b9dc6fd6ce5 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Tue, 11 Jan 2022 20:58:05 -0300 Subject: [PATCH 06/79] Refactor duplicate link checker --- scripts/validate/links.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/scripts/validate/links.py b/scripts/validate/links.py index 45bd174e..bb84aac8 100644 --- a/scripts/validate/links.py +++ b/scripts/validate/links.py @@ -2,7 +2,7 @@ import sys import re -from typing import List +from typing import List, Tuple def find_links_in_text(text: str) -> List[str]: @@ -32,10 +32,11 @@ def find_links_in_file(filename: str) -> List[str]: return links -def check_duplicate_links(links: List[str]) -> bool: - """Check for duplicated links and return True or False.""" +def check_duplicate_links(links: List[str]) -> Tuple[bool, List]: + """Check for duplicated links. - print('Checking for duplicated links...') + Returns a tuple with True or False and duplicate list. + """ seen = {} duplicates = [] @@ -48,13 +49,10 @@ def check_duplicate_links(links: List[str]) -> bool: if seen[link] == 1: duplicates.append(link) - if not duplicates: - print(f'No duplicate links.') - else: - print(f'Found duplicate links: {duplicates}') + if duplicates: has_duplicate = True - - return has_duplicate + + return (has_duplicate, duplicates) if __name__ == '__main__': @@ -66,4 +64,11 @@ if __name__ == '__main__': links = find_links_in_file(sys.argv[1]) - has_duplicate = check_duplicate_links(links) + print('Checking for duplicate links...') + + has_duplicate_link, duplicates_links = check_duplicate_links(links) + + if has_duplicate_link: + print(f'Found duplicate links: {duplicates_links}') + else: + print('No duplicate links.') From d86c263f4523ee38e01e4fba4847d60e409cbece Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Tue, 11 Jan 2022 21:09:43 -0300 Subject: [PATCH 07/79] Create test structure --- scripts/tests/__init__.py | 1 + scripts/tests/test_validate_links.py | 1 + 2 files changed, 2 insertions(+) create mode 100644 scripts/tests/__init__.py create mode 100644 scripts/tests/test_validate_links.py diff --git a/scripts/tests/__init__.py b/scripts/tests/__init__.py new file mode 100644 index 00000000..40a96afc --- /dev/null +++ b/scripts/tests/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/scripts/tests/test_validate_links.py b/scripts/tests/test_validate_links.py new file mode 100644 index 00000000..40a96afc --- /dev/null +++ b/scripts/tests/test_validate_links.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- From 80ba7262faac234387e0c537caa8838420050c8f Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Tue, 11 Jan 2022 21:13:46 -0300 Subject: [PATCH 08/79] Create TestValidateLinks class --- scripts/tests/test_validate_links.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/tests/test_validate_links.py b/scripts/tests/test_validate_links.py index 40a96afc..031a8658 100644 --- a/scripts/tests/test_validate_links.py +++ b/scripts/tests/test_validate_links.py @@ -1 +1,7 @@ # -*- coding: utf-8 -*- + +import unittest + + +class TestValidateLinks(unittest.TestCase): + ... From bf4dfcd39dcec6d2fbe964bc837390b19ce25b09 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Tue, 11 Jan 2022 22:16:26 -0300 Subject: [PATCH 09/79] Create basic test to find_link_in_text function --- scripts/tests/test_validate_links.py | 41 +++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/scripts/tests/test_validate_links.py b/scripts/tests/test_validate_links.py index 031a8658..f9dea3e0 100644 --- a/scripts/tests/test_validate_links.py +++ b/scripts/tests/test_validate_links.py @@ -2,6 +2,45 @@ import unittest +from validate.links import find_links_in_text + class TestValidateLinks(unittest.TestCase): - ... + + def setUp(self): + self.text = """ + # this is valid + + http://example.com?param1=1¶m2=2#anchor + https://www.example.com?param1=1¶m2=2#anchor + https://www.example.com.br + https://www.example.com.gov.br + [Example](https://www.example.com?param1=1¶m2=2#anchor) + lorem ipsum https://www.example.com?param1=1¶m2=2#anchor + https://www.example.com?param1=1¶m2=2#anchor lorem ipsum + + # this not is valid + + example.com + https:example.com + https:/example.com + https//example.com + https//.com + """ + + + def test_find_link_in_text(self): + links = find_links_in_text(self.text) + + self.assertIsInstance(links, list) + self.assertEqual(len(links), 7) + + for link in links: + with self.subTest(): + self.assertIsInstance(link, str) + + def test_find_link_in_text_with_invalid_argument(self): + with self.assertRaises(TypeError): + find_links_in_text() + find_links_in_text(1) + find_links_in_text(True) From 3cc0f49b29c26b42e0c91f20740ec404eaf290f0 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Tue, 11 Jan 2022 22:19:54 -0300 Subject: [PATCH 10/79] Remove setUp method --- scripts/tests/test_validate_links.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/scripts/tests/test_validate_links.py b/scripts/tests/test_validate_links.py index f9dea3e0..97833f09 100644 --- a/scripts/tests/test_validate_links.py +++ b/scripts/tests/test_validate_links.py @@ -5,10 +5,10 @@ import unittest from validate.links import find_links_in_text -class TestValidateLinks(unittest.TestCase): +class TestValidateLinks(unittest.TestCase): - def setUp(self): - self.text = """ + def test_find_link_in_text(self): + text = """ # this is valid http://example.com?param1=1¶m2=2#anchor @@ -27,10 +27,8 @@ class TestValidateLinks(unittest.TestCase): https//example.com https//.com """ - - def test_find_link_in_text(self): - links = find_links_in_text(self.text) + links = find_links_in_text(text) self.assertIsInstance(links, list) self.assertEqual(len(links), 7) From d06a3717d487b3a24a1a268b97c5e881ff9bdf28 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Tue, 11 Jan 2022 22:32:43 -0300 Subject: [PATCH 11/79] Remove whitespace --- scripts/tests/test_validate_links.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/test_validate_links.py b/scripts/tests/test_validate_links.py index 97833f09..c655f4af 100644 --- a/scripts/tests/test_validate_links.py +++ b/scripts/tests/test_validate_links.py @@ -5,7 +5,7 @@ import unittest from validate.links import find_links_in_text -class TestValidateLinks(unittest.TestCase): +class TestValidateLinks(unittest.TestCase): def test_find_link_in_text(self): text = """ From 7be0512b5438edf4d434f325181e2e586ee65009 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Wed, 12 Jan 2022 00:55:23 -0300 Subject: [PATCH 12/79] Check if a link is working --- scripts/validate/links.py | 79 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/scripts/validate/links.py b/scripts/validate/links.py index bb84aac8..2dfbf7db 100644 --- a/scripts/validate/links.py +++ b/scripts/validate/links.py @@ -1,9 +1,12 @@ # -*- coding: utf-8 -*- -import sys import re +import sys +import random from typing import List, Tuple +import requests + def find_links_in_text(text: str) -> List[str]: """Find links in a text and return a list of URLs.""" @@ -55,6 +58,80 @@ def check_duplicate_links(links: List[str]) -> Tuple[bool, List]: return (has_duplicate, duplicates) +def fake_user_agent() -> str: + """Faking user agent as some hosting services block not-whitelisted UA.""" + + user_agents = [ + 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1467.0 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko)', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36', + ] + + return random.choice(user_agents) + + +def get_host_from_link(link: str) -> str: + + host = link.split('://', 1)[1] + + # Remove routes, arguments and anchors + if '/' in host: + host = host.split('/', 1)[0] + + elif '?' in host: + host = host.split('?', 1)[0] + + elif '#' in host: + host = host.split('#', 1)[0] + + return host + + +def check_if_link_is_working(link: str) -> Tuple[bool, str]: + """Checks if a link is working. + + If an error is identified when the request for the link occurs, + the return will be a tuple with the first value True and the second + value a string containing the error message. + + If no errors are identified, the return will be a tuple with the + first value False and the second an empty string. + """ + + has_error = False + error_message = '' + + try: + resp = requests.get(link + '/', timeout=25, headers={ + 'User-Agent': fake_user_agent(), + 'host': get_host_from_link(link) + }) + + code = resp.status_code + if code >= 400: + has_error = True + error_message = f'ERR:CLT: {code} : {link}' + + except (TimeoutError, requests.exceptions.ConnectTimeout): + has_error = True + error_message = f'ERR:TMO: {link}' + + except requests.exceptions.SSLError as error: + has_error = True + error_message = f'ERR:SSL: {error} : {link}' + + except requests.exceptions.TooManyRedirects as error: + has_error = True + error_message = f'ERR:TMR: {error} : {link}' + + except Exception as error: + has_error = True + error_message = f'ERR:UKN: {error} : {link}' + + return (has_error, error_message) + + if __name__ == '__main__': num_args = len(sys.argv) From 95ee79818f70180085e6b97256ccb9c23d2db28c Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Wed, 12 Jan 2022 00:58:54 -0300 Subject: [PATCH 13/79] Remove whitespaces --- scripts/validate/links.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/validate/links.py b/scripts/validate/links.py index 2dfbf7db..ad73e608 100644 --- a/scripts/validate/links.py +++ b/scripts/validate/links.py @@ -90,7 +90,7 @@ def get_host_from_link(link: str) -> str: def check_if_link_is_working(link: str) -> Tuple[bool, str]: """Checks if a link is working. - + If an error is identified when the request for the link occurs, the return will be a tuple with the first value True and the second value a string containing the error message. @@ -116,11 +116,11 @@ def check_if_link_is_working(link: str) -> Tuple[bool, str]: except (TimeoutError, requests.exceptions.ConnectTimeout): has_error = True error_message = f'ERR:TMO: {link}' - + except requests.exceptions.SSLError as error: has_error = True error_message = f'ERR:SSL: {error} : {link}' - + except requests.exceptions.TooManyRedirects as error: has_error = True error_message = f'ERR:TMR: {error} : {link}' @@ -128,7 +128,7 @@ def check_if_link_is_working(link: str) -> Tuple[bool, str]: except Exception as error: has_error = True error_message = f'ERR:UKN: {error} : {link}' - + return (has_error, error_message) From 8650f7f979c769f5dc0e906e1f4d64da580783c7 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Wed, 12 Jan 2022 01:23:59 -0300 Subject: [PATCH 14/79] Make sure the link contains :// --- scripts/validate/links.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/validate/links.py b/scripts/validate/links.py index ad73e608..5de8b334 100644 --- a/scripts/validate/links.py +++ b/scripts/validate/links.py @@ -73,7 +73,7 @@ def fake_user_agent() -> str: def get_host_from_link(link: str) -> str: - host = link.split('://', 1)[1] + host = link.split('://', 1)[1] if '://' in link else link # Remove routes, arguments and anchors if '/' in host: From ef8c4f04e56ac00c5bcf881b63d09313dc3a2214 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Wed, 12 Jan 2022 01:26:39 -0300 Subject: [PATCH 15/79] Create basic test to get host from link --- scripts/tests/test_validate_links.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/scripts/tests/test_validate_links.py b/scripts/tests/test_validate_links.py index c655f4af..c5a3fd4b 100644 --- a/scripts/tests/test_validate_links.py +++ b/scripts/tests/test_validate_links.py @@ -3,6 +3,7 @@ import unittest from validate.links import find_links_in_text +from validate.links import get_host_from_link class TestValidateLinks(unittest.TestCase): @@ -42,3 +43,25 @@ class TestValidateLinks(unittest.TestCase): find_links_in_text() find_links_in_text(1) find_links_in_text(True) + + def test_get_host_from_link(self): + links = [ + 'example.com', + 'https://example.com', + 'https://www.example.com', + 'https://www.example.com.br', + 'https://www.example.com/route', + 'https://www.example.com?p=1&q=2', + 'https://www.example.com#anchor' + ] + + for link in links: + host = get_host_from_link(link) + + with self.subTest(): + self.assertIsInstance(host, str) + + self.assertNotIn('://', host) + self.assertNotIn('/', host) + self.assertNotIn('?', host) + self.assertNotIn('#', host) From a707c4b89dc8229a8200becc7329b0352c893c67 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Wed, 12 Jan 2022 01:30:08 -0300 Subject: [PATCH 16/79] Test get_host_from_link with invalid argument --- scripts/tests/test_validate_links.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/tests/test_validate_links.py b/scripts/tests/test_validate_links.py index c5a3fd4b..a559f712 100644 --- a/scripts/tests/test_validate_links.py +++ b/scripts/tests/test_validate_links.py @@ -65,3 +65,6 @@ class TestValidateLinks(unittest.TestCase): self.assertNotIn('/', host) self.assertNotIn('?', host) self.assertNotIn('#', host) + + with self.assertRaises(TypeError): + get_host_from_link() From 978147ca3963896dbb2da0811075f360eef81f53 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Wed, 12 Jan 2022 02:15:39 -0300 Subject: [PATCH 17/79] Check if a list of links are working --- scripts/validate/links.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/scripts/validate/links.py b/scripts/validate/links.py index 5de8b334..5cfb6e8d 100644 --- a/scripts/validate/links.py +++ b/scripts/validate/links.py @@ -132,6 +132,17 @@ def check_if_link_is_working(link: str) -> Tuple[bool, str]: return (has_error, error_message) +def check_if_list_of_links_are_working(list_of_links: List[str]) -> List[str]: + error_messages = [] + for link in list_of_links: + has_error, error_message = check_if_link_is_working(link) + + if has_error: + error_messages.append(error_message) + + return error_messages + + if __name__ == '__main__': num_args = len(sys.argv) @@ -147,5 +158,14 @@ if __name__ == '__main__': if has_duplicate_link: print(f'Found duplicate links: {duplicates_links}') + sys.exit(1) else: print('No duplicate links.') + + print(f'Checking if {len(links)} links are working...') + + errors = check_if_list_of_links_are_working(links) + if errors: + for error_message in errors: + print(error_message) + sys.exit(1) From 8136aa9ed1c97aae2cc883934c7d2dc011bb746b Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Wed, 12 Jan 2022 19:20:31 -0300 Subject: [PATCH 18/79] Implementing the cloudflare protection check Related: - #2409 - #2960 --- scripts/validate/links.py | 61 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/scripts/validate/links.py b/scripts/validate/links.py index 5cfb6e8d..fde7bad3 100644 --- a/scripts/validate/links.py +++ b/scripts/validate/links.py @@ -6,6 +6,7 @@ import random from typing import List, Tuple import requests +from requests.models import Response def find_links_in_text(text: str) -> List[str]: @@ -88,6 +89,63 @@ def get_host_from_link(link: str) -> str: return host +def has_cloudflare_protection(resp: Response) -> bool: + """Checks if there is any cloudflare protection in the response. + + Cloudflare implements multiple network protections on a given link, + this script tries to detect if any of them exist in the response from request. + + Common protections have the following HTTP code as a response: + - 403: When host header is missing or incorrect (and more) + - 503: When DDOS protection exists + + See more about it at: + - https://support.cloudflare.com/hc/en-us/articles/115003014512-4xx-Client-Error + - https://support.cloudflare.com/hc/en-us/articles/115003011431-Troubleshooting-Cloudflare-5XX-errors + - https://www.cloudflare.com/ddos/ + - https://superuser.com/a/888526 + + Discussions in issues and pull requests: + - https://github.com/public-apis/public-apis/pull/2409 + - https://github.com/public-apis/public-apis/issues/2960 + """ + + code = resp.status_code + server = resp.headers.get('Server') or resp.headers.get('server') + cloudflare_flags = [ + '403 Forbidden', + 'cloudflare', + 'Cloudflare', + 'Security check', + 'Please Wait... | Cloudflare', + 'We are checking your browser...', + 'Please stand by, while we are checking your browser...', + 'Checking your browser before accessing', + 'This process is automatic.', + 'Your browser will redirect to your requested content shortly.', + 'Please allow up to 5 seconds', + 'DDoS protection by', + 'Ray ID:', + 'Cloudflare Ray ID:', + '_cf_chl', + '_cf_chl_opt', + '__cf_chl_rt_tk', + 'cf-spinner-please-wait', + 'cf-spinner-redirecting' + ] + + if code in [403, 503] and server == 'cloudflare': + html = resp.text + + flags_found = [flag in html for flag in cloudflare_flags] + any_flag_found = any(flags_found) + + if any_flag_found: + return True + + return False + + def check_if_link_is_working(link: str) -> Tuple[bool, str]: """Checks if a link is working. @@ -109,7 +167,8 @@ def check_if_link_is_working(link: str) -> Tuple[bool, str]: }) code = resp.status_code - if code >= 400: + + if code >= 400 and not has_cloudflare_protection(resp): has_error = True error_message = f'ERR:CLT: {code} : {link}' From 8ac60e31aaf5520c93f8843b7c485039afc0ac5c Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Wed, 12 Jan 2022 19:25:08 -0300 Subject: [PATCH 19/79] Remove whitespaces --- scripts/validate/links.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/validate/links.py b/scripts/validate/links.py index fde7bad3..a64cdde3 100644 --- a/scripts/validate/links.py +++ b/scripts/validate/links.py @@ -91,14 +91,14 @@ def get_host_from_link(link: str) -> str: def has_cloudflare_protection(resp: Response) -> bool: """Checks if there is any cloudflare protection in the response. - + Cloudflare implements multiple network protections on a given link, this script tries to detect if any of them exist in the response from request. Common protections have the following HTTP code as a response: - 403: When host header is missing or incorrect (and more) - 503: When DDOS protection exists - + See more about it at: - https://support.cloudflare.com/hc/en-us/articles/115003014512-4xx-Client-Error - https://support.cloudflare.com/hc/en-us/articles/115003011431-Troubleshooting-Cloudflare-5XX-errors From 4edd0d5f54a18cff6780e16dee1946574f98be4f Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Wed, 12 Jan 2022 20:00:56 -0300 Subject: [PATCH 20/79] Create basic tests to has_cloudflare_protection --- scripts/tests/test_validate_links.py | 65 ++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/scripts/tests/test_validate_links.py b/scripts/tests/test_validate_links.py index a559f712..73103661 100644 --- a/scripts/tests/test_validate_links.py +++ b/scripts/tests/test_validate_links.py @@ -4,10 +4,29 @@ import unittest from validate.links import find_links_in_text from validate.links import get_host_from_link +from validate.links import has_cloudflare_protection + + +class FakeResponse(): + def __init__(self, code: int, headers: dict, text: str) -> None: + self.status_code = code + self.headers = headers + self.text = text class TestValidateLinks(unittest.TestCase): + def setUp(self): + self.code_200 = 200 + self.code_403 = 403 + self.code_503 = 503 + + self.cloudflare_headers = {'Server': 'cloudflare'} + self.no_cloudflare_headers = {'Server': 'google'} + + self.text_with_cloudflare_flags = '403 Forbidden Cloudflare We are checking your browser...' + self.text_without_cloudflare_flags = 'Lorem Ipsum' + def test_find_link_in_text(self): text = """ # this is valid @@ -68,3 +87,49 @@ class TestValidateLinks(unittest.TestCase): with self.assertRaises(TypeError): get_host_from_link() + + def test_has_cloudflare_protection_with_code_403_and_503_in_response(self): + resp_with_cloudflare_protection_code_403 = FakeResponse( + code=self.code_403, + headers=self.cloudflare_headers, + text=self.text_with_cloudflare_flags + ) + + resp_with_cloudflare_protection_code_503 = FakeResponse( + code=self.code_503, + headers=self.cloudflare_headers, + text=self.text_with_cloudflare_flags + ) + + result1 = has_cloudflare_protection(resp_with_cloudflare_protection_code_403) + result2 = has_cloudflare_protection(resp_with_cloudflare_protection_code_503) + + self.assertTrue(result1) + self.assertTrue(result2) + + def test_has_cloudflare_protection_when_there_is_no_protection(self): + resp_without_cloudflare_protection1 = FakeResponse( + code=self.code_200, + headers=self.no_cloudflare_headers, + text=self.text_without_cloudflare_flags + ) + + resp_without_cloudflare_protection2 = FakeResponse( + code=self.code_403, + headers=self.no_cloudflare_headers, + text=self.text_without_cloudflare_flags + ) + + resp_without_cloudflare_protection3 = FakeResponse( + code=self.code_503, + headers=self.no_cloudflare_headers, + text=self.text_without_cloudflare_flags + ) + + result1 = has_cloudflare_protection(resp_without_cloudflare_protection1) + result2 = has_cloudflare_protection(resp_without_cloudflare_protection2) + result3 = has_cloudflare_protection(resp_without_cloudflare_protection3) + + self.assertFalse(result1) + self.assertFalse(result2) + self.assertFalse(result3) From 50ad807e5c9ef953ea88b496f67e8fc4aff249ca Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Thu, 13 Jan 2022 16:23:01 -0300 Subject: [PATCH 21/79] Show number of links with a possible error --- scripts/validate/links.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/validate/links.py b/scripts/validate/links.py index a64cdde3..b0adeca8 100644 --- a/scripts/validate/links.py +++ b/scripts/validate/links.py @@ -225,6 +225,11 @@ if __name__ == '__main__': errors = check_if_list_of_links_are_working(links) if errors: + + num_errors = len(errors) + print(f'Apparently {num_errors} links are not working properly. See in:') + for error_message in errors: print(error_message) + sys.exit(1) From b62683f96eac5c08f6b54ec437240c64c4122c16 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Thu, 13 Jan 2022 16:33:03 -0300 Subject: [PATCH 22/79] Show one duplicate link per line --- scripts/validate/links.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/validate/links.py b/scripts/validate/links.py index b0adeca8..efa918ad 100644 --- a/scripts/validate/links.py +++ b/scripts/validate/links.py @@ -216,7 +216,10 @@ if __name__ == '__main__': has_duplicate_link, duplicates_links = check_duplicate_links(links) if has_duplicate_link: - print(f'Found duplicate links: {duplicates_links}') + print(f'Found duplicate links:') + for duplicate_link in duplicates_links: + print(duplicate_link) + sys.exit(1) else: print('No duplicate links.') From 1866c0d83a1e6fdbaa3a8dd73d9718bb5ef9df0b Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Thu, 13 Jan 2022 16:45:14 -0300 Subject: [PATCH 23/79] Create basic test to fake_user_agent --- scripts/tests/test_validate_links.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/tests/test_validate_links.py b/scripts/tests/test_validate_links.py index 73103661..a77950cb 100644 --- a/scripts/tests/test_validate_links.py +++ b/scripts/tests/test_validate_links.py @@ -3,6 +3,7 @@ import unittest from validate.links import find_links_in_text +from validate.links import fake_user_agent from validate.links import get_host_from_link from validate.links import has_cloudflare_protection @@ -63,6 +64,10 @@ class TestValidateLinks(unittest.TestCase): find_links_in_text(1) find_links_in_text(True) + def test_if_fake_user_agent_has_a_str_as_return(self): + user_agent = fake_user_agent() + self.assertIsInstance(user_agent, str) + def test_get_host_from_link(self): links = [ 'example.com', From 8d590690e83ee71bde898cfb5c936d710523b694 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Thu, 13 Jan 2022 17:14:13 -0300 Subject: [PATCH 24/79] Create basic test to check_duplicate_links --- scripts/tests/test_validate_links.py | 34 +++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/scripts/tests/test_validate_links.py b/scripts/tests/test_validate_links.py index a77950cb..64eae888 100644 --- a/scripts/tests/test_validate_links.py +++ b/scripts/tests/test_validate_links.py @@ -3,6 +3,7 @@ import unittest from validate.links import find_links_in_text +from validate.links import check_duplicate_links from validate.links import fake_user_agent from validate.links import get_host_from_link from validate.links import has_cloudflare_protection @@ -18,6 +19,18 @@ class FakeResponse(): class TestValidateLinks(unittest.TestCase): def setUp(self): + self.duplicate_links = [ + 'https://www.example.com', + 'https://www.example.com', + 'https://www.example.com', + 'https://www.anotherexample.com', + ] + self.no_duplicate_links = [ + 'https://www.firstexample.com', + 'https://www.secondexample.com', + 'https://www.anotherexample.com', + ] + self.code_200 = 200 self.code_403 = 403 self.code_503 = 503 @@ -63,7 +76,26 @@ class TestValidateLinks(unittest.TestCase): find_links_in_text() find_links_in_text(1) find_links_in_text(True) - + + def test_if_check_duplicate_links_has_the_correct_return(self): + result_1 = check_duplicate_links(self.duplicate_links) + result_2 = check_duplicate_links(self.no_duplicate_links) + + self.assertIsInstance(result_1, tuple) + self.assertIsInstance(result_2, tuple) + + has_duplicate_links, links = result_1 + no_duplicate_links, no_links = result_2 + + self.assertTrue(has_duplicate_links) + self.assertFalse(no_duplicate_links) + + self.assertIsInstance(links, list) + self.assertIsInstance(no_links, list) + + self.assertEqual(len(links), 2) + self.assertEqual(len(no_links), 0) + def test_if_fake_user_agent_has_a_str_as_return(self): user_agent = fake_user_agent() self.assertIsInstance(user_agent, str) From 8c28be1497a3f64722b7627109b123767afef21f Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Thu, 13 Jan 2022 17:49:11 -0300 Subject: [PATCH 25/79] Create start_* functions to start checkers --- scripts/validate/links.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/scripts/validate/links.py b/scripts/validate/links.py index efa918ad..d6d47b00 100644 --- a/scripts/validate/links.py +++ b/scripts/validate/links.py @@ -202,14 +202,7 @@ def check_if_list_of_links_are_working(list_of_links: List[str]) -> List[str]: return error_messages -if __name__ == '__main__': - num_args = len(sys.argv) - - if num_args < 2: - print('No .md file passed') - sys.exit(1) - - links = find_links_in_file(sys.argv[1]) +def start_duplicate_links_checker(links: List[str]) -> None: print('Checking for duplicate links...') @@ -217,6 +210,7 @@ if __name__ == '__main__': if has_duplicate_link: print(f'Found duplicate links:') + for duplicate_link in duplicates_links: print(duplicate_link) @@ -224,6 +218,9 @@ if __name__ == '__main__': else: print('No duplicate links.') + +def start_links_working_checker(links: List[str]) -> None: + print(f'Checking if {len(links)} links are working...') errors = check_if_list_of_links_are_working(links) @@ -236,3 +233,23 @@ if __name__ == '__main__': print(error_message) sys.exit(1) + + +def main(filename: str) -> None: + + links = find_links_in_file(filename) + + start_duplicate_links_checker(links) + start_links_working_checker(links) + + +if __name__ == '__main__': + num_args = len(sys.argv) + + if num_args < 2: + print('No .md file passed') + sys.exit(1) + + filename = sys.argv[1] + + main(filename) From a553dd5909199a70ee8b7d41f5136ef56dfabd38 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Sat, 15 Jan 2022 17:12:44 -0300 Subject: [PATCH 26/79] Mapped RequestException Exceptions --- scripts/validate/links.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/validate/links.py b/scripts/validate/links.py index d6d47b00..5d57416f 100644 --- a/scripts/validate/links.py +++ b/scripts/validate/links.py @@ -184,7 +184,7 @@ def check_if_link_is_working(link: str) -> Tuple[bool, str]: has_error = True error_message = f'ERR:TMR: {error} : {link}' - except Exception as error: + except (Exception, requests.exceptions.RequestException) as error: has_error = True error_message = f'ERR:UKN: {error} : {link}' From ae7ac0058690872469b5c450e811b7303e180f18 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Sat, 15 Jan 2022 17:40:32 -0300 Subject: [PATCH 27/79] Mapped network problem (DNS failure, connection refused, etc) --- scripts/validate/links.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/validate/links.py b/scripts/validate/links.py index 5d57416f..90038684 100644 --- a/scripts/validate/links.py +++ b/scripts/validate/links.py @@ -172,14 +172,18 @@ def check_if_link_is_working(link: str) -> Tuple[bool, str]: has_error = True error_message = f'ERR:CLT: {code} : {link}' - except (TimeoutError, requests.exceptions.ConnectTimeout): - has_error = True - error_message = f'ERR:TMO: {link}' - except requests.exceptions.SSLError as error: has_error = True error_message = f'ERR:SSL: {error} : {link}' + except requests.exceptions.ConnectionError as error: + has_error = True + error_message = f'ERR:CNT: {error} : {link}' + + except (TimeoutError, requests.exceptions.ConnectTimeout): + has_error = True + error_message = f'ERR:TMO: {link}' + except requests.exceptions.TooManyRedirects as error: has_error = True error_message = f'ERR:TMR: {error} : {link}' From f3146c9e246c404395d39e0fad872ded04d8a684 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Sat, 15 Jan 2022 19:24:35 -0300 Subject: [PATCH 28/79] Get content from index 0 if Index section does not exist --- scripts/validate/links.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/validate/links.py b/scripts/validate/links.py index 90038684..e727f03d 100644 --- a/scripts/validate/links.py +++ b/scripts/validate/links.py @@ -29,6 +29,8 @@ def find_links_in_file(filename: str) -> List[str]: with open(filename, mode='r', encoding='utf-8') as file: readme = file.read() index_section = readme.find('## Index') + if index_section == -1: + index_section = 0 content = readme[index_section:] links = find_links_in_text(content) From 1360d618cf6bee302925a95f3e75f54b369456ee Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Sat, 15 Jan 2022 19:29:13 -0300 Subject: [PATCH 29/79] Create format.py file --- scripts/validate/format.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 scripts/validate/format.py diff --git a/scripts/validate/format.py b/scripts/validate/format.py new file mode 100644 index 00000000..40a96afc --- /dev/null +++ b/scripts/validate/format.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- From c659e65677c1737c72093d0f7cc9e374ae353293 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Sat, 15 Jan 2022 20:35:02 -0300 Subject: [PATCH 30/79] Set constants --- scripts/validate/format.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/scripts/validate/format.py b/scripts/validate/format.py index 40a96afc..b3e06d53 100644 --- a/scripts/validate/format.py +++ b/scripts/validate/format.py @@ -1 +1,25 @@ # -*- coding: utf-8 -*- + +import re + + +anchor = '###' +min_entries_per_section = 3 +auth_keys = ['apiKey', 'OAuth', 'X-Mashape-Key', 'User-Agent', 'No'] +punctuation = ['.', '?', '!'] +https_keys = ['Yes', 'No'] +cors_keys = ['Yes', 'No', 'Unknown'] + +index_title = 0 +index_desc = 1 +index_auth = 2 +index_https = 3 +index_cors = 4 +index_link = 5 +num_segments = 5 + +errors = [] +title_links = [] +anchor_re = re.compile(anchor + '\s(.+)') +section_title_re = re.compile('\*\s\[(.*)\]') +link_re = re.compile('\[(.+)\]\((http.*)\)') From 2501df6b93171ab28e73ef5e9fc003fc8db0797d Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Sun, 16 Jan 2022 01:08:49 -0300 Subject: [PATCH 31/79] Create a error message generator --- scripts/validate/format.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/validate/format.py b/scripts/validate/format.py index b3e06d53..0cc66fc9 100644 --- a/scripts/validate/format.py +++ b/scripts/validate/format.py @@ -23,3 +23,8 @@ title_links = [] anchor_re = re.compile(anchor + '\s(.+)') section_title_re = re.compile('\*\s\[(.*)\]') link_re = re.compile('\[(.+)\]\((http.*)\)') + + +def error_message(line_number: int, message: str) -> str: + line = line_number + 1 + return f'(L{line:03d}) {message}' From 6bfe284191bc08f21358d7f23a0d4e4b34f98df2 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Sun, 16 Jan 2022 02:53:25 -0300 Subject: [PATCH 32/79] Get categories content and check alphabetical order --- scripts/validate/format.py | 47 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/scripts/validate/format.py b/scripts/validate/format.py index 0cc66fc9..1f0c96aa 100644 --- a/scripts/validate/format.py +++ b/scripts/validate/format.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import re +from typing import List, Tuple, Dict anchor = '###' @@ -24,7 +25,53 @@ anchor_re = re.compile(anchor + '\s(.+)') section_title_re = re.compile('\*\s\[(.*)\]') link_re = re.compile('\[(.+)\]\((http.*)\)') +# Type aliases +APIList = List[str] +Categories = Dict[str, APIList] +CategoriesLineNumber = Dict[str, int] + def error_message(line_number: int, message: str) -> str: line = line_number + 1 return f'(L{line:03d}) {message}' + + +def get_categories_content(contents: List[str]) -> Tuple[Categories, CategoriesLineNumber]: + + categories = {} + category_line_num = {} + + for line_num, line_content in enumerate(contents): + + if line_content.startswith(anchor): + category = line_content.split(anchor)[1].strip() + categories[category] = [] + category_line_num[category] = line_num + continue + + if not line_content.startswith('|') or line_content.startswith('|---'): + continue + + raw_title = [ + raw_content.strip() for raw_content in line_content.split('|')[1:-1] + ][0] + + title_match = link_re.match(raw_title) + if title_match: + title = title_match.group(1).upper() + categories[category].append(title) + + return (categories, category_line_num) + + +def check_alphabetical_order(lines: List[str]) -> None: + + categories, category_line_num = get_categories_content(contents=lines) + + for category, api_list in categories.items(): + if sorted(api_list) != api_list: + message = error_message( + category_line_num[category], + f'{category} category is not alphabetical order' + ) + errors.append(message) From de1b6f59d1b72ac56c32761d5927ccf8cd31b42e Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Sun, 16 Jan 2022 19:59:59 -0300 Subject: [PATCH 33/79] Check markdown file format --- scripts/validate/format.py | 164 +++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) diff --git a/scripts/validate/format.py b/scripts/validate/format.py index 1f0c96aa..2a7e7822 100644 --- a/scripts/validate/format.py +++ b/scripts/validate/format.py @@ -75,3 +75,167 @@ def check_alphabetical_order(lines: List[str]) -> None: f'{category} category is not alphabetical order' ) errors.append(message) + + +def check_title(line_num: int, raw_title: str) -> List[str]: + + err_msgs = [] + + title_match = link_re.match(raw_title) + + # url should be wrapped in "[TITLE](LINK)" Markdown syntax + if not title_match: + err_msg = error_message(line_num, 'Title syntax should be "[TITLE](LINK)"') + err_msgs.append(err_msg) + else: + # do not allow "... API" in the entry title + title = title_match.group(1) + if title.upper().endswith(' API'): + err_msg = error_message(line_num, 'Title should not end with "... API". Every entry is an API here!') + err_msgs.append(err_msg) + + return err_msgs + + +def check_description(line_num: int, description: str) -> List[str]: + + err_msgs = [] + + first_char = description[0] + if first_char.upper() != first_char: + err_msg = error_message(line_num, 'first character of description is not capitalized') + err_msgs.append(err_msg) + + last_char = description[-1] + if last_char in punctuation: + err_msg = error_message(line_num, f'description should not end with {last_char}') + err_msgs.append(err_msg) + + desc_length = len(description) + if desc_length > 100: + err_msg = error_message(line_num, f'description should not exceed 100 characters (currently {desc_length})') + err_msgs.append(err_msg) + + return err_msgs + + +def check_auth(line_num: int, auth: str) -> List[str]: + + err_msgs = [] + + backtick = '`' + if auth != 'No' and (not auth.startswith(backtick) or not auth.endswith(backtick)): + err_msg = error_message(line_num, 'auth value is not enclosed with `backticks`') + err_msgs.append(err_msg) + + if auth.replace(backtick, '') not in auth_keys: + err_msg = error_message(line_num, f'{auth} is not a valid Auth option') + err_msgs.append(err_msg) + + return err_msgs + + +def check_https(line_num: int, https: str) -> List[str]: + + err_msgs = [] + + if https not in https_keys: + err_msg = error_message(line_num, f'{https} is not a valid HTTPS option') + err_msgs.append(err_msg) + + return err_msgs + + +def check_cors(line_num: int, cors: str) -> List[str]: + + err_msgs = [] + + if cors not in cors_keys: + err_msg = error_message(line_num, f'{cors} is not a valid CORS option') + err_msgs.append(err_msg) + + return err_msgs + + +def check_entry(line_num: int, segments: List[str]) -> List[str]: + + raw_title = segments[index_title] + description = segments[index_desc] + auth = segments[index_auth] + https = segments[index_https] + cors = segments[index_cors] + + title_err_msgs = check_title(line_num, raw_title) + desc_err_msgs = check_description(line_num, description) + auth_err_msgs = check_auth(line_num, auth) + https_err_msgs = check_https(line_num, https) + cors_err_msgs = check_cors(line_num, cors) + + err_msgs = [ + *title_err_msgs, + *desc_err_msgs, + *auth_err_msgs, + *https_err_msgs, + *cors_err_msgs + ] + + return err_msgs + + +def check_file_format(filename: str) -> None: + + with open(filename, mode='r', encoding='utf-8') as file: + lines = list(line.rstrip() for line in file) + + check_alphabetical_order(lines) + + num_in_category = min_entries_per_section + 1 + category = '' + category_line = 0 + + for line_num, line in enumerate(lines): + + section_title_match = section_title_re.match(line) + if section_title_match: + title_links.append(section_title_match.group(1)) + + # check each section for the minimum number of entries + if line.startswith(anchor): + category_match = anchor_re.match(line) + if category_match: + if category_match.group(1) not in title_links: + message = error_message(line_num, f'section header ({category_match.group(1)}) not added as a title link') + errors.append(message) + else: + message = error_message(line_num, 'section header is not formatted correctly') + errors.append(message) + + if num_in_category < min_entries_per_section: + message = error_message(category_line, f'{category} section does not have the minimum {min_entries_per_section} entries (only has {num_in_category})') + errors.append(message) + + category = line.split(' ')[1] + category_line = line_num + num_in_category = 0 + continue + + # skips lines that we do not care about + if not line.startswith('|') or line.startswith('|---'): + continue + + num_in_category += 1 + segments = line.split('|')[1:-1] + if len(segments) < num_segments: + message = error_message(line_num, f'entry does not have all the required sections (have {len(segments)}, need {num_segments})') + errors.append(message) + continue + + for segment in segments: + # every line segment should start and end with exactly 1 space + if len(segment) - len(segment.lstrip()) != 1 or len(segment) - len(segment.rstrip()) != 1: + message = error_message(line_num, 'each segment must start and end with exactly 1 space') + errors.append(message) + + segments = [segment.strip() for segment in segments] + entry_err_msgs = check_entry(line_num, segments) + errors.extend(entry_err_msgs) From a935a05d6a34c810baa54f39a80174965119e105 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Sun, 16 Jan 2022 22:41:33 -0300 Subject: [PATCH 34/79] Change section -> category --- scripts/validate/format.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/validate/format.py b/scripts/validate/format.py index 2a7e7822..b8e5193f 100644 --- a/scripts/validate/format.py +++ b/scripts/validate/format.py @@ -22,7 +22,7 @@ num_segments = 5 errors = [] title_links = [] anchor_re = re.compile(anchor + '\s(.+)') -section_title_re = re.compile('\*\s\[(.*)\]') +category_title_in_index_re = re.compile('\*\s\[(.*)\]') link_re = re.compile('\[(.+)\]\((http.*)\)') # Type aliases @@ -195,23 +195,23 @@ def check_file_format(filename: str) -> None: for line_num, line in enumerate(lines): - section_title_match = section_title_re.match(line) - if section_title_match: - title_links.append(section_title_match.group(1)) + category_title_match = category_title_in_index_re.match(line) + if category_title_match: + title_links.append(category_title_match.group(1)) - # check each section for the minimum number of entries + # check each category for the minimum number of entries if line.startswith(anchor): category_match = anchor_re.match(line) if category_match: if category_match.group(1) not in title_links: - message = error_message(line_num, f'section header ({category_match.group(1)}) not added as a title link') + message = error_message(line_num, f'category header ({category_match.group(1)}) not added to Index section') errors.append(message) else: - message = error_message(line_num, 'section header is not formatted correctly') + message = error_message(line_num, 'category header is not formatted correctly') errors.append(message) if num_in_category < min_entries_per_section: - message = error_message(category_line, f'{category} section does not have the minimum {min_entries_per_section} entries (only has {num_in_category})') + message = error_message(category_line, f'{category} category does not have the minimum {min_entries_per_section} entries (only has {num_in_category})') errors.append(message) category = line.split(' ')[1] From d578ba7a3211e3887398a4e7d38c9e9a42471a9c Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Sun, 16 Jan 2022 22:47:17 -0300 Subject: [PATCH 35/79] Change line -> line_content --- scripts/validate/format.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/validate/format.py b/scripts/validate/format.py index b8e5193f..34e08f12 100644 --- a/scripts/validate/format.py +++ b/scripts/validate/format.py @@ -193,15 +193,15 @@ def check_file_format(filename: str) -> None: category = '' category_line = 0 - for line_num, line in enumerate(lines): + for line_num, line_content in enumerate(lines): - category_title_match = category_title_in_index_re.match(line) + category_title_match = category_title_in_index_re.match(line_content) if category_title_match: title_links.append(category_title_match.group(1)) # check each category for the minimum number of entries - if line.startswith(anchor): - category_match = anchor_re.match(line) + if line_content.startswith(anchor): + category_match = anchor_re.match(line_content) if category_match: if category_match.group(1) not in title_links: message = error_message(line_num, f'category header ({category_match.group(1)}) not added to Index section') @@ -214,17 +214,17 @@ def check_file_format(filename: str) -> None: message = error_message(category_line, f'{category} category does not have the minimum {min_entries_per_section} entries (only has {num_in_category})') errors.append(message) - category = line.split(' ')[1] + category = line_content.split(' ')[1] category_line = line_num num_in_category = 0 continue # skips lines that we do not care about - if not line.startswith('|') or line.startswith('|---'): + if not line_content.startswith('|') or line_content.startswith('|---'): continue num_in_category += 1 - segments = line.split('|')[1:-1] + segments = line_content.split('|')[1:-1] if len(segments) < num_segments: message = error_message(line_num, f'entry does not have all the required sections (have {len(segments)}, need {num_segments})') errors.append(message) From 57e288b5c96507addb148f898b80d4373083ff1e Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Sun, 16 Jan 2022 23:11:34 -0300 Subject: [PATCH 36/79] Refactor check_alphabetical_order and check_file_format to exist return --- scripts/validate/format.py | 42 +++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/scripts/validate/format.py b/scripts/validate/format.py index 34e08f12..6bcdd218 100644 --- a/scripts/validate/format.py +++ b/scripts/validate/format.py @@ -19,7 +19,6 @@ index_cors = 4 index_link = 5 num_segments = 5 -errors = [] title_links = [] anchor_re = re.compile(anchor + '\s(.+)') category_title_in_index_re = re.compile('\*\s\[(.*)\]') @@ -64,17 +63,21 @@ def get_categories_content(contents: List[str]) -> Tuple[Categories, CategoriesL return (categories, category_line_num) -def check_alphabetical_order(lines: List[str]) -> None: +def check_alphabetical_order(lines: List[str]) -> List[str]: + + err_msgs = [] categories, category_line_num = get_categories_content(contents=lines) for category, api_list in categories.items(): if sorted(api_list) != api_list: - message = error_message( + err_msg = error_message( category_line_num[category], f'{category} category is not alphabetical order' ) - errors.append(message) + err_msgs.append(err_msg) + + return err_msgs def check_title(line_num: int, raw_title: str) -> List[str]: @@ -182,12 +185,15 @@ def check_entry(line_num: int, segments: List[str]) -> List[str]: return err_msgs -def check_file_format(filename: str) -> None: +def check_file_format(filename: str) -> List[str]: + + err_msgs = [] with open(filename, mode='r', encoding='utf-8') as file: lines = list(line.rstrip() for line in file) - check_alphabetical_order(lines) + alphabetical_err_msgs = check_alphabetical_order(lines) + err_msgs.extend(alphabetical_err_msgs) num_in_category = min_entries_per_section + 1 category = '' @@ -204,15 +210,15 @@ def check_file_format(filename: str) -> None: category_match = anchor_re.match(line_content) if category_match: if category_match.group(1) not in title_links: - message = error_message(line_num, f'category header ({category_match.group(1)}) not added to Index section') - errors.append(message) + err_msg = error_message(line_num, f'category header ({category_match.group(1)}) not added to Index section') + err_msgs.append(err_msg) else: - message = error_message(line_num, 'category header is not formatted correctly') - errors.append(message) + err_msg = error_message(line_num, 'category header is not formatted correctly') + err_msgs.append(err_msg) if num_in_category < min_entries_per_section: - message = error_message(category_line, f'{category} category does not have the minimum {min_entries_per_section} entries (only has {num_in_category})') - errors.append(message) + err_msg = error_message(category_line, f'{category} category does not have the minimum {min_entries_per_section} entries (only has {num_in_category})') + err_msgs.append(err_msg) category = line_content.split(' ')[1] category_line = line_num @@ -226,16 +232,18 @@ def check_file_format(filename: str) -> None: num_in_category += 1 segments = line_content.split('|')[1:-1] if len(segments) < num_segments: - message = error_message(line_num, f'entry does not have all the required sections (have {len(segments)}, need {num_segments})') - errors.append(message) + err_msg = error_message(line_num, f'entry does not have all the required sections (have {len(segments)}, need {num_segments})') + err_msgs.append(err_msg) continue for segment in segments: # every line segment should start and end with exactly 1 space if len(segment) - len(segment.lstrip()) != 1 or len(segment) - len(segment.rstrip()) != 1: - message = error_message(line_num, 'each segment must start and end with exactly 1 space') - errors.append(message) + err_msg = error_message(line_num, 'each segment must start and end with exactly 1 space') + err_msgs.append(err_msg) segments = [segment.strip() for segment in segments] entry_err_msgs = check_entry(line_num, segments) - errors.extend(entry_err_msgs) + err_msgs.extend(entry_err_msgs) + + return err_msgs From f1cbf0144a5afff9489bcb23e148dadf9a00e415 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Sun, 16 Jan 2022 23:18:06 -0300 Subject: [PATCH 37/79] Change title_links global to category_title_in_index local --- scripts/validate/format.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/validate/format.py b/scripts/validate/format.py index 6bcdd218..97a169e1 100644 --- a/scripts/validate/format.py +++ b/scripts/validate/format.py @@ -19,7 +19,6 @@ index_cors = 4 index_link = 5 num_segments = 5 -title_links = [] anchor_re = re.compile(anchor + '\s(.+)') category_title_in_index_re = re.compile('\*\s\[(.*)\]') link_re = re.compile('\[(.+)\]\((http.*)\)') @@ -188,6 +187,7 @@ def check_entry(line_num: int, segments: List[str]) -> List[str]: def check_file_format(filename: str) -> List[str]: err_msgs = [] + category_title_in_index = [] with open(filename, mode='r', encoding='utf-8') as file: lines = list(line.rstrip() for line in file) @@ -203,13 +203,13 @@ def check_file_format(filename: str) -> List[str]: category_title_match = category_title_in_index_re.match(line_content) if category_title_match: - title_links.append(category_title_match.group(1)) + category_title_in_index.append(category_title_match.group(1)) # check each category for the minimum number of entries if line_content.startswith(anchor): category_match = anchor_re.match(line_content) if category_match: - if category_match.group(1) not in title_links: + if category_match.group(1) not in category_title_in_index: err_msg = error_message(line_num, f'category header ({category_match.group(1)}) not added to Index section') err_msgs.append(err_msg) else: From bff7f5e0096ee7f665fe41f7439540296a9a77ac Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Sun, 16 Jan 2022 23:20:43 -0300 Subject: [PATCH 38/79] Reorganize constants --- scripts/validate/format.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/validate/format.py b/scripts/validate/format.py index 97a169e1..89df6bee 100644 --- a/scripts/validate/format.py +++ b/scripts/validate/format.py @@ -5,7 +5,6 @@ from typing import List, Tuple, Dict anchor = '###' -min_entries_per_section = 3 auth_keys = ['apiKey', 'OAuth', 'X-Mashape-Key', 'User-Agent', 'No'] punctuation = ['.', '?', '!'] https_keys = ['Yes', 'No'] @@ -17,7 +16,9 @@ index_auth = 2 index_https = 3 index_cors = 4 index_link = 5 + num_segments = 5 +min_entries_per_section = 3 anchor_re = re.compile(anchor + '\s(.+)') category_title_in_index_re = re.compile('\*\s\[(.*)\]') From 6bab1616abea2a5508996969120c5cf7fefad948 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Sun, 16 Jan 2022 23:22:23 -0300 Subject: [PATCH 39/79] Remove index_link: unused variable --- scripts/validate/format.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/validate/format.py b/scripts/validate/format.py index 89df6bee..1a541902 100644 --- a/scripts/validate/format.py +++ b/scripts/validate/format.py @@ -15,7 +15,6 @@ index_desc = 1 index_auth = 2 index_https = 3 index_cors = 4 -index_link = 5 num_segments = 5 min_entries_per_section = 3 From d200415479a8888b90e941079d30a42b0b7b599b Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Sun, 16 Jan 2022 23:25:09 -0300 Subject: [PATCH 40/79] Change min_entries_per_section -> min_entries_per_category --- scripts/validate/format.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/validate/format.py b/scripts/validate/format.py index 1a541902..4db980a4 100644 --- a/scripts/validate/format.py +++ b/scripts/validate/format.py @@ -17,7 +17,7 @@ index_https = 3 index_cors = 4 num_segments = 5 -min_entries_per_section = 3 +min_entries_per_category = 3 anchor_re = re.compile(anchor + '\s(.+)') category_title_in_index_re = re.compile('\*\s\[(.*)\]') @@ -195,7 +195,7 @@ def check_file_format(filename: str) -> List[str]: alphabetical_err_msgs = check_alphabetical_order(lines) err_msgs.extend(alphabetical_err_msgs) - num_in_category = min_entries_per_section + 1 + num_in_category = min_entries_per_category + 1 category = '' category_line = 0 @@ -216,8 +216,8 @@ def check_file_format(filename: str) -> List[str]: err_msg = error_message(line_num, 'category header is not formatted correctly') err_msgs.append(err_msg) - if num_in_category < min_entries_per_section: - err_msg = error_message(category_line, f'{category} category does not have the minimum {min_entries_per_section} entries (only has {num_in_category})') + if num_in_category < min_entries_per_category: + err_msg = error_message(category_line, f'{category} category does not have the minimum {min_entries_per_category} entries (only has {num_in_category})') err_msgs.append(err_msg) category = line_content.split(' ')[1] From 1d421d8856bcfe7f21ed16623c3e83738c23b59b Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Sun, 16 Jan 2022 23:29:55 -0300 Subject: [PATCH 41/79] Add max_description_length constant --- scripts/validate/format.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/validate/format.py b/scripts/validate/format.py index 4db980a4..6c8681e9 100644 --- a/scripts/validate/format.py +++ b/scripts/validate/format.py @@ -18,6 +18,7 @@ index_cors = 4 num_segments = 5 min_entries_per_category = 3 +max_description_length = 100 anchor_re = re.compile(anchor + '\s(.+)') category_title_in_index_re = re.compile('\*\s\[(.*)\]') @@ -114,7 +115,7 @@ def check_description(line_num: int, description: str) -> List[str]: err_msgs.append(err_msg) desc_length = len(description) - if desc_length > 100: + if desc_length > max_description_length: err_msg = error_message(line_num, f'description should not exceed 100 characters (currently {desc_length})') err_msgs.append(err_msg) From f447f7c76e330ce9ea203e93929ecfa8cca75113 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Sun, 16 Jan 2022 23:38:14 -0300 Subject: [PATCH 42/79] Implement main function --- scripts/validate/format.py | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/scripts/validate/format.py b/scripts/validate/format.py index 6c8681e9..8f88a7e4 100644 --- a/scripts/validate/format.py +++ b/scripts/validate/format.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import re +import sys from typing import List, Tuple, Dict @@ -185,14 +186,11 @@ def check_entry(line_num: int, segments: List[str]) -> List[str]: return err_msgs -def check_file_format(filename: str) -> List[str]: +def check_file_format(lines: List[str]) -> List[str]: err_msgs = [] category_title_in_index = [] - with open(filename, mode='r', encoding='utf-8') as file: - lines = list(line.rstrip() for line in file) - alphabetical_err_msgs = check_alphabetical_order(lines) err_msgs.extend(alphabetical_err_msgs) @@ -248,3 +246,29 @@ def check_file_format(filename: str) -> List[str]: err_msgs.extend(entry_err_msgs) return err_msgs + + +def main(filename: str) -> None: + + with open(filename, mode='r', encoding='utf-8') as file: + lines = list(line.rstrip() for line in file) + + file_format_err_msgs = check_file_format(lines) + + if file_format_err_msgs: + for err_msg in file_format_err_msgs: + print(err_msg) + sys.exit(1) + + +if __name__ == '__main__': + + num_args = len(sys.argv) + + if num_args < 2: + print('No .md file passed (file should contain Markdown table syntax)') + sys.exit(1) + + filename = sys.argv[1] + + main(filename) From 71167b27168379711ebd77147764f131169c8db7 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Sun, 16 Jan 2022 23:41:41 -0300 Subject: [PATCH 43/79] Use punctuation constant built-in --- scripts/validate/format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/validate/format.py b/scripts/validate/format.py index 8f88a7e4..c70a7756 100644 --- a/scripts/validate/format.py +++ b/scripts/validate/format.py @@ -2,12 +2,12 @@ import re import sys +from string import punctuation from typing import List, Tuple, Dict anchor = '###' auth_keys = ['apiKey', 'OAuth', 'X-Mashape-Key', 'User-Agent', 'No'] -punctuation = ['.', '?', '!'] https_keys = ['Yes', 'No'] cors_keys = ['Yes', 'No', 'Unknown'] From e2482af4485a2f8f6da15a3cef7574473963ea45 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Sun, 16 Jan 2022 23:49:12 -0300 Subject: [PATCH 44/79] Add temporary replacement of punctuation --- scripts/validate/format.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/validate/format.py b/scripts/validate/format.py index c70a7756..b22696d8 100644 --- a/scripts/validate/format.py +++ b/scripts/validate/format.py @@ -5,6 +5,9 @@ import sys from string import punctuation from typing import List, Tuple, Dict +# Temporary replacement +# The descriptions that contain () at the end must adapt to the new policy later +punctuation = punctuation.replace('()', '') anchor = '###' auth_keys = ['apiKey', 'OAuth', 'X-Mashape-Key', 'User-Agent', 'No'] From d831102a5db20f21bec77c9563c37ca0980920f5 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 01:40:49 -0300 Subject: [PATCH 45/79] Create test case to validate format --- scripts/tests/test_validate_format.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 scripts/tests/test_validate_format.py diff --git a/scripts/tests/test_validate_format.py b/scripts/tests/test_validate_format.py new file mode 100644 index 00000000..a66f703c --- /dev/null +++ b/scripts/tests/test_validate_format.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- + +import unittest + + +class TestValidadeFormat(unittest.TestCase): + ... From 40c5fa7a0c13f894444e846f81e9a846b890c236 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 02:13:34 -0300 Subject: [PATCH 46/79] Test error message return and return type --- scripts/tests/test_validate_format.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/scripts/tests/test_validate_format.py b/scripts/tests/test_validate_format.py index a66f703c..824e426f 100644 --- a/scripts/tests/test_validate_format.py +++ b/scripts/tests/test_validate_format.py @@ -1,7 +1,32 @@ # -*- coding: utf-8 -*- +from email import message import unittest +from validate.format import error_message + class TestValidadeFormat(unittest.TestCase): - ... + + def test_error_message_return_and_return_type(self): + line_num_unity = 1 + line_num_ten = 10 + line_num_hundred = 100 + line_num_thousand = 1000 + + msg = 'This is a unit test' + + err_msg_unity = error_message(line_num_unity, msg) + err_msg_ten = error_message(line_num_ten, msg) + err_msg_hundred = error_message(line_num_hundred, msg) + err_msg_thousand = error_message(line_num_thousand, msg) + + self.assertIsInstance(err_msg_unity, str) + self.assertIsInstance(err_msg_ten, str) + self.assertIsInstance(err_msg_hundred, str) + self.assertIsInstance(err_msg_thousand, str) + + self.assertEqual(err_msg_unity, '(L002) This is a unit test') + self.assertEqual(err_msg_ten, '(L011) This is a unit test') + self.assertEqual(err_msg_hundred, '(L101) This is a unit test') + self.assertEqual(err_msg_thousand, '(L1001) This is a unit test') From 9d1bd19383ff34525d0e1344adccadb083e57f27 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 02:53:02 -0300 Subject: [PATCH 47/79] Create test to get_categories_content --- scripts/tests/test_validate_format.py | 30 +++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/scripts/tests/test_validate_format.py b/scripts/tests/test_validate_format.py index 824e426f..4d32c80c 100644 --- a/scripts/tests/test_validate_format.py +++ b/scripts/tests/test_validate_format.py @@ -4,6 +4,7 @@ from email import message import unittest from validate.format import error_message +from validate.format import get_categories_content class TestValidadeFormat(unittest.TestCase): @@ -30,3 +31,32 @@ class TestValidadeFormat(unittest.TestCase): self.assertEqual(err_msg_ten, '(L011) This is a unit test') self.assertEqual(err_msg_hundred, '(L101) This is a unit test') self.assertEqual(err_msg_thousand, '(L1001) This is a unit test') + + def test_if_get_categories_content_return_correct_data_of_categories(self): + fake_contents = [ + '### A', + 'API | Description | Auth | HTTPS | CORS |', + '|---|---|---|---|---|', + '| [AA](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |', + '| [AB](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |', + '', + '### B', + 'API | Description | Auth | HTTPS | CORS |', + '|---|---|---|---|---|', + '| [BA](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |', + '| [BB](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |' + ] + + result = get_categories_content(fake_contents) + self.assertIsInstance(result, tuple) + + categories, category_line_num = result + self.assertIsInstance(categories, dict) + self.assertIsInstance(category_line_num, dict) + + expected_result = ({'A': ['AA', 'AB'], 'B': ['BA', 'BB']}, {'A': 0, 'B': 6}) + + for res, ex_res in zip(result, expected_result): + + with self.subTest(): + self.assertEqual(res, ex_res) From b68d653a108967f350e8a1349d4c7652b88d9899 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 02:53:35 -0300 Subject: [PATCH 48/79] Remove whitespace --- scripts/tests/test_validate_format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/test_validate_format.py b/scripts/tests/test_validate_format.py index 4d32c80c..417b3e5d 100644 --- a/scripts/tests/test_validate_format.py +++ b/scripts/tests/test_validate_format.py @@ -53,7 +53,7 @@ class TestValidadeFormat(unittest.TestCase): categories, category_line_num = result self.assertIsInstance(categories, dict) self.assertIsInstance(category_line_num, dict) - + expected_result = ({'A': ['AA', 'AB'], 'B': ['BA', 'BB']}, {'A': 0, 'B': 6}) for res, ex_res in zip(result, expected_result): From b42c3054646d44069e9e8c6c0d149f5e07f08611 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 03:19:14 -0300 Subject: [PATCH 49/79] Create tests to check_alphabetical_order --- scripts/tests/test_validate_format.py | 50 +++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/scripts/tests/test_validate_format.py b/scripts/tests/test_validate_format.py index 417b3e5d..47881d42 100644 --- a/scripts/tests/test_validate_format.py +++ b/scripts/tests/test_validate_format.py @@ -5,6 +5,7 @@ import unittest from validate.format import error_message from validate.format import get_categories_content +from validate.format import check_alphabetical_order class TestValidadeFormat(unittest.TestCase): @@ -60,3 +61,52 @@ class TestValidadeFormat(unittest.TestCase): with self.subTest(): self.assertEqual(res, ex_res) + + def test_if_check_alphabetical_order_return_correct_msg_error(self): + correct_lines = [ + '### A', + 'API | Description | Auth | HTTPS | CORS |', + '|---|---|---|---|---|', + '| [AA](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |', + '| [AB](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |', + '', + '### B', + 'API | Description | Auth | HTTPS | CORS |', + '|---|---|---|---|---|', + '| [BA](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |', + '| [BB](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |' + ] + + incorrect_lines = [ + '### A', + 'API | Description | Auth | HTTPS | CORS |', + '|---|---|---|---|---|', + '| [AB](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |', + '| [AA](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |', + '', + '### B', + 'API | Description | Auth | HTTPS | CORS |', + '|---|---|---|---|---|', + '| [BB](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |', + '| [BA](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |' + ] + + + err_msgs_1 = check_alphabetical_order(correct_lines) + err_msgs_2 = check_alphabetical_order(incorrect_lines) + + self.assertIsInstance(err_msgs_1, list) + self.assertIsInstance(err_msgs_2, list) + + self.assertEqual(len(err_msgs_1), 0) + self.assertEqual(len(err_msgs_2), 2) + + expected_err_msgs = [ + '(L001) A category is not alphabetical order', + '(L007) B category is not alphabetical order' + ] + + for err_msg, ex_err_msg in zip(err_msgs_2, expected_err_msgs): + + with self.subTest(): + self.assertEqual(err_msg, ex_err_msg) From 22e22b3ce27a15f3121336d79fdb956a6c5bd993 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 03:55:55 -0300 Subject: [PATCH 50/79] Create tests to check_title --- scripts/tests/test_validate_format.py | 54 ++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/scripts/tests/test_validate_format.py b/scripts/tests/test_validate_format.py index 47881d42..a4cb1803 100644 --- a/scripts/tests/test_validate_format.py +++ b/scripts/tests/test_validate_format.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- -from email import message import unittest from validate.format import error_message from validate.format import get_categories_content from validate.format import check_alphabetical_order +from validate.format import check_title class TestValidadeFormat(unittest.TestCase): @@ -110,3 +110,55 @@ class TestValidadeFormat(unittest.TestCase): with self.subTest(): self.assertEqual(err_msg, ex_err_msg) + + def test_check_title_return_type(self): + raw_title_1 = '[A](https://www.ex.com)' + raw_title_2 = '[A(https://www.ex.com)' + raw_title_3 = '[A API](https://www.ex.com)' + + result_1 = check_title(0, raw_title_1) + result_2 = check_title(0, raw_title_2) + result_3 = check_title(0, raw_title_3) + + self.assertIsInstance(result_1, list) + self.assertIsInstance(result_2, list) + self.assertIsInstance(result_3, list) + + err_msg_1 = result_2[0] + err_msg_2 = result_3[0] + + self.assertIsInstance(err_msg_1, str) + self.assertIsInstance(err_msg_2, str) + + def test_check_title_with_correct_title(self): + raw_title = '[A](https://www.ex.com)' + + err_msgs = check_title(0, raw_title) + + self.assertEqual(len(err_msgs), 0) + + self.assertEqual(err_msgs, []) + + def test_check_title_with_markdown_syntax_incorrect(self): + raw_title = '[A(https://www.ex.com)' + + err_msgs = check_title(0, raw_title) + + self.assertEqual(len(err_msgs), 1) + + err_msg = err_msgs[0] + expected_err_msg = '(L001) Title syntax should be "[TITLE](LINK)"' + + self.assertEqual(err_msg, expected_err_msg) + + def test_check_title_with_api_at_the_end_of_the_title(self): + raw_title = '[A API](https://www.ex.com)' + + err_msgs = check_title(0, raw_title) + + self.assertEqual(len(err_msgs), 1) + + err_msg = err_msgs[0] + expected_err_msg = '(L001) Title should not end with "... API". Every entry is an API here!' + + self.assertEqual(err_msg, expected_err_msg) From 9f0174fd96b5900be3053afc0137fe65dee02cdd Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 04:27:58 -0300 Subject: [PATCH 51/79] Fix error message of max description length --- scripts/validate/format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/validate/format.py b/scripts/validate/format.py index b22696d8..2d1f90cb 100644 --- a/scripts/validate/format.py +++ b/scripts/validate/format.py @@ -120,7 +120,7 @@ def check_description(line_num: int, description: str) -> List[str]: desc_length = len(description) if desc_length > max_description_length: - err_msg = error_message(line_num, f'description should not exceed 100 characters (currently {desc_length})') + err_msg = error_message(line_num, f'description should not exceed {max_description_length} characters (currently {desc_length})') err_msgs.append(err_msg) return err_msgs From 0843afeda1f78ce86f7fef7fa78e10746df39d44 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 04:35:52 -0300 Subject: [PATCH 52/79] Create tests to check_description --- scripts/tests/test_validate_format.py | 76 +++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/scripts/tests/test_validate_format.py b/scripts/tests/test_validate_format.py index a4cb1803..c69d2bcf 100644 --- a/scripts/tests/test_validate_format.py +++ b/scripts/tests/test_validate_format.py @@ -6,6 +6,7 @@ from validate.format import error_message from validate.format import get_categories_content from validate.format import check_alphabetical_order from validate.format import check_title +from validate.format import check_description, max_description_length class TestValidadeFormat(unittest.TestCase): @@ -162,3 +163,78 @@ class TestValidadeFormat(unittest.TestCase): expected_err_msg = '(L001) Title should not end with "... API". Every entry is an API here!' self.assertEqual(err_msg, expected_err_msg) + + def test_checK_description_return_type(self): + desc_1 = 'This is a fake description' + desc_2 = 'this is a fake description' + desc_3 = 'This is a fake description!' + desc_4 = 'This is a fake descriptionThis is a fake descriptionThis is a fake descriptionThis is a fake description' + + result_1 = check_title(0, desc_1) + result_2 = check_title(0, desc_2) + result_3 = check_title(0, desc_3) + result_4 = check_title(0, desc_4) + + self.assertIsInstance(result_1, list) + self.assertIsInstance(result_2, list) + self.assertIsInstance(result_3, list) + self.assertIsInstance(result_4, list) + + err_msg_1 = result_2[0] + err_msg_2 = result_3[0] + err_msg_3 = result_4[0] + + self.assertIsInstance(err_msg_1, str) + self.assertIsInstance(err_msg_2, str) + self.assertIsInstance(err_msg_3, str) + + def test_check_description_with_correct_description(self): + desc = 'This is a fake description' + + err_msgs = check_description(0, desc) + + self.assertEqual(len(err_msgs), 0) + + self.assertEqual(err_msgs, []) + + def test_check_description_with_first_char_is_not_capitalized(self): + desc = 'this is a fake description' + + err_msgs = check_description(0, desc) + + self.assertEqual(len(err_msgs), 1) + + err_msg = err_msgs[0] + expected_err_msg = '(L001) first character of description is not capitalized' + + self.assertEqual(err_msg, expected_err_msg) + + def test_check_description_with_punctuation_in_the_end(self): + base_desc = 'This is a fake description' + punctuation = r"""!"#$%&'*+,-./:;<=>?@[\]^_`{|}~""" + desc_with_punc = [base_desc + punc for punc in punctuation] + + for desc in desc_with_punc: + + with self.subTest(): + err_msgs = check_description(0, desc) + + self.assertEqual(len(err_msgs), 1) + + err_msg = err_msgs[0] + expected_err_msg = f'(L001) description should not end with {desc[-1]}' + + self.assertEqual(err_msg, expected_err_msg) + + def test_check_description_that_exceeds_the_character_limit(self): + long_desc = 'Desc' * max_description_length + long_desc_length = len(long_desc) + + err_msgs = check_description(0, long_desc) + + self.assertEqual(len(err_msgs), 1) + + err_msg = err_msgs[0] + expected_err_msg = f'(L001) description should not exceed {max_description_length} characters (currently {long_desc_length})' + + self.assertEqual(err_msg, expected_err_msg) From 6ba6c4d1a6180deea07fde94515acf43de741e2f Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 05:39:07 -0300 Subject: [PATCH 53/79] Create tests to check_auth --- scripts/tests/test_validate_format.py | 73 +++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/scripts/tests/test_validate_format.py b/scripts/tests/test_validate_format.py index c69d2bcf..d076f035 100644 --- a/scripts/tests/test_validate_format.py +++ b/scripts/tests/test_validate_format.py @@ -7,6 +7,7 @@ from validate.format import get_categories_content from validate.format import check_alphabetical_order from validate.format import check_title from validate.format import check_description, max_description_length +from validate.format import check_auth, auth_keys class TestValidadeFormat(unittest.TestCase): @@ -238,3 +239,75 @@ class TestValidadeFormat(unittest.TestCase): expected_err_msg = f'(L001) description should not exceed {max_description_length} characters (currently {long_desc_length})' self.assertEqual(err_msg, expected_err_msg) + + def test_check_auth_return_type(self): + auth_with_backtick = [f'`{auth}`' for auth in auth_keys if auth != 'No'] + auth_with_backtick.append('No') + auth_without_backtick = [auth for auth in auth_keys if auth != 'No'] + auth_invalid = ['Yes', 'yes', 'no', 'random', 'Unknown'] + + for auth in auth_with_backtick: + with self.subTest(): + result = check_auth(0, auth) + self.assertIsInstance(result, list) + + for auth in auth_without_backtick: + with self.subTest(): + err_msgs = check_auth(0, auth) + self.assertIsInstance(result, list) + err_msg = err_msgs[0] + self.assertIsInstance(err_msg, str) + + for auth in auth_invalid: + with self.subTest(): + err_msgs = check_auth(0, auth) + self.assertIsInstance(result, list) + err_msg = err_msgs[0] + self.assertIsInstance(err_msg, str) + + def test_check_auth_with_correct_auth(self): + auth_valid = [f'`{auth}`' for auth in auth_keys if auth != 'No'] + auth_valid.append('No') + + for auth in auth_valid: + with self.subTest(): + err_msgs = check_auth(0, auth) + self.assertIsInstance(err_msgs, list) + + self.assertEqual(len(err_msgs), 0) + + self.assertEqual(err_msgs, []) + + def test_check_auth_without_backtick(self): + auth_without_backtick = [auth for auth in auth_keys if auth != 'No'] + + for auth in auth_without_backtick: + with self.subTest(): + err_msgs = check_auth(0, auth) + self.assertIsInstance(err_msgs, list) + + self.assertEqual(len(err_msgs), 1) + + err_msg = err_msgs[0] + expected_err_msg = '(L001) auth value is not enclosed with `backticks`' + + self.assertEqual(err_msg, expected_err_msg) + + def test_check_auth_with_invalid_auth(self): + auth_invalid = ['Yes', 'yes', 'no', 'random', 'Unknown'] + + for auth in auth_invalid: + with self.subTest(): + err_msgs = check_auth(0, auth) + self.assertIsInstance(err_msgs, list) + + self.assertEqual(len(err_msgs), 2) + + err_msg_1 = err_msgs[0] + err_msg_2 = err_msgs[1] + + expected_err_msg_1 = f'(L001) auth value is not enclosed with `backticks`' + expected_err_msg_2 = f'(L001) {auth} is not a valid Auth option' + + self.assertEqual(err_msg_1, expected_err_msg_1) + self.assertEqual(err_msg_2, expected_err_msg_2) From 347cd46036a387ece3eeb5ffe63dcf1d3564ab2e Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 05:55:01 -0300 Subject: [PATCH 54/79] Create tests to check_https --- scripts/tests/test_validate_format.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/scripts/tests/test_validate_format.py b/scripts/tests/test_validate_format.py index d076f035..ed4f0d6c 100644 --- a/scripts/tests/test_validate_format.py +++ b/scripts/tests/test_validate_format.py @@ -8,6 +8,7 @@ from validate.format import check_alphabetical_order from validate.format import check_title from validate.format import check_description, max_description_length from validate.format import check_auth, auth_keys +from validate.format import check_https, https_keys class TestValidadeFormat(unittest.TestCase): @@ -311,3 +312,29 @@ class TestValidadeFormat(unittest.TestCase): self.assertEqual(err_msg_1, expected_err_msg_1) self.assertEqual(err_msg_2, expected_err_msg_2) + + def test_check_https_with_valid_https(self): + for https in https_keys: + with self.subTest(): + err_msgs = check_https(0, https) + self.assertIsInstance(err_msgs, list) + + self.assertEqual(len(err_msgs), 0) + + self.assertEqual(err_msgs, []) + + def test_check_https_with_invalid_https(self): + invalid_https_keys = ['yes', 'no', 'Unknown', 'https', 'http'] + + for https in invalid_https_keys: + with self.subTest(): + err_msgs = check_https(0, https) + self.assertIsInstance(err_msgs, list) + + self.assertEqual(len(err_msgs), 1) + + err_msg = err_msgs[0] + expected_err_msg = f'(L001) {https} is not a valid HTTPS option' + + self.assertIsInstance(err_msg, str) + self.assertEqual(err_msg, expected_err_msg) From d04c02a7d96f8507b5e1abd7c9e22c8c7714c8cf Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 05:59:17 -0300 Subject: [PATCH 55/79] Create tests to check_cors --- scripts/tests/test_validate_format.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/scripts/tests/test_validate_format.py b/scripts/tests/test_validate_format.py index ed4f0d6c..53bf7868 100644 --- a/scripts/tests/test_validate_format.py +++ b/scripts/tests/test_validate_format.py @@ -9,6 +9,7 @@ from validate.format import check_title from validate.format import check_description, max_description_length from validate.format import check_auth, auth_keys from validate.format import check_https, https_keys +from validate.format import check_cors, cors_keys class TestValidadeFormat(unittest.TestCase): @@ -338,3 +339,29 @@ class TestValidadeFormat(unittest.TestCase): self.assertIsInstance(err_msg, str) self.assertEqual(err_msg, expected_err_msg) + + def test_check_cors_with_valid_cors(self): + for cors in cors_keys: + with self.subTest(): + err_msgs = check_cors(0, cors) + self.assertIsInstance(err_msgs, list) + + self.assertEqual(len(err_msgs), 0) + + self.assertEqual(err_msgs, []) + + def test_check_cors_with_invalid_cors(self): + invalid_cors_keys = ['yes', 'no', 'unknown', 'cors'] + + for cors in invalid_cors_keys: + with self.subTest(): + err_msgs = check_cors(0, cors) + self.assertIsInstance(err_msgs, list) + + self.assertEqual(len(err_msgs), 1) + + err_msg = err_msgs[0] + expected_err_msg = f'(L001) {cors} is not a valid CORS option' + + self.assertIsInstance(err_msg, str) + self.assertEqual(err_msg, expected_err_msg) From 229b77a41301a28398c08f9fa51b172c32a7f360 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 06:05:15 -0300 Subject: [PATCH 56/79] Fix tests of check_title --- scripts/tests/test_validate_format.py | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/scripts/tests/test_validate_format.py b/scripts/tests/test_validate_format.py index 53bf7868..94d5973a 100644 --- a/scripts/tests/test_validate_format.py +++ b/scripts/tests/test_validate_format.py @@ -114,33 +114,14 @@ class TestValidadeFormat(unittest.TestCase): with self.subTest(): self.assertEqual(err_msg, ex_err_msg) - - def test_check_title_return_type(self): - raw_title_1 = '[A](https://www.ex.com)' - raw_title_2 = '[A(https://www.ex.com)' - raw_title_3 = '[A API](https://www.ex.com)' - - result_1 = check_title(0, raw_title_1) - result_2 = check_title(0, raw_title_2) - result_3 = check_title(0, raw_title_3) - - self.assertIsInstance(result_1, list) - self.assertIsInstance(result_2, list) - self.assertIsInstance(result_3, list) - - err_msg_1 = result_2[0] - err_msg_2 = result_3[0] - - self.assertIsInstance(err_msg_1, str) - self.assertIsInstance(err_msg_2, str) def test_check_title_with_correct_title(self): raw_title = '[A](https://www.ex.com)' err_msgs = check_title(0, raw_title) + self.assertIsInstance(err_msgs, list) self.assertEqual(len(err_msgs), 0) - self.assertEqual(err_msgs, []) def test_check_title_with_markdown_syntax_incorrect(self): @@ -148,6 +129,7 @@ class TestValidadeFormat(unittest.TestCase): err_msgs = check_title(0, raw_title) + self.assertIsInstance(err_msgs, list) self.assertEqual(len(err_msgs), 1) err_msg = err_msgs[0] @@ -159,7 +141,8 @@ class TestValidadeFormat(unittest.TestCase): raw_title = '[A API](https://www.ex.com)' err_msgs = check_title(0, raw_title) - + + self.assertIsInstance(err_msgs, list) self.assertEqual(len(err_msgs), 1) err_msg = err_msgs[0] From 720c2fdc8fc129fba205d2a05c4196ddb9368de6 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 06:10:16 -0300 Subject: [PATCH 57/79] Fix tests of check_description --- scripts/tests/test_validate_format.py | 31 ++++++--------------------- 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/scripts/tests/test_validate_format.py b/scripts/tests/test_validate_format.py index 94d5973a..e5506614 100644 --- a/scripts/tests/test_validate_format.py +++ b/scripts/tests/test_validate_format.py @@ -150,35 +150,12 @@ class TestValidadeFormat(unittest.TestCase): self.assertEqual(err_msg, expected_err_msg) - def test_checK_description_return_type(self): - desc_1 = 'This is a fake description' - desc_2 = 'this is a fake description' - desc_3 = 'This is a fake description!' - desc_4 = 'This is a fake descriptionThis is a fake descriptionThis is a fake descriptionThis is a fake description' - - result_1 = check_title(0, desc_1) - result_2 = check_title(0, desc_2) - result_3 = check_title(0, desc_3) - result_4 = check_title(0, desc_4) - - self.assertIsInstance(result_1, list) - self.assertIsInstance(result_2, list) - self.assertIsInstance(result_3, list) - self.assertIsInstance(result_4, list) - - err_msg_1 = result_2[0] - err_msg_2 = result_3[0] - err_msg_3 = result_4[0] - - self.assertIsInstance(err_msg_1, str) - self.assertIsInstance(err_msg_2, str) - self.assertIsInstance(err_msg_3, str) - def test_check_description_with_correct_description(self): desc = 'This is a fake description' err_msgs = check_description(0, desc) + self.assertIsInstance(err_msgs, list) self.assertEqual(len(err_msgs), 0) self.assertEqual(err_msgs, []) @@ -188,11 +165,13 @@ class TestValidadeFormat(unittest.TestCase): err_msgs = check_description(0, desc) + self.assertIsInstance(err_msgs, list) self.assertEqual(len(err_msgs), 1) err_msg = err_msgs[0] expected_err_msg = '(L001) first character of description is not capitalized' + self.assertIsInstance(err_msg, str) self.assertEqual(err_msg, expected_err_msg) def test_check_description_with_punctuation_in_the_end(self): @@ -205,11 +184,13 @@ class TestValidadeFormat(unittest.TestCase): with self.subTest(): err_msgs = check_description(0, desc) + self.assertIsInstance(err_msgs, list) self.assertEqual(len(err_msgs), 1) err_msg = err_msgs[0] expected_err_msg = f'(L001) description should not end with {desc[-1]}' + self.assertIsInstance(err_msg, str) self.assertEqual(err_msg, expected_err_msg) def test_check_description_that_exceeds_the_character_limit(self): @@ -218,11 +199,13 @@ class TestValidadeFormat(unittest.TestCase): err_msgs = check_description(0, long_desc) + self.assertIsInstance(err_msgs, list) self.assertEqual(len(err_msgs), 1) err_msg = err_msgs[0] expected_err_msg = f'(L001) description should not exceed {max_description_length} characters (currently {long_desc_length})' + self.assertIsInstance(err_msg, str) self.assertEqual(err_msg, expected_err_msg) def test_check_auth_return_type(self): From e3f51ae6c26a9b2cf88b18e5594eea5a8cec0575 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 06:13:58 -0300 Subject: [PATCH 58/79] Fix tests of check_auth --- scripts/tests/test_validate_format.py | 28 +++------------------------ 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/scripts/tests/test_validate_format.py b/scripts/tests/test_validate_format.py index e5506614..fd634afe 100644 --- a/scripts/tests/test_validate_format.py +++ b/scripts/tests/test_validate_format.py @@ -208,31 +208,6 @@ class TestValidadeFormat(unittest.TestCase): self.assertIsInstance(err_msg, str) self.assertEqual(err_msg, expected_err_msg) - def test_check_auth_return_type(self): - auth_with_backtick = [f'`{auth}`' for auth in auth_keys if auth != 'No'] - auth_with_backtick.append('No') - auth_without_backtick = [auth for auth in auth_keys if auth != 'No'] - auth_invalid = ['Yes', 'yes', 'no', 'random', 'Unknown'] - - for auth in auth_with_backtick: - with self.subTest(): - result = check_auth(0, auth) - self.assertIsInstance(result, list) - - for auth in auth_without_backtick: - with self.subTest(): - err_msgs = check_auth(0, auth) - self.assertIsInstance(result, list) - err_msg = err_msgs[0] - self.assertIsInstance(err_msg, str) - - for auth in auth_invalid: - with self.subTest(): - err_msgs = check_auth(0, auth) - self.assertIsInstance(result, list) - err_msg = err_msgs[0] - self.assertIsInstance(err_msg, str) - def test_check_auth_with_correct_auth(self): auth_valid = [f'`{auth}`' for auth in auth_keys if auth != 'No'] auth_valid.append('No') @@ -259,6 +234,7 @@ class TestValidadeFormat(unittest.TestCase): err_msg = err_msgs[0] expected_err_msg = '(L001) auth value is not enclosed with `backticks`' + self.assertIsInstance(err_msg, str) self.assertEqual(err_msg, expected_err_msg) def test_check_auth_with_invalid_auth(self): @@ -277,6 +253,8 @@ class TestValidadeFormat(unittest.TestCase): expected_err_msg_1 = f'(L001) auth value is not enclosed with `backticks`' expected_err_msg_2 = f'(L001) {auth} is not a valid Auth option' + self.assertIsInstance(err_msg_1, str) + self.assertIsInstance(err_msg_2, str) self.assertEqual(err_msg_1, expected_err_msg_1) self.assertEqual(err_msg_2, expected_err_msg_2) From 89e254c1412bfcf590fc63935c03e1c1d34bfd14 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 06:16:23 -0300 Subject: [PATCH 59/79] Fix test title of check_auth --- scripts/tests/test_validate_format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/test_validate_format.py b/scripts/tests/test_validate_format.py index fd634afe..81d28b87 100644 --- a/scripts/tests/test_validate_format.py +++ b/scripts/tests/test_validate_format.py @@ -208,7 +208,7 @@ class TestValidadeFormat(unittest.TestCase): self.assertIsInstance(err_msg, str) self.assertEqual(err_msg, expected_err_msg) - def test_check_auth_with_correct_auth(self): + def test_check_auth_with_valid_auth(self): auth_valid = [f'`{auth}`' for auth in auth_keys if auth != 'No'] auth_valid.append('No') From ad92d5c400e61195873742b9a2cd9cabc90d5d60 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 06:19:41 -0300 Subject: [PATCH 60/79] Remove whitespaces --- scripts/tests/test_validate_format.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/tests/test_validate_format.py b/scripts/tests/test_validate_format.py index 81d28b87..d3faf423 100644 --- a/scripts/tests/test_validate_format.py +++ b/scripts/tests/test_validate_format.py @@ -230,7 +230,7 @@ class TestValidadeFormat(unittest.TestCase): self.assertIsInstance(err_msgs, list) self.assertEqual(len(err_msgs), 1) - + err_msg = err_msgs[0] expected_err_msg = '(L001) auth value is not enclosed with `backticks`' @@ -246,7 +246,7 @@ class TestValidadeFormat(unittest.TestCase): self.assertIsInstance(err_msgs, list) self.assertEqual(len(err_msgs), 2) - + err_msg_1 = err_msgs[0] err_msg_2 = err_msgs[1] From 54787556381dc79628dc026d2015c26614375806 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 14:57:21 -0300 Subject: [PATCH 61/79] Test to check_auth with invalid auth with and without backtick --- scripts/tests/test_validate_format.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/scripts/tests/test_validate_format.py b/scripts/tests/test_validate_format.py index d3faf423..5821246a 100644 --- a/scripts/tests/test_validate_format.py +++ b/scripts/tests/test_validate_format.py @@ -238,9 +238,10 @@ class TestValidadeFormat(unittest.TestCase): self.assertEqual(err_msg, expected_err_msg) def test_check_auth_with_invalid_auth(self): - auth_invalid = ['Yes', 'yes', 'no', 'random', 'Unknown'] + auth_invalid_without_backtick = ['Yes', 'yes', 'no', 'random', 'Unknown'] + auth_invalid_with_backtick = ['`Yes`', '`yes`', '`no`', '`random`', '`Unknown`'] - for auth in auth_invalid: + for auth in auth_invalid_without_backtick: with self.subTest(): err_msgs = check_auth(0, auth) self.assertIsInstance(err_msgs, list) @@ -258,6 +259,19 @@ class TestValidadeFormat(unittest.TestCase): self.assertEqual(err_msg_1, expected_err_msg_1) self.assertEqual(err_msg_2, expected_err_msg_2) + for auth in auth_invalid_with_backtick: + with self.subTest(): + err_msgs = check_auth(0, auth) + self.assertIsInstance(err_msgs, list) + + self.assertEqual(len(err_msgs), 1) + + err_msg = err_msgs[0] + expected_err_msg = f'(L001) {auth} is not a valid Auth option' + + self.assertIsInstance(err_msg, str) + self.assertEqual(err_msg, expected_err_msg) + def test_check_https_with_valid_https(self): for https in https_keys: with self.subTest(): From eeaa7c073e6097fed589532563134b72fbca2098 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 15:19:36 -0300 Subject: [PATCH 62/79] Create tests to check_entry --- scripts/tests/test_validate_format.py | 31 +++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/scripts/tests/test_validate_format.py b/scripts/tests/test_validate_format.py index 5821246a..c0ed621b 100644 --- a/scripts/tests/test_validate_format.py +++ b/scripts/tests/test_validate_format.py @@ -10,6 +10,7 @@ from validate.format import check_description, max_description_length from validate.format import check_auth, auth_keys from validate.format import check_https, https_keys from validate.format import check_cors, cors_keys +from validate.format import check_entry class TestValidadeFormat(unittest.TestCase): @@ -323,3 +324,33 @@ class TestValidadeFormat(unittest.TestCase): self.assertIsInstance(err_msg, str) self.assertEqual(err_msg, expected_err_msg) + + def test_check_entry_with_correct_segments(self): + correct_segments = ['[A](https://www.ex.com)', 'Desc', '`apiKey`', 'Yes', 'Yes'] + + err_msgs = check_entry(0, correct_segments) + + self.assertIsInstance(err_msgs, list) + self.assertEqual(len(err_msgs), 0) + self.assertEqual(err_msgs, []) + + def test_check_entry_with_incorrect_segments(self): + incorrect_segments = ['[A API](https://www.ex.com)', 'desc.', 'yes', 'yes', 'yes'] + + err_msgs = check_entry(0, incorrect_segments) + expected_err_msgs = [ + '(L001) Title should not end with "... API". Every entry is an API here!', + '(L001) first character of description is not capitalized', + '(L001) description should not end with .', + '(L001) auth value is not enclosed with `backticks`', + '(L001) yes is not a valid Auth option', + '(L001) yes is not a valid HTTPS option', + '(L001) yes is not a valid CORS option' + ] + + self.assertIsInstance(err_msgs, list) + self.assertEqual(len(err_msgs), 7) + for err_msg in err_msgs: + with self.subTest(): + self.assertIsInstance(err_msg, str) + self.assertEqual(err_msgs, expected_err_msgs) From e0468d13cb612dba6573863e36280d9ca827fea0 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 15:21:48 -0300 Subject: [PATCH 63/79] Remove unnecessary whitespace --- scripts/tests/test_validate_format.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/scripts/tests/test_validate_format.py b/scripts/tests/test_validate_format.py index c0ed621b..906676b4 100644 --- a/scripts/tests/test_validate_format.py +++ b/scripts/tests/test_validate_format.py @@ -158,7 +158,6 @@ class TestValidadeFormat(unittest.TestCase): self.assertIsInstance(err_msgs, list) self.assertEqual(len(err_msgs), 0) - self.assertEqual(err_msgs, []) def test_check_description_with_first_char_is_not_capitalized(self): @@ -217,9 +216,7 @@ class TestValidadeFormat(unittest.TestCase): with self.subTest(): err_msgs = check_auth(0, auth) self.assertIsInstance(err_msgs, list) - self.assertEqual(len(err_msgs), 0) - self.assertEqual(err_msgs, []) def test_check_auth_without_backtick(self): @@ -229,7 +226,6 @@ class TestValidadeFormat(unittest.TestCase): with self.subTest(): err_msgs = check_auth(0, auth) self.assertIsInstance(err_msgs, list) - self.assertEqual(len(err_msgs), 1) err_msg = err_msgs[0] @@ -246,7 +242,6 @@ class TestValidadeFormat(unittest.TestCase): with self.subTest(): err_msgs = check_auth(0, auth) self.assertIsInstance(err_msgs, list) - self.assertEqual(len(err_msgs), 2) err_msg_1 = err_msgs[0] @@ -264,7 +259,6 @@ class TestValidadeFormat(unittest.TestCase): with self.subTest(): err_msgs = check_auth(0, auth) self.assertIsInstance(err_msgs, list) - self.assertEqual(len(err_msgs), 1) err_msg = err_msgs[0] @@ -278,9 +272,7 @@ class TestValidadeFormat(unittest.TestCase): with self.subTest(): err_msgs = check_https(0, https) self.assertIsInstance(err_msgs, list) - self.assertEqual(len(err_msgs), 0) - self.assertEqual(err_msgs, []) def test_check_https_with_invalid_https(self): @@ -290,7 +282,6 @@ class TestValidadeFormat(unittest.TestCase): with self.subTest(): err_msgs = check_https(0, https) self.assertIsInstance(err_msgs, list) - self.assertEqual(len(err_msgs), 1) err_msg = err_msgs[0] @@ -304,9 +295,7 @@ class TestValidadeFormat(unittest.TestCase): with self.subTest(): err_msgs = check_cors(0, cors) self.assertIsInstance(err_msgs, list) - self.assertEqual(len(err_msgs), 0) - self.assertEqual(err_msgs, []) def test_check_cors_with_invalid_cors(self): @@ -316,7 +305,6 @@ class TestValidadeFormat(unittest.TestCase): with self.subTest(): err_msgs = check_cors(0, cors) self.assertIsInstance(err_msgs, list) - self.assertEqual(len(err_msgs), 1) err_msg = err_msgs[0] From 7d9eac49202db40f11bb7f20f3aafc4b4eec0667 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 16:07:21 -0300 Subject: [PATCH 64/79] Change sections -> columns in error message --- scripts/validate/format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/validate/format.py b/scripts/validate/format.py index 2d1f90cb..092595dc 100644 --- a/scripts/validate/format.py +++ b/scripts/validate/format.py @@ -234,7 +234,7 @@ def check_file_format(lines: List[str]) -> List[str]: num_in_category += 1 segments = line_content.split('|')[1:-1] if len(segments) < num_segments: - err_msg = error_message(line_num, f'entry does not have all the required sections (have {len(segments)}, need {num_segments})') + err_msg = error_message(line_num, f'entry does not have all the required columns (have {len(segments)}, need {num_segments})') err_msgs.append(err_msg) continue From bc02fa517f1f91a7f97bdfde775451b3d7eef840 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 16:19:39 -0300 Subject: [PATCH 65/79] Create tests to check_file_format --- scripts/tests/test_validate_format.py | 122 ++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/scripts/tests/test_validate_format.py b/scripts/tests/test_validate_format.py index 906676b4..7cf55604 100644 --- a/scripts/tests/test_validate_format.py +++ b/scripts/tests/test_validate_format.py @@ -11,6 +11,7 @@ from validate.format import check_auth, auth_keys from validate.format import check_https, https_keys from validate.format import check_cors, cors_keys from validate.format import check_entry +from validate.format import check_file_format, min_entries_per_category, num_segments class TestValidadeFormat(unittest.TestCase): @@ -342,3 +343,124 @@ class TestValidadeFormat(unittest.TestCase): with self.subTest(): self.assertIsInstance(err_msg, str) self.assertEqual(err_msgs, expected_err_msgs) + + def test_check_file_format_with_correct_format(self): + correct_format = [ + '## Index', + '* [A](#a)', + '* [B](#b)', + '', + '### A', + 'API | Description | Auth | HTTPS | CORS |', + '|---|---|---|---|---|', + '| [AA](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |', + '| [AB](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |', + '| [AC](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |', + '', + '### B', + 'API | Description | Auth | HTTPS | CORS |', + '|---|---|---|---|---|', + '| [BA](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |', + '| [BB](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |', + '| [BC](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |' + ] + + err_msgs = check_file_format(lines=correct_format) + + self.assertIsInstance(err_msgs, list) + self.assertEqual(len(err_msgs), 0) + self.assertEqual(err_msgs, []) + + def test_check_file_format_with_category_header_not_added_to_index(self): + incorrect_format = [ + '## Index', + '', + '### A', + 'API | Description | Auth | HTTPS | CORS |', + '|---|---|---|---|---|', + '| [AA](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |', + '| [AB](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |', + '| [AC](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |', + ] + + err_msgs = check_file_format(lines=incorrect_format) + expected_err_msg = '(L003) category header (A) not added to Index section' + + self.assertIsInstance(err_msgs, list) + self.assertEqual(len(err_msgs), 1) + err_msg = err_msgs[0] + self.assertEqual(err_msg, expected_err_msg) + + def test_check_file_format_with_category_without_min_entries(self): + incorrect_format = [ + '## Index', + '* [A](#a)', + '* [B](#b)', + '', + '### A', + 'API | Description | Auth | HTTPS | CORS |', + '|---|---|---|---|---|', + '| [AA](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |', + '', + '### B', + 'API | Description | Auth | HTTPS | CORS |', + '|---|---|---|---|---|', + '| [BA](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |', + '| [BB](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |', + '| [BC](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |' + ] + + category_with_err = 'A' + num_in_category = 1 + + err_msgs = check_file_format(lines=incorrect_format) + expected_err_msg = f'(L005) {category_with_err} category does not have the minimum {min_entries_per_category} entries (only has {num_in_category})' + + self.assertIsInstance(err_msgs, list) + self.assertEqual(len(err_msgs), 1) + err_msg = err_msgs[0] + self.assertEqual(err_msg, expected_err_msg) + + def test_check_file_format_entry_without_all_necessary_columns(self): + incorrect_format = [ + '## Index', + '* [A](#a)', + '', + '### A', + 'API | Description | Auth | HTTPS | CORS |', + '|---|---|---|---|---|', + '| [AA](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |', + '| [AB](https://www.ex.com) | Desc | `apiKey` |', # missing https and cors + '| [AC](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |', + ] + + current_segments_num = 3 + + err_msgs = check_file_format(lines=incorrect_format) + expected_err_msg = f'(L008) entry does not have all the required columns (have {current_segments_num}, need {num_segments})' + + self.assertIsInstance(err_msgs, list) + self.assertEqual(len(err_msgs), 1) + err_msg = err_msgs[0] + self.assertEqual(err_msg, expected_err_msg) + + def test_check_file_format_without_1_space_between_the_segments(self): + incorrect_format = [ + '## Index', + '* [A](#a)', + '', + '### A', + 'API | Description | Auth | HTTPS | CORS |', + '|---|---|---|---|---|', + '| [AA](https://www.ex.com) | Desc |`apiKey`| Yes | Yes |', # space between segment of auth column missing + '| [AB](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |', + '| [AC](https://www.ex.com) | Desc | `apiKey` | Yes | Yes |', + ] + + err_msgs = check_file_format(lines=incorrect_format) + expected_err_msg = f'(L007) each segment must start and end with exactly 1 space' + + self.assertIsInstance(err_msgs, list) + self.assertEqual(len(err_msgs), 1) + err_msg = err_msgs[0] + self.assertEqual(err_msg, expected_err_msg) From 9bb225ef3dba4b6713eeca94574b3b6827b36f01 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 18:18:39 -0300 Subject: [PATCH 66/79] Create new workflow to run test of push and pull --- .github/workflows/test.yml | 29 ----------- .github/workflows/test_of_push_and_pull.yml | 33 ++++++++++++ scripts/github_pull_request.sh | 57 +++++++++++++++++++++ 3 files changed, 90 insertions(+), 29 deletions(-) delete mode 100644 .github/workflows/test.yml create mode 100644 .github/workflows/test_of_push_and_pull.yml create mode 100644 scripts/github_pull_request.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 65e5f699..00000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: "Run tests" - -on: - schedule: - - cron: '0 0 * * *' - push: - branches: - - master - pull_request: - branches: - - master - -env: - FORMAT_FILE: README.md - -jobs: - test: - name: 'Validate README.md' - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Validate Markdown format - run: build/validate_format.py ${FORMAT_FILE} - - - name: Validate pull request changes - run: build/github-pull.sh ${{ github.repository }} ${{ github.event.pull_request.number }} ${FORMAT_FILE} - if: github.event_name == 'pull_request' diff --git a/.github/workflows/test_of_push_and_pull.yml b/.github/workflows/test_of_push_and_pull.yml new file mode 100644 index 00000000..32bdc912 --- /dev/null +++ b/.github/workflows/test_of_push_and_pull.yml @@ -0,0 +1,33 @@ +name: "Tests of push & pull" + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +env: + FILENAME: README.md + +jobs: + tests: + name: 'Validate README.md' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + + - name: Install dependencies + run: python -m pip install -r scripts/requirements.txt + + - name: Validate Markdown format + run: python scripts/validate/format.py ${FILENAME} + + - name: Validate pull request changes + run: scripts/github_pull_request.sh ${{ github.repository }} ${{ github.event.pull_request.number }} ${FILENAME} + if: github.event_name == 'pull_request' diff --git a/scripts/github_pull_request.sh b/scripts/github_pull_request.sh new file mode 100644 index 00000000..7dd7fa0f --- /dev/null +++ b/scripts/github_pull_request.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +set -e + +# Argument validation +if [ $# -ne 3 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Assign variables +GITHUB_REPOSITORY="$1" +GITHUB_PULL_REQUEST="$2" +FILENAME="$3" + +# Move to root of project +cd "$GITHUB_WORKSPACE" + +# Determine files +FILENAME="$( realpath "${FILENAME}" )" + +# Skip if build number could not be determined +if [ -z "$GITHUB_REPOSITORY" -o -z "$GITHUB_PULL_REQUEST" ]; then + echo "No pull request and/or repository is provided" + exit 1 +fi + +# Pull changes on PR +echo "running on Pull Request #$GITHUB_PULL_REQUEST" + +# Trick the URL validator python script into not seeing this as a URL +DUMMY_SCHEME="https" +DIFF_URL="$DUMMY_SCHEME://patch-diff.githubusercontent.com/raw/$GITHUB_REPOSITORY/pull/$GITHUB_PULL_REQUEST.diff" +curl -L "$DIFF_URL" -o diff.txt + +# Construct diff +echo "------- BEGIN DIFF -------" +cat diff.txt +echo "-------- END DIFF --------" +cat diff.txt | egrep "\+" > additions.txt + +echo "------ BEGIN ADDITIONS -----" +cat additions.txt +echo "------- END ADDITIONS ------" +LINK_FILE=additions.txt + +# Validate links +echo "Running link validation on additions..." +python scripts/validate/links.py "$LINK_FILE" + +# Vebosity +if [[ $? != 0 ]]; then + echo "link validation failed on additions!" + exit 1 +else + echo "link validation passed on additions!" +fi From 9e790e5b467fa3bc0f661a62d45ce2fcbfedf3a3 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 18:23:39 -0300 Subject: [PATCH 67/79] Remove old scripts --- build/github-pull.sh | 57 ------------- build/requirements.txt | 2 - build/validate_format.py | 168 --------------------------------------- build/validate_links.py | 100 ----------------------- 4 files changed, 327 deletions(-) delete mode 100755 build/github-pull.sh delete mode 100644 build/requirements.txt delete mode 100755 build/validate_format.py delete mode 100755 build/validate_links.py diff --git a/build/github-pull.sh b/build/github-pull.sh deleted file mode 100755 index 4c5c3a4b..00000000 --- a/build/github-pull.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash - -set -e - -# Argument validation -if [ $# -ne 3 ]; then - echo "Usage: $0 " - exit 1 -fi - -# Assign variables -GITHUB_REPOSITORY="$1" -GITHUB_PULL_REQUEST="$2" -FORMAT_FILE="$3" - -# Move to root of project -cd "$GITHUB_WORKSPACE" - -# Determine files -FORMAT_FILE="$( realpath "${FORMAT_FILE}" )" - -# Skip if build number could not be determined -if [ -z "$GITHUB_REPOSITORY" -o -z "$GITHUB_PULL_REQUEST" ]; then - echo "No pull request and/or repository is provided" - exit 1 -fi - -# Pull changes on PR -echo "running on Pull Request #$GITHUB_PULL_REQUEST" - -# Trick the URL validator python script into not seeing this as a URL -DUMMY_SCHEME="https" -DIFF_URL="$DUMMY_SCHEME://patch-diff.githubusercontent.com/raw/$GITHUB_REPOSITORY/pull/$GITHUB_PULL_REQUEST.diff" -curl -L -o diff.txt "$DIFF_URL" - -# Construct diff -echo "------- BEGIN DIFF -------" -cat diff.txt -echo "-------- END DIFF --------" -cat diff.txt | egrep "\+" > additions.txt - -echo "------ BEGIN ADDITIONS -----" -cat additions.txt -echo "------- END ADDITIONS ------" -LINK_FILE=additions.txt - -# Validate links -echo "Running link validation..." -./build/validate_links.py "$LINK_FILE" - -# Vebosity -if [[ $? != 0 ]]; then - echo "link validation failed!" - exit 1 -else - echo "link validation passed!" -fi diff --git a/build/requirements.txt b/build/requirements.txt deleted file mode 100644 index ebed23a8..00000000 --- a/build/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -flake8>=3.5.0 -httplib2==0.19.0 diff --git a/build/validate_format.py b/build/validate_format.py deleted file mode 100755 index 4610f379..00000000 --- a/build/validate_format.py +++ /dev/null @@ -1,168 +0,0 @@ -#!/usr/bin/env python3 - -import re -import sys - -anchor = '###' -min_entries_per_section = 3 -auth_keys = ['apiKey', 'OAuth', 'X-Mashape-Key', 'No', 'User-Agent'] -punctuation = ['.', '?', '!'] -https_keys = ['Yes', 'No'] -cors_keys = ['Yes', 'No', 'Unknown'] - -index_title = 0 -index_desc = 1 -index_auth = 2 -index_https = 3 -index_cors = 4 -index_link = 5 -num_segments = 5 - -errors = [] -title_links = [] -anchor_re = re.compile(anchor + '\s(.+)') -section_title_re = re.compile('\*\s\[(.*)\]') -link_re = re.compile('\[(.+)\]\((http.*)\)') - - -def add_error(line_num, message): - """adds an error to the dynamic error list""" - err = '(L{:03d}) {}'.format(line_num + 1, message) - errors.append(err) - - -def check_alphabetical(lines): - """ - checks if all entries per section are in alphabetical order based in entry title - """ - sections = {} - section_line_num = {} - for line_num, line in enumerate(lines): - if line.startswith(anchor): - category = line.split(anchor)[1].strip() - sections[category] = [] - section_line_num[category] = line_num - continue - if not line.startswith('|') or line.startswith('|---'): - continue - raw_title = [x.strip() for x in line.split('|')[1:-1]][0] - title_re_match = link_re.match(raw_title) - if title_re_match: - sections[category].append(title_re_match.group(1).upper()) - - for category, entries in sections.items(): - if sorted(entries) != entries: - add_error(section_line_num[category], "{} section is not in alphabetical order".format(category)) - - -def check_entry(line_num, segments): - # START Title - raw_title = segments[index_title] - title_re_match = link_re.match(raw_title) - # url should be wrapped in '[TITLE](LINK)' Markdown syntax - if not title_re_match: - add_error(line_num, 'Title syntax should be "[TITLE](LINK)"') - else: - # do not allow "... API" in the entry title - title = title_re_match.group(1) - if title.upper().endswith(' API'): - add_error(line_num, 'Title should not end with "... API". Every entry is an API here!') - # END Title - # START Description - # first character should be capitalized - char = segments[index_desc][0] - if char.upper() != char: - add_error(line_num, "first character of description is not capitalized") - # last character should not punctuation - char = segments[index_desc][-1] - if char in punctuation: - add_error(line_num, "description should not end with {}".format(char)) - desc_length = len(segments[index_desc]) - if desc_length > 100: - add_error(line_num, "description should not exceed 100 characters (currently {})".format(desc_length)) - # END Description - # START Auth - # values should conform to valid options only - auth = segments[index_auth] - if auth != 'No' and (not auth.startswith('`') or not auth.endswith('`')): - add_error(line_num, "auth value is not enclosed with `backticks`") - if auth.replace('`', '') not in auth_keys: - add_error(line_num, "{} is not a valid Auth option".format(auth)) - # END Auth - # START HTTPS - # values should conform to valid options only - https = segments[index_https] - if https not in https_keys: - add_error(line_num, "{} is not a valid HTTPS option".format(https)) - # END HTTPS - # START CORS - # values should conform to valid options only - cors = segments[index_cors] - if cors not in cors_keys: - add_error(line_num, "{} is not a valid CORS option".format(cors)) - # END CORS - - -def check_format(filename): - """ - validates that each line is formatted correctly, - appending to error list as needed - """ - with open(filename) as fp: - lines = list(line.rstrip() for line in fp) - check_alphabetical(lines) - # START Check Entries - num_in_category = min_entries_per_section + 1 - category = "" - category_line = 0 - for line_num, line in enumerate(lines): - if section_title_re.match(line): - title_links.append(section_title_re.match(line).group(1)) - # check each section for the minimum number of entries - if line.startswith(anchor): - match = anchor_re.match(line) - if match: - if match.group(1) not in title_links: - add_error(line_num, "section header ({}) not added as a title link".format(match.group(1))) - else: - add_error(line_num, "section header is not formatted correctly") - if num_in_category < min_entries_per_section: - add_error(category_line, "{} section does not have the minimum {} entries (only has {})".format( - category, min_entries_per_section, num_in_category)) - category = line.split(' ')[1] - category_line = line_num - num_in_category = 0 - continue - # skips lines that we do not care about - if not line.startswith('|') or line.startswith('|---'): - continue - num_in_category += 1 - segments = line.split('|')[1:-1] - if len(segments) < num_segments: - add_error(line_num, "entry does not have all the required sections (have {}, need {})".format( - len(segments), num_segments)) - continue - # START Global - for segment in segments: - # every line segment should start and end with exactly 1 space - if len(segment) - len(segment.lstrip()) != 1 or len(segment) - len(segment.rstrip()) != 1: - add_error(line_num, "each segment must start and end with exactly 1 space") - # END Global - segments = [seg.strip() for seg in segments] - check_entry(line_num, segments) - # END Check Entries - - -def main(): - if len(sys.argv) < 2: - print("No file passed (file should contain Markdown table syntax)") - sys.exit(1) - check_format(sys.argv[1]) - if len(errors) > 0: - for err in errors: - print(err) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/build/validate_links.py b/build/validate_links.py deleted file mode 100755 index c0331d4d..00000000 --- a/build/validate_links.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python3 - -import httplib2 -import re -import socket -import sys - - -def parse_links(filename): - """Returns a list of URLs from text file""" - with open(filename, mode='r', encoding='utf-8') as fp: - readme = fp.read() - index_section = readme.find('## Index') - content = readme[index_section:] - - raw_links = re.findall( - '((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:\'\".,<>?«»“”‘’]))', - content) - - links = [ - str(raw_link[0]).rstrip('/') for raw_link in raw_links - ] - - return links - -def dup_links(links): - """Check for duplicated links""" - print(f'Checking for duplicated links...') - hasError = False - seen = {} - dupes = [] - - for link in links: - if link not in seen: - seen[link] = 1 - else: - if seen[link] == 1: - dupes.append(link) - - if not dupes: - print(f"No duplicate links") - else: - print(f"Found duplicate links: {dupes}") - hasError = True - return hasError - -def validate_links(links): - """Checks each entry in JSON file for live link""" - print(f'Validating {len(links)} links...') - hasError = False - for link in links: - h = httplib2.Http(disable_ssl_certificate_validation=True, timeout=25) - try: - # fetching host name, removing leading www - host = link.split('//', 1)[1].split('/', 1)[0] - if host[:3] == 'www': - host = host[4:] - - resp = h.request(link + "/", headers={ - # Faking user agent as some hosting services block not-whitelisted UA - 'User-Agent': 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1467.0 Safari/537.36', - # setting host because Cloudflare returns 403 asking for captcha if host is missing - 'host': host - }) - code = int(resp[0]['status']) - # Checking status code errors - if (code >= 400): - hasError = True - print(f"ERR:CLT:{code} : {link}") - except TimeoutError: - hasError = True - print(f"ERR:TMO: {link}") - except socket.error as socketerror: - hasError = True - print(f"ERR:SOC: {socketerror} : {link}") - except Exception as e: - hasError = True - # Ignore some exceptions which are not actually errors. - # The list below should be extended with other exceptions in the future if needed - if (-1 != str(e).find("[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:852)")): - print(f"ERR:SSL: {e} : {link}") - elif (-1 != str(e).find("Content purported to be compressed with gzip but failed to decompress.")): - print(f"ERR:GZP: {e} : {link}") - elif (-1 != str(e).find("Unable to find the server at")): - print(f"ERR:SRV: {e} : {link}") - else: - print(f"ERR:UKN: {e} : {link}") - return hasError - -if __name__ == "__main__": - num_args = len(sys.argv) - if num_args < 2: - print("No .md file passed") - sys.exit(1) - links = parse_links(sys.argv[1]) - hasError = dup_links(links) - if not hasError: - hasError = validate_links(links) - if hasError: - sys.exit(1) From 75e1d2e44f25282149603efce84132e65ce89ab0 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 18:51:08 -0300 Subject: [PATCH 68/79] Update validate_links action --- .github/workflows/validate_links.yml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/validate_links.yml b/.github/workflows/validate_links.yml index 92afb843..229a60b0 100644 --- a/.github/workflows/validate_links.yml +++ b/.github/workflows/validate_links.yml @@ -3,20 +3,25 @@ name: "Validate links" on: schedule: - cron: '0 0 * * *' - push: - branches: - - master env: FORMAT_FILE: README.md jobs: - test: - name: 'Validate links' + validate_links: + name: 'Check all links are working' runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + + - name: Install dependencies + run: python -m pip install -r scripts/requirements.txt + - name: Validate all links from README.md - run: build/validate_links.py ${FORMAT_FILE} + run: python scripts/validate/links.py ${FORMAT_FILE} From be086f293fa3af3b57b74bdfbf93e1799a342693 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 18:55:21 -0300 Subject: [PATCH 69/79] Update name section in workflow --- .github/workflows/test_of_push_and_pull.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_of_push_and_pull.yml b/.github/workflows/test_of_push_and_pull.yml index 32bdc912..29798b9c 100644 --- a/.github/workflows/test_of_push_and_pull.yml +++ b/.github/workflows/test_of_push_and_pull.yml @@ -11,7 +11,7 @@ env: jobs: tests: - name: 'Validate README.md' + name: 'Validate README.md changes' runs-on: ubuntu-latest steps: - name: Checkout repository From 46279792f282abfd39d4fac07e1fc43a27240eb4 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 19:14:39 -0300 Subject: [PATCH 70/79] Create test_of_validate_package action --- .../workflows/test_of_validate_package.yml | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/test_of_validate_package.yml diff --git a/.github/workflows/test_of_validate_package.yml b/.github/workflows/test_of_validate_package.yml new file mode 100644 index 00000000..700a3035 --- /dev/null +++ b/.github/workflows/test_of_validate_package.yml @@ -0,0 +1,26 @@ +name: "Tests of validate package" + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + unittest: + name: 'Run tests of validate package' + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + + - name: Run Unittest + run: | + cd scripts + python -m unittest discover tests/ --verbose From ff31b266ab67b6f16826abf8ea7d9a9aec164bbb Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 19:24:45 -0300 Subject: [PATCH 71/79] Add format and links in validate scope --- scripts/validate/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/validate/__init__.py b/scripts/validate/__init__.py index 40a96afc..4287dcc8 100644 --- a/scripts/validate/__init__.py +++ b/scripts/validate/__init__.py @@ -1 +1,4 @@ # -*- coding: utf-8 -*- + +from validate import format +from validate import links From 3a08c1518f9ec510bdc5ce7428bf3fc8f81049ed Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 19:27:14 -0300 Subject: [PATCH 72/79] Change FORMAT_FILE -> FILENAME in validate_links action --- .github/workflows/validate_links.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate_links.yml b/.github/workflows/validate_links.yml index 229a60b0..4f14df1c 100644 --- a/.github/workflows/validate_links.yml +++ b/.github/workflows/validate_links.yml @@ -5,7 +5,7 @@ on: - cron: '0 0 * * *' env: - FORMAT_FILE: README.md + FILENAME: README.md jobs: validate_links: @@ -24,4 +24,4 @@ jobs: run: python -m pip install -r scripts/requirements.txt - name: Validate all links from README.md - run: python scripts/validate/links.py ${FORMAT_FILE} + run: python scripts/validate/links.py ${FILENAME} From 08ba4e64582a77a267c6aa30826ae6a2ba90b47c Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 20:04:31 -0300 Subject: [PATCH 73/79] Create README.md to scripts directory --- scripts/README.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 scripts/README.md diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000..951c91d2 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,45 @@ +# Public APIs Scripts + +This directory contains all validation and testing scripts used by Public APIs. + +```bash +scripts +│ github_pull_request.sh # used to validate changes of a pull request +│ requirements.txt # contains dependencies of validate package +│ +├───tests # contains all unit tests from the validate package +│ test_validate_format.py +│ test_validate_links.py +│ +└───validate # validate package + format.py + links.py +``` + +## Running Tests + +To run all tests it is necessary to change to the scripts directory: + +```bash +$ cd scripts +``` + +then run: + +```bash +$ python -m unittest discover tests/ --verbose +``` + +To run only the format tests, run: + +```bash +$ python -m unittest discover tests/ --verbose --pattern "test_validate_format.py" +``` + +To run only the links tests, run: + +```bash +$ python -m unittest discover tests/ --verbose --pattern "test_validate_links.py" +``` + +*Note that it is necessary to have [python](https://www.python.org/) installed.* From 1c3e4eba3784a28a74de185d5af2b7072e2f18b3 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 20:18:54 -0300 Subject: [PATCH 74/79] Create run validations section --- scripts/README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/README.md b/scripts/README.md index 951c91d2..3bcacdc8 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -16,6 +16,20 @@ scripts links.py ``` +## Run validations + +To run format validation on the `README.md` file, being in the root directory of public-apis, run: + +```bash +$ python scripts/validate/format.py README.md +``` + +To run link validation on the `README.md` file, being in the root directory of public-apis, run: + +```bash +$ python scripts/validate/links.py README.md +``` + ## Running Tests To run all tests it is necessary to change to the scripts directory: From fb70084377dc05dc54532ea6a7f3835573d4d15f Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Mon, 17 Jan 2022 20:25:30 -0300 Subject: [PATCH 75/79] Create install dependencies section --- scripts/README.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index 3bcacdc8..eae70ff9 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -16,6 +16,16 @@ scripts links.py ``` +## Install dependencies + +You must have [python](https://www.python.org/) installed to use these scripts. + +it is also necessary to install the validation package dependencies, use [pip package manager](https://pypi.org/project/pip/) for this: + +```bash +$ python -m pip install -r scripts/requirements.txt +``` + ## Run validations To run format validation on the `README.md` file, being in the root directory of public-apis, run: @@ -55,5 +65,3 @@ To run only the links tests, run: ```bash $ python -m unittest discover tests/ --verbose --pattern "test_validate_links.py" ``` - -*Note that it is necessary to have [python](https://www.python.org/) installed.* From 4bc90a81ccf327f94b2ef813d572a7778b628bdc Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Tue, 18 Jan 2022 02:06:21 -0300 Subject: [PATCH 76/79] Add possibility to run duplicate link check only Changes: - It is now possible to only check for duplicate links (without checking if the links are working) - Check for duplicate links when a push occurs - Update the scripts documentation accordingly --- .github/workflows/test_of_push_and_pull.yml | 4 ++++ scripts/README.md | 8 ++++++++ scripts/validate/links.py | 17 ++++++++++++++--- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_of_push_and_pull.yml b/.github/workflows/test_of_push_and_pull.yml index 29798b9c..4ee1316f 100644 --- a/.github/workflows/test_of_push_and_pull.yml +++ b/.github/workflows/test_of_push_and_pull.yml @@ -31,3 +31,7 @@ jobs: - name: Validate pull request changes run: scripts/github_pull_request.sh ${{ github.repository }} ${{ github.event.pull_request.number }} ${FILENAME} if: github.event_name == 'pull_request' + + - name: Checking if push changes are duplicated + run: python scripts/validate/links.py ${FILENAME} --only_duplicate_links_checker + if: github.event_name == 'push' diff --git a/scripts/README.md b/scripts/README.md index eae70ff9..2e926a37 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -40,6 +40,14 @@ To run link validation on the `README.md` file, being in the root directory of p $ python scripts/validate/links.py README.md ``` +As there are many links to check, this process can take some time. If your goal is not to check if the links are working, you can only check for duplicate links. Run: + +```bash +$ python scripts/validate/links.py README.md -odlc +``` + +*`-odlc` is an abbreviation of `--only_duplicate_links_checker`* + ## Running Tests To run all tests it is necessary to change to the scripts directory: diff --git a/scripts/validate/links.py b/scripts/validate/links.py index e727f03d..a5172b8b 100644 --- a/scripts/validate/links.py +++ b/scripts/validate/links.py @@ -241,21 +241,32 @@ def start_links_working_checker(links: List[str]) -> None: sys.exit(1) -def main(filename: str) -> None: +def main(filename: str, only_duplicate_links_checker: bool) -> None: links = find_links_in_file(filename) start_duplicate_links_checker(links) - start_links_working_checker(links) + + if not only_duplicate_links_checker: + start_links_working_checker(links) if __name__ == '__main__': num_args = len(sys.argv) + only_duplicate_links_checker = False if num_args < 2: print('No .md file passed') sys.exit(1) + elif num_args == 3: + third_arg = sys.argv[2].lower() + + if third_arg == '-odlc' or third_arg == '--only_duplicate_links_checker': + only_duplicate_links_checker = True + else: + print(f'Third invalid argument. Usage: python {__file__} [-odlc | --only_duplicate_links_checker]') + sys.exit(1) filename = sys.argv[1] - main(filename) + main(filename, only_duplicate_links_checker) From 434c9fc292abf74b8aeca5c85200b3f77f38a20e Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Tue, 18 Jan 2022 03:10:27 -0300 Subject: [PATCH 77/79] Install dependencies in action --- .github/workflows/test_of_validate_package.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test_of_validate_package.yml b/.github/workflows/test_of_validate_package.yml index 700a3035..c41c886c 100644 --- a/.github/workflows/test_of_validate_package.yml +++ b/.github/workflows/test_of_validate_package.yml @@ -19,6 +19,9 @@ jobs: uses: actions/setup-python@v2 with: python-version: '3.8' + + - name: Install dependencies + run: python -m pip install -r scripts/requirements.txt - name: Run Unittest run: | From 183a8ece0631c3193b628a67fef890aa4587d256 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Tue, 18 Jan 2022 03:16:11 -0300 Subject: [PATCH 78/79] Fix yaml syntax --- .github/workflows/test_of_validate_package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_of_validate_package.yml b/.github/workflows/test_of_validate_package.yml index c41c886c..f739df81 100644 --- a/.github/workflows/test_of_validate_package.yml +++ b/.github/workflows/test_of_validate_package.yml @@ -21,7 +21,7 @@ jobs: python-version: '3.8' - name: Install dependencies - run: python -m pip install -r scripts/requirements.txt + run: python -m pip install -r scripts/requirements.txt - name: Run Unittest run: | From 6d4e269f126567e466b0ad9f7395448742aaf313 Mon Sep 17 00:00:00 2001 From: Matheus Felipe <50463866+matheusfelipeog@users.noreply.github.com> Date: Tue, 18 Jan 2022 03:30:13 -0300 Subject: [PATCH 79/79] Modify execute permission --- scripts/github_pull_request.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/github_pull_request.sh diff --git a/scripts/github_pull_request.sh b/scripts/github_pull_request.sh old mode 100644 new mode 100755