Add/asyncio aiohttp acceleration #52
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| asyncio==3.4.3 | ||
| aiohttp==3.7.3 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,5 @@ | ||
| [DEFAULT] | ||
| git_path_test_value = https://github.com/urlstechie/urlchecker-test-repo | ||
| file_types_test_values = .md,.c,.txt | ||
| exclude_test_urls = https://github.com/SuperKogito/URLs-checker/issues/2,https://github.com/SuperKogito/URLs-checker/issues/3 | ||
| exclude_test_patterns = https://github.com/SuperKogito/Voice-based-gender-recognition/issues,https://img.shields.io/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,17 +1,37 @@ | ||
| import os | ||
| import pytest | ||
| import argparse | ||
| import tempfile | ||
| import subprocess | ||
| import configparser | ||
| from urlchecker.client import check | ||
|
|
||
|
|
||
| def test_client_general(): | ||
| # excute scripts | ||
| pipe = subprocess.run( | ||
| ["urlchecker", "-h"], stdout=subprocess.PIPE, stderr=subprocess.PIPE | ||
| ) | ||
| assert pipe.stderr.decode("utf-8") == "" | ||
|
|
||
| pipe = subprocess.run( | ||
| ["urlchecker", "--help"], stdout=subprocess.PIPE, stderr=subprocess.PIPE | ||
| ) | ||
| assert pipe.stderr.decode("utf-8") == "" | ||
|
|
||
| pipe = subprocess.run( | ||
| ["urlchecker", "--version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE | ||
| ) | ||
| assert pipe.stderr.decode("utf-8") == "" | ||
|
|
||
|
|
||
| @pytest.mark.parametrize("config_fname", ["./tests/_local_test_config.conf"]) | ||
| @pytest.mark.parametrize("cleanup", [False, True]) | ||
| @pytest.mark.parametrize("print_all", [False, True]) | ||
| @pytest.mark.parametrize("force_pass", [False, True]) | ||
| @pytest.mark.parametrize("rcount", [1, 3]) | ||
| @pytest.mark.parametrize("timeout", [5, 7]) | ||
| def test_client_check(config_fname, cleanup, print_all, force_pass, rcount, timeout): | ||
|
|
||
| # init config parser | ||
| config = configparser.ConfigParser() | ||
|
|
@@ -101,3 +121,39 @@ def test_client_save(save): | |
| if save: | ||
| if not os.path.exists(output_csv.name): | ||
| raise AssertionError | ||
|
|
||
|
|
||
| @pytest.mark.parametrize("config_fname", ["./tests/_local_test_config.conf"]) | ||
| def test_client_check_main(config_fname): | ||
|
|
||
| # init config parser | ||
| config = configparser.ConfigParser() | ||
| config.read(config_fname) | ||
|
|
||
| # init env variables | ||
| path = config["DEFAULT"]["git_path_test_value"] | ||
| file_types = config["DEFAULT"]["file_types_test_values"] | ||
| exclude_urls = config["DEFAULT"]["exclude_test_urls"] | ||
| exclude_patterns = config["DEFAULT"]["exclude_test_patterns"] | ||
|
|
||
| # init args | ||
| args = argparse.Namespace() | ||
| args.path = path | ||
| args.branch = "master" | ||
| args.subfolder = "test_files" | ||
| args.cleanup = True | ||
| args.force_pass = True | ||
| args.no_print = True | ||
| args.file_types = file_types | ||
| args.files = "" | ||
| args.exclude_urls = "" | ||
| args.exclude_patterns = "" | ||
| args.exclude_files = "" | ||
| args.save = "" | ||
| args.retry_count = 1 | ||
| args.timeout = 5 | ||
|
|
||
| # excute script | ||
| with pytest.raises(SystemExit) as e: | ||
| check.main(args=args, extra=[]) | ||
| assert e.value.code == 0 | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -12,7 +12,7 @@ | |
| "file_paths", | ||
| [ | ||
| ["tests/test_files/sample_test_file.md"], | ||
| ["tests/test_files/sample_test_file.c"], | ||
| ["tests/test_files/sample_test_file.rst"], | ||
| ], | ||
| ) | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -31,9 +31,9 @@ def test_check_file_type(file_path, file_types): | |||||
|
|
||||||
| @pytest.mark.parametrize( | ||||||
| "file_path", | ||||||
| ["tests/test_files/sample_test_file.txt", "tests/test_files/sample_test_file.py"], | ||||||
| ) | ||||||
| @pytest.mark.parametrize("file_types", [[".txt", ".py"]]) | ||||||
| def test_check_file_type(file_path, file_types): | ||||||
| """ | ||||||
| test check file types | ||||||
|
|
@@ -53,18 +53,18 @@ def test_check_file_type(file_path, file_types): | |||||
| ["tests/test_files/sample_test_file.md", "tests/test_files/sample_test_file.py"], | ||||||
| ) | ||||||
| @pytest.mark.parametrize( | ||||||
| "exclude_patterns", [["[.py]"], ["[.md]"], ["tests/test_file"]] | ||||||
|
There was a problem hiding this comment.
Suggested change
There was a problem hiding this comment. can you explain this change to me please? There was a problem hiding this comment. Sure! So your current regular expressions are matching exactly .py. We usually only need brackets when we want an exact match of a character that could be a regular expression (in the example above, the period). So I moved the brackets around just the periods, left the letters as is, and added a $ to indicate we only want to match the end of the line (e.g., we wouldn't want to match filename.python-bindings or |
||||||
| ) | ||||||
| def test_include_files(file_path, exclude_patterns): | ||||||
| """ | ||||||
| test if a file should be included based on patterns (using extension for test) | ||||||
| """ | ||||||
| _, extension = os.path.splitext(file_path) | ||||||
| expected = not extension in file_path | ||||||
| result = include_file(file_path, exclude_patterns) | ||||||
|
|
||||||
| # No files should be included for a global path pattern | ||||||
| if "tests/test_file" in exclude_patterns: | ||||||
| if result: | ||||||
| raise AssertionError | ||||||
|
|
||||||
|
|
||||||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -61,10 +61,10 @@ def test_get_user_agent(): | |
|
|
||
| def test_check_response_status_code(): | ||
| class failedResponse: | ||
| status = 500 | ||
|
|
||
| class successResponse: | ||
| status = 200 | ||
|
|
||
| # Any failure returns True (indicating a retry is needed) | ||
| assert not check_response_status_code( | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,4 @@ | ||
| https://github.com/urlstechie/urlchecker-action | ||
| https://github.com/urlstechie/urlchecker-python | ||
| https://github.com/urlstechie/urlstechie.github.io | ||
| https://urlstechie.github.io/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| # This is a test file | ||
| include<stdio.h> | ||
|
|
||
| int main() { | ||
| printf("https://www.google.com/"); | ||
SuperKogito marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| printf("https://www.youtube.com/"); | ||
| printf("https://stackoverflow.com/"); | ||
| printf("https://github.com/"); | ||
| return 0; | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,14 +2,14 @@ | |
| The following is a list of test urls to extract. | ||
| - [test url 1](https://www.google.com/) | ||
| - [test url 2](https://github.com/SuperKogito) | ||
| - [test url 3](https://github.com/vsoch) | ||
| - [test url 4](https://github.com/SuperKogito/URLs-checker/blob/master/README.md) | ||
| - [test url 5](https://github.com/SuperKogito/URLs-checker/issues) | ||
| - [test url 6](https://travis-ci.com/github/urlstechie) | ||
|
|
||
| - [test url 7](https://github.com/SuperKogito/spafe/) | ||
| - [test url 8](https://codecov.io/gh/urlstechie) | ||
| - [test url 9](https://github.com/urlstechie/urlchecker-action) | ||
|
|
||
| - [test url 10](https://github.com/SuperKogito/Voice-based-gender-recognition) | ||
| - [test url 11](https://github.com/SuperKogito/Voice-based-gender-recognition/issues) | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -8,6 +8,7 @@ | |
| print("This is a test file with some URLs") | ||
| url1 = "https://www.google.com/" | ||
| url2 = "https://github.com/SuperKogito" | ||
| url3 = "https://github.com/vsoch" | ||
| url4 = "https://github.com/SuperKogito/URLs-checker/README.md" | ||
| url5 = {"url": "https://github.com/SuperKogito/URLs-checker/README.md"} | ||
| print("Done.") | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| # This is a test file | ||
| https://github.com/urlstechie | ||
| https://github.com/urlstechie/urlchecker-python | ||
| https://urlstechie.github.io/ | ||
| https://superkogito.github.io/blog/urlstechie.html | ||
| https://twitter.com | ||
| https://anaconda.org/conda-forge/urlchecker | ||
| https://urlchecker-python.readthedocs.io/en/latest/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -136,7 +136,7 @@ def get_parser(): | |||||
|
|
||||||
| check.add_argument( | ||||||
| "--timeout", | ||||||
| help="timeout (minutes) to provide to the aiohttp library (defaults to 5)", | ||||||
|
There was a problem hiding this comment.
Suggested change
|
||||||
| type=int, | ||||||
| default=5, | ||||||
| ) | ||||||
|
|
||||||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -48,7 +48,11 @@ def main(args, extra): | |
| sys.exit("Error %s does not exist." % path) | ||
|
|
||
| # Parse file types, and excluded urls and files (includes absolute and patterns) | ||
| file_types = [] | ||
|
There was a problem hiding this comment. Also, if the user puts a singe character expansion, this wouldn't be included since it's == 1. There was a problem hiding this comment. true, but as far as I know the shortest file type is 2 characters like There was a problem hiding this comment. Try doing There was a problem hiding this comment. yes but doing There was a problem hiding this comment. This issue seems separate from asyncio - should it be a separate PR? There was a problem hiding this comment. yes I can add it in a separate one, I only did it here because we were waiting on the asyncio code so I thought I ship them together. |
||
| if "," in args.file_types: | ||
| file_types = [ft for ft in args.file_types.split(",") if len(ft) > 1] | ||
| else: | ||
| file_types.append(args.file_types) | ||
| exclude_urls = remove_empty(args.exclude_urls.split(",")) | ||
| exclude_patterns = remove_empty(args.exclude_patterns.split(",")) | ||
| exclude_files = remove_empty(args.exclude_files.split(",")) | ||
|
|
||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This should be a version bump,up to 0.1.0 since we would be fundamentally changing the core library and making it un-usable for older python versions.