|
@@ -1,3 +1,5 @@
|
|
|
|
+import validators
|
|
|
|
+
|
|
from typing import Optional
|
|
from typing import Optional
|
|
from urllib.parse import urlparse
|
|
from urllib.parse import urlparse
|
|
|
|
|
|
@@ -10,6 +12,8 @@ def get_filtered_results(results, filter_list):
|
|
filtered_results = []
|
|
filtered_results = []
|
|
for result in results:
|
|
for result in results:
|
|
url = result.get("url") or result.get("link", "")
|
|
url = result.get("url") or result.get("link", "")
|
|
|
|
+ if not validators.url(url):
|
|
|
|
+ continue
|
|
domain = urlparse(url).netloc
|
|
domain = urlparse(url).netloc
|
|
if any(domain.endswith(filtered_domain) for filtered_domain in filter_list):
|
|
if any(domain.endswith(filtered_domain) for filtered_domain in filter_list):
|
|
filtered_results.append(result)
|
|
filtered_results.append(result)
|