Source code for scylla.providers.http_proxy_provider
import re
from requests_html import HTML
from scylla.database import ProxyIP
from .base_provider import BaseProvider
[docs]class HttpProxyProvider(BaseProvider):
[docs] def parse(self, html: HTML) -> [ProxyIP]:
ip_list: [ProxyIP] = []
for ip_row in html.find('table.proxytbl tr'):
ip_element = ip_row.find('td:nth-child(1)', first=True)
port_element = ip_row.find('td:nth-child(2)', first=True)
try:
if ip_element and port_element:
port_str = re.search(r'//]]> (\d+)', port_element.text).group(1)
p = ProxyIP(ip=ip_element.text, port=port_str)
ip_list.append(p)
except AttributeError:
pass
return ip_list
[docs] def urls(self) -> [str]:
return [
'https://proxyhttp.net/free-list/proxy-anonymous-hide-ip-address/',
'https://proxyhttp.net/',
'https://proxyhttp.net/free-list/anonymous-server-hide-ip-address/2#proxylist',
]
[docs] @staticmethod
def should_render_js() -> bool:
return True