Refactor UC Mode
This commit is contained in:
Michael Mintz
parent
0a4a49be87
commit
bdd383476d
|
|
@ -33,19 +33,8 @@ sys_plat = sys.platform
|
|||
|
||||
|
||||
class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
|
||||
"""
|
||||
Controls chromedriver to drive a browser.
|
||||
The driver gets downloaded automatically.
|
||||
|
||||
*** Methods ***
|
||||
---------------
|
||||
|
||||
* reconnect()
|
||||
This can be useful when sites use heavy detection methods:
|
||||
- Stops the chromedriver service that runs in the background.
|
||||
- Starts the chromedriver service that runs in the background.
|
||||
- Recreates the session.
|
||||
"""
|
||||
"""Controls chromedriver to drive a browser.
|
||||
The driver gets downloaded automatically."""
|
||||
_instances = set()
|
||||
session_id = None
|
||||
debug = False
|
||||
|
|
@ -66,7 +55,7 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
|
|||
suppress_welcome=True,
|
||||
use_subprocess=True,
|
||||
debug=False,
|
||||
**kw
|
||||
**kw,
|
||||
):
|
||||
"""
|
||||
Starts the Chrome service and creates a new instance of chromedriver.
|
||||
|
|
@ -216,7 +205,11 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
|
|||
language = locale.getlocale()[0].replace("_", "-")
|
||||
except Exception:
|
||||
pass
|
||||
if not language:
|
||||
if (
|
||||
not language
|
||||
or "English" in language
|
||||
or "United States" in language
|
||||
):
|
||||
language = "en-US"
|
||||
options.add_argument("--lang=%s" % language)
|
||||
if not options.binary_location:
|
||||
|
|
@ -245,9 +238,14 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
|
|||
import json
|
||||
|
||||
with open(
|
||||
os.path.join(user_data_dir, "Default/Preferences"),
|
||||
encoding="latin1",
|
||||
os.path.join(
|
||||
os.path.abspath(user_data_dir),
|
||||
"Default",
|
||||
"Preferences",
|
||||
),
|
||||
encoding="utf-8",
|
||||
mode="r+",
|
||||
errors="ignore",
|
||||
) as fs:
|
||||
config = json.load(fs)
|
||||
if (
|
||||
|
|
@ -405,6 +403,10 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
|
|||
return [PageElement(o) for o in retval]
|
||||
|
||||
def reconnect(self, timeout=0.1):
|
||||
"""This can be useful when sites use heavy detection methods:
|
||||
- Stops the chromedriver service that runs in the background.
|
||||
- Starts the chromedriver service that runs in the background.
|
||||
- Recreates the session. """
|
||||
try:
|
||||
self.service.stop()
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -74,7 +74,7 @@ class CDP:
|
|||
opentabs = [s for s in sessions if s["type"] == "page"]
|
||||
return self.post(self.endpoints["close"].format(id=opentabs[-1]["id"]))
|
||||
|
||||
async def send(self, method, params): # noqa
|
||||
async def send(self, method, params):
|
||||
import websockets
|
||||
|
||||
self._reqid += 1
|
||||
|
|
|
|||
|
|
@ -11,20 +11,17 @@ REGISTERED = []
|
|||
|
||||
|
||||
def start_detached(executable, *args):
|
||||
"""
|
||||
Starts a fully independent subprocess with no parent.
|
||||
"""Starts a fully independent subprocess with no parent.
|
||||
:param executable: executable
|
||||
:param args: arguments to the executable,
|
||||
eg: ["--param1_key=param1_val", "-vvv"]
|
||||
:return: pid of the grandchild process
|
||||
"""
|
||||
:return: pid of the grandchild process """
|
||||
import multiprocessing
|
||||
|
||||
reader, writer = multiprocessing.Pipe(False) # Create pipe
|
||||
# Do not keep reference
|
||||
multiprocessing.Process(
|
||||
target=_start_detached,
|
||||
args=(executable, *args), # noqa
|
||||
args=(executable, *args),
|
||||
kwargs={"writer": writer},
|
||||
daemon=True,
|
||||
).start()
|
||||
|
|
|
|||
|
|
@ -51,7 +51,9 @@ class ChromeOptions(ChromiumOptions):
|
|||
prefs_file = os.path.join(default_path, "Preferences")
|
||||
try:
|
||||
if os.path.exists(prefs_file):
|
||||
with open(prefs_file, encoding="utf-8", mode="r") as f:
|
||||
with open(
|
||||
prefs_file, encoding="utf-8", mode="r", errors="ignore"
|
||||
) as f:
|
||||
undot_prefs = self._merge_nested(
|
||||
json.load(f), undot_prefs
|
||||
)
|
||||
|
|
|
|||
|
|
@ -39,15 +39,13 @@ class Patcher(object):
|
|||
data_path = os.path.abspath(os.path.expanduser(downloads_folder))
|
||||
|
||||
def __init__(self, executable_path=None, force=False, version_main=0):
|
||||
"""
|
||||
Args:
|
||||
executable_path: None = automatic
|
||||
A full file path to the chromedriver executable
|
||||
force: False
|
||||
Terminate processes which are holding lock
|
||||
version_main: 0 = auto
|
||||
Specify main chrome version (rounded, ex: 82)
|
||||
"""
|
||||
"""Args:
|
||||
executable_path: None = automatic
|
||||
A full file path to the chromedriver executable
|
||||
force: False
|
||||
Terminate processes which are holding lock
|
||||
version_main: 0 = auto
|
||||
Specify main chrome version (rounded, ex: 82) """
|
||||
self.force = force
|
||||
self.executable_path = None
|
||||
prefix = "undetected"
|
||||
|
|
@ -89,7 +87,7 @@ class Patcher(object):
|
|||
self.force = force
|
||||
try:
|
||||
os.unlink(self.executable_path)
|
||||
except PermissionError: # noqa
|
||||
except PermissionError:
|
||||
if self.force:
|
||||
self.force_kill_instances(self.executable_path)
|
||||
not_force = not self.force
|
||||
|
|
@ -97,9 +95,9 @@ class Patcher(object):
|
|||
try:
|
||||
if self.is_binary_patched():
|
||||
return True # Running AND patched
|
||||
except PermissionError: # noqa
|
||||
except PermissionError:
|
||||
pass
|
||||
except FileNotFoundError: # noqa
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
release = self.fetch_release_number()
|
||||
self.version_main = release.split(".")[0]
|
||||
|
|
@ -122,10 +120,8 @@ class Patcher(object):
|
|||
return urlopen(self.url_repo + path).read().decode()
|
||||
|
||||
def fetch_package(self):
|
||||
"""
|
||||
Downloads chromedriver from source.
|
||||
:return: path to downloaded file
|
||||
"""
|
||||
"""Downloads chromedriver from source.
|
||||
:return: path to downloaded file """
|
||||
from urllib.request import urlretrieve
|
||||
|
||||
u = "%s/%s/%s" % (
|
||||
|
|
@ -135,13 +131,11 @@ class Patcher(object):
|
|||
return urlretrieve(u)[0]
|
||||
|
||||
def unzip_package(self, fp):
|
||||
"""
|
||||
:return: path to unpacked executable
|
||||
"""
|
||||
""" :return: path to unpacked executable """
|
||||
logger.debug("unzipping %s" % fp)
|
||||
try:
|
||||
os.unlink(self.zip_path)
|
||||
except (FileNotFoundError, OSError): # noqa
|
||||
except (FileNotFoundError, OSError):
|
||||
pass
|
||||
os.makedirs(self.zip_path, mode=0o755, exist_ok=True)
|
||||
with zipfile.ZipFile(fp, mode="r") as zf:
|
||||
|
|
@ -163,10 +157,9 @@ class Patcher(object):
|
|||
|
||||
@staticmethod
|
||||
def force_kill_instances(exe_name):
|
||||
"""
|
||||
""" Terminate instances of UC.
|
||||
:param: executable name to kill, may be a path as well
|
||||
:return: True on success else False
|
||||
"""
|
||||
:return: True on success else False """
|
||||
exe_name = os.path.basename(exe_name)
|
||||
if IS_POSIX:
|
||||
r = os.system("kill -f -9 $(pidof %s)" % exe_name)
|
||||
|
|
@ -288,8 +281,8 @@ class Patcher(object):
|
|||
% self.executable_path
|
||||
)
|
||||
break
|
||||
except (OSError, RuntimeError, PermissionError): # noqa
|
||||
except (OSError, RuntimeError, PermissionError):
|
||||
time.sleep(0.1)
|
||||
continue
|
||||
except FileNotFoundError: # noqa
|
||||
except FileNotFoundError:
|
||||
break
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ class Reactor(threading.Thread):
|
|||
except Exception as e:
|
||||
logger.warning("Reactor.run() => %s", e)
|
||||
|
||||
async def _wait_service_started(self): # noqa
|
||||
async def _wait_service_started(self):
|
||||
while True:
|
||||
with self.lock:
|
||||
if (
|
||||
|
|
@ -53,7 +53,7 @@ class Reactor(threading.Thread):
|
|||
else:
|
||||
break
|
||||
|
||||
async def listen(self): # noqa
|
||||
async def listen(self):
|
||||
while self.running:
|
||||
await self._wait_service_started()
|
||||
await asyncio.sleep(1)
|
||||
|
|
|
|||
Loading…
Reference in New Issue