From 981e38c0d2b6976fbfe8f08ca9529963276dd1ad Mon Sep 17 00:00:00 2001 From: JavaScriptDude Date: Thu, 20 Jan 2022 19:50:38 -0500 Subject: [PATCH 01/11] Adding more detailed ConsoleMessage Objects Sample usage: ``` page: Page = await browser.newPage() def h_js_console(cm: ConsoleMessage): for row in cm.toString().split('\n'): print(f"[JS CONSOLE] {row}") page.on(Page.Events.Console, h_js_console) ``` --- pyppeteer/page.py | 56 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/pyppeteer/page.py b/pyppeteer/page.py index 3cb5896f..c74b2183 100644 --- a/pyppeteer/page.py +++ b/pyppeteer/page.py @@ -11,6 +11,7 @@ import mimetypes from types import SimpleNamespace from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, List, Optional, Union +from datetime import datetime from pyee import EventEmitter from pyppeteer import helper @@ -218,7 +219,7 @@ def _onLogEntryAdded(self, event: Dict) -> None: helper.releaseObject(self._client, arg) if source != 'worker': - self.emit(Page.Events.Console, ConsoleMessage(level, text)) + self.emit(Page.Events.Console, ConsoleMessage(level, text, _getTopCallFrame(entry.get('stackTrace', None)), event.get('timestamp', -1))) @property def mainFrame(self) -> Optional['Frame']: @@ -689,7 +690,7 @@ def _onConsoleAPI(self, event: dict) -> None: values: List[JSHandle] = [] for arg in event.get('args', []): values.append(self._frameManager.createJSHandle(context, arg)) - self._addConsoleMessage(event['type'], values) + self._addConsoleMessage(event['type'], values, _getTopCallFrame(event.get('stackTrace', None)), event.get('timestamp', -1)) def _onBindingCalled(self, event: Dict) -> None: obj = json.loads(event['payload']) @@ -711,7 +712,7 @@ def _onBindingCalled(self, event: Dict) -> None: except Exception as e: helper.debugError(logger, e) - def _addConsoleMessage(self, type: str, args: List[JSHandle]) -> None: + def _addConsoleMessage(self, type: str, args: List[JSHandle], jsFrameInfo: dict, tstamp: float) -> None: if not self.listeners(Page.Events.Console): for arg in args: self._client._loop.create_task(arg.dispose()) @@ -725,7 +726,7 @@ def _addConsoleMessage(self, type: str, args: List[JSHandle]) -> None: else: textTokens.append(str(helper.valueFromRemoteObject(remoteObject))) - message = ConsoleMessage(type, ' '.join(textTokens), args) + message = ConsoleMessage(type, ' '.join(textTokens), args, jsFrameInfo, tstamp) self.emit(Page.Events.Console, message) def _onDialog(self, event: Any) -> None: @@ -1689,13 +1690,17 @@ class ConsoleMessage(object): ConsoleMessage objects are dispatched by page via the ``console`` event. """ - def __init__(self, type: str, text: str, args: List[JSHandle] = None) -> None: + def __init__(self, type: str, text: str, args: List[JSHandle] = None, jsFrameInfo: dict = None, tstamp: float = -1) -> None: #: (str) type of console message self._type = type #: (str) console message string self._text = text #: list of JSHandle self._args = args if args is not None else [] + #: (dict) JS Call frame -or- {'empty': True} + self._jsFrameInfo = jsFrameInfo + #: (float) JS Timestamp + self._tstamp = datetime.fromtimestamp(tstamp/1000) if tstamp and tstamp > 0 else None @property def type(self) -> str: @@ -1711,3 +1716,44 @@ def text(self) -> str: def args(self) -> List[JSHandle]: """Return list of args (JSHandle) of this message.""" return self._args + + @property + def frame(self) -> dict: + """Details from CallFrame (dict) that invoked JS console.""" + return self._jsFrameInfo + + @property + def timestamp(self) -> datetime: + """Timestamp of console write. Returns None date if empty""" + return self._tstamp + + def toString(self) -> str: + try: + _f = self.frame + if _f and not _f.get('empty', False): + call_str = f"{_f.get('url', '-')}:{_f.get('lineNumber', '-')}:{_f.get('columnNumber', '-')} " + else: + call_str = "" + sb = [] + ts = f"{self.timestamp.strftime('%y%m%d-%H%M%S.%f')} " if isinstance(self.timestamp, datetime) else "" + for jsh in self.args: + sb.append(f"{ts}{call_str}{jsh._remoteObject.get('value', '-')}") + + return '\n'.join(sb) + + except Exception as ex: + exc_type, exc_value, exc_traceback = sys.exc_info() + sTB = '\n'.join(traceback.format_tb(exc_traceback)) + print(f"toString() failed: {exc_type}\n - msg: {exc_value}\n stack: {sTB}") + + return "" + + + def __repr__(self) -> str: + return f" {self.toString()}" + + + +def _getTopCallFrame(st: dict) -> dict: + sf = st.get('callFrames', None) if st else [] + return (sf[0] if len(sf) > 0 else {'empty': True}) From 8cd00e2f141d50c7101c752a5b91732938580c4e Mon Sep 17 00:00:00 2001 From: JavaScriptDude Date: Thu, 20 Jan 2022 20:19:47 -0500 Subject: [PATCH 02/11] Prevent Failure Under IIS Process When running `Pyppeteer` under windows via IIS, it fails because the `AppDirs` scan happens regardless of whether the `PYPPETEER_HOME` environment variable is set or not: `from pyppeteer import launch` triggers the following Exception: ``` File "C:\apps\sdk\python\Python38\lib\site-packages\pyppeteer\__init__.py", line 27, in __pyppeteer_home__ = os.environ.get('PYPPETEER_HOME', AppDirs('pyppeteer').user_data_dir) # type: str File "C:\apps\sdk\python\Python38\lib\site-packages\appdirs.py", line 419, in user_data_dir return user_data_dir(self.appname, self.appauthor, File "C:\apps\sdk\python\Python38\lib\site-packages\appdirs.py", line 81, in user_data_dir path = os.path.normpath(_get_win_folder(const)) File "C:\apps\sdk\python\Python38\lib\site-packages\appdirs.py", line 481, in _get_win_folder_with_pywin32 dir = shell.SHGetFolderPath(0, getattr(shellcon, csidl_name), 0, 0) pywintypes.com_error: (-2147024891, 'Access is denied.', None, None) ``` Reason for failure: The System user used by IIS does not have a `user_data_dir`. This issue will likely occur for other system processes under windows. --- pyppeteer/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyppeteer/__init__.py b/pyppeteer/__init__.py index 4d365853..de486b11 100644 --- a/pyppeteer/__init__.py +++ b/pyppeteer/__init__.py @@ -24,7 +24,8 @@ __chromium_revision__ = '588429' __base_puppeteer_version__ = 'v1.6.0' -__pyppeteer_home__ = os.environ.get('PYPPETEER_HOME', AppDirs('pyppeteer').user_data_dir) # type: str +__pyppeteer_home__ = os.environ.get('PYPPETEER_HOME') # type: str +if not __pyppeteer_home__: __pyppeteer_home__ = AppDirs('pyppeteer').user_data_dir DEBUG = False from pyppeteer.launcher import connect, executablePath, launch, defaultArgs # noqa: E402; noqa: E402 From 9f6af9fb6f0b98f3f57031b4167c1c12fbd32bc3 Mon Sep 17 00:00:00 2001 From: JavaScriptDude Date: Thu, 20 Jan 2022 21:25:13 -0500 Subject: [PATCH 03/11] Fix Concurrency bug with Err Handler and Another --- pyppeteer/page.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pyppeteer/page.py b/pyppeteer/page.py index c74b2183..8604bf3e 100644 --- a/pyppeteer/page.py +++ b/pyppeteer/page.py @@ -219,7 +219,8 @@ def _onLogEntryAdded(self, event: Dict) -> None: helper.releaseObject(self._client, arg) if source != 'worker': - self.emit(Page.Events.Console, ConsoleMessage(level, text, _getTopCallFrame(entry.get('stackTrace', None)), event.get('timestamp', -1))) + jsFrameInfo = _getTopCallFrame(entry.get('stackTrace', None)) + self.emit(Page.Events.Console, ConsoleMessage(level, text, None, jsFrameInfo, event.get('timestamp', -1))) @property def mainFrame(self) -> Optional['Frame']: @@ -690,7 +691,8 @@ def _onConsoleAPI(self, event: dict) -> None: values: List[JSHandle] = [] for arg in event.get('args', []): values.append(self._frameManager.createJSHandle(context, arg)) - self._addConsoleMessage(event['type'], values, _getTopCallFrame(event.get('stackTrace', None)), event.get('timestamp', -1)) + jsFrameInfo = _getTopCallFrame(event.get('stackTrace', None)) + self._addConsoleMessage(event['type'], values, jsFrameInfo, event.get('timestamp', -1)) def _onBindingCalled(self, event: Dict) -> None: obj = json.loads(event['payload']) @@ -1738,13 +1740,10 @@ def toString(self) -> str: ts = f"{self.timestamp.strftime('%y%m%d-%H%M%S.%f')} " if isinstance(self.timestamp, datetime) else "" for jsh in self.args: sb.append(f"{ts}{call_str}{jsh._remoteObject.get('value', '-')}") - return '\n'.join(sb) except Exception as ex: - exc_type, exc_value, exc_traceback = sys.exc_info() - sTB = '\n'.join(traceback.format_tb(exc_traceback)) - print(f"toString() failed: {exc_type}\n - msg: {exc_value}\n stack: {sTB}") + print(f"toString() failed: {ex}") return "" @@ -1756,4 +1755,5 @@ def __repr__(self) -> str: def _getTopCallFrame(st: dict) -> dict: sf = st.get('callFrames', None) if st else [] - return (sf[0] if len(sf) > 0 else {'empty': True}) + return (sf[0].copy() if len(sf) > 0 else {'empty': True}) + From f209bc5bc3d29de0ef062ed4b2a092989afcc246 Mon Sep 17 00:00:00 2001 From: JavaScriptDude Date: Thu, 20 Jan 2022 22:09:33 -0500 Subject: [PATCH 04/11] Add Ability To Strip Base URL in toString() --- pyppeteer/page.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pyppeteer/page.py b/pyppeteer/page.py index 8604bf3e..3c835fec 100644 --- a/pyppeteer/page.py +++ b/pyppeteer/page.py @@ -1729,11 +1729,13 @@ def timestamp(self) -> datetime: """Timestamp of console write. Returns None date if empty""" return self._tstamp - def toString(self) -> str: + def toString(self, base_url: str = None) -> str: try: _f = self.frame if _f and not _f.get('empty', False): - call_str = f"{_f.get('url', '-')}:{_f.get('lineNumber', '-')}:{_f.get('columnNumber', '-')} " + url = _f.get('url', '-') + if base_url: url = url.replace(base_url, "") + call_str = f"{url}:{_f.get('lineNumber', '-')}:{_f.get('columnNumber', '-')} " else: call_str = "" sb = [] @@ -1743,7 +1745,7 @@ def toString(self) -> str: return '\n'.join(sb) except Exception as ex: - print(f"toString() failed: {ex}") + print(f"ConsoleMessage.toString() failed: {ex}") return "" From b9604317f1a7d6ec5455eeedf4ef9931664c71bf Mon Sep 17 00:00:00 2001 From: JavaScriptDude Date: Thu, 20 Jan 2022 23:05:32 -0500 Subject: [PATCH 05/11] Give toString A Handler To Trim URL Also truncate the machine date/time format --- pyppeteer/page.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pyppeteer/page.py b/pyppeteer/page.py index 3c835fec..50b1d9b8 100644 --- a/pyppeteer/page.py +++ b/pyppeteer/page.py @@ -10,7 +10,7 @@ import math import mimetypes from types import SimpleNamespace -from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, List, Optional, Union, types from datetime import datetime from pyee import EventEmitter @@ -1729,17 +1729,20 @@ def timestamp(self) -> datetime: """Timestamp of console write. Returns None date if empty""" return self._tstamp - def toString(self, base_url: str = None) -> str: + def toString(self, url_fixer: types.FunctionType = None) -> str: try: _f = self.frame if _f and not _f.get('empty', False): url = _f.get('url', '-') - if base_url: url = url.replace(base_url, "") + try: + if url_fixer: url = url_fixer(url) + except: + pass call_str = f"{url}:{_f.get('lineNumber', '-')}:{_f.get('columnNumber', '-')} " else: call_str = "" sb = [] - ts = f"{self.timestamp.strftime('%y%m%d-%H%M%S.%f')} " if isinstance(self.timestamp, datetime) else "" + ts = f"{self.timestamp.strftime('%y%m%d-%H%M%S.%f')[:17]} " if isinstance(self.timestamp, datetime) else "" for jsh in self.args: sb.append(f"{ts}{call_str}{jsh._remoteObject.get('value', '-')}") return '\n'.join(sb) From af2616338faa0d781b5c43952a9d81bb89b9a360 Mon Sep 17 00:00:00 2001 From: JavaScriptDude Date: Fri, 21 Jan 2022 15:22:49 -0500 Subject: [PATCH 06/11] Updating To Conform To Puppeteer 2.1.1 Design With the exception of adding the timestamp which puppeteer did not implement at this time. --- pyppeteer/page.py | 59 ++++++++++++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/pyppeteer/page.py b/pyppeteer/page.py index 50b1d9b8..5b596519 100644 --- a/pyppeteer/page.py +++ b/pyppeteer/page.py @@ -219,8 +219,7 @@ def _onLogEntryAdded(self, event: Dict) -> None: helper.releaseObject(self._client, arg) if source != 'worker': - jsFrameInfo = _getTopCallFrame(entry.get('stackTrace', None)) - self.emit(Page.Events.Console, ConsoleMessage(level, text, None, jsFrameInfo, event.get('timestamp', -1))) + self.emit(Page.Events.Console, ConsoleMessage(level, text, None, _getLocationFromStacktrace(entry.get('stackTrace', None)), event.get('timestamp', -1))) @property def mainFrame(self) -> Optional['Frame']: @@ -687,12 +686,28 @@ def _handleException(self, exceptionDetails: Dict) -> None: def _onConsoleAPI(self, event: dict) -> None: _id = event['executionContextId'] + + if _id == 0: + # DevTools protocol stores the last 1000 console messages. These + # messages are always reported even for removed execution contexts. In + # this case, they are marked with executionContextId = 0 and are + # reported upon enabling Runtime agent. + # + # Ignore these messages since: + # - there's no execution context we can use to operate with message + # arguments + # - these messages are reported before Puppeteer clients can subscribe + # to the 'console' + # page event. + # + # @see https://github.com/puppeteer/puppeteer/issues/3865 + return + context = self._frameManager.executionContextById(_id) values: List[JSHandle] = [] for arg in event.get('args', []): values.append(self._frameManager.createJSHandle(context, arg)) - jsFrameInfo = _getTopCallFrame(event.get('stackTrace', None)) - self._addConsoleMessage(event['type'], values, jsFrameInfo, event.get('timestamp', -1)) + self._addConsoleMessage(event['type'], values, event.get('stackTrace', None), event.get('timestamp', -1)) def _onBindingCalled(self, event: Dict) -> None: obj = json.loads(event['payload']) @@ -714,7 +729,7 @@ def _onBindingCalled(self, event: Dict) -> None: except Exception as e: helper.debugError(logger, e) - def _addConsoleMessage(self, type: str, args: List[JSHandle], jsFrameInfo: dict, tstamp: float) -> None: + def _addConsoleMessage(self, type: str, args: List[JSHandle], stackTrace, tstamp: float) -> None: if not self.listeners(Page.Events.Console): for arg in args: self._client._loop.create_task(arg.dispose()) @@ -728,7 +743,7 @@ def _addConsoleMessage(self, type: str, args: List[JSHandle], jsFrameInfo: dict, else: textTokens.append(str(helper.valueFromRemoteObject(remoteObject))) - message = ConsoleMessage(type, ' '.join(textTokens), args, jsFrameInfo, tstamp) + message = ConsoleMessage(type, ' '.join(textTokens), args, _getLocationFromStacktrace(stackTrace), tstamp) self.emit(Page.Events.Console, message) def _onDialog(self, event: Any) -> None: @@ -1692,15 +1707,15 @@ class ConsoleMessage(object): ConsoleMessage objects are dispatched by page via the ``console`` event. """ - def __init__(self, type: str, text: str, args: List[JSHandle] = None, jsFrameInfo: dict = None, tstamp: float = -1) -> None: + def __init__(self, type: str, text: str, args: List[JSHandle] = None, location: dict = None, tstamp: float = -1) -> None: #: (str) type of console message self._type = type #: (str) console message string self._text = text #: list of JSHandle self._args = args if args is not None else [] - #: (dict) JS Call frame -or- {'empty': True} - self._jsFrameInfo = jsFrameInfo + #: (dict) JS Console Caller info + self._location = location #: (float) JS Timestamp self._tstamp = datetime.fromtimestamp(tstamp/1000) if tstamp and tstamp > 0 else None @@ -1720,9 +1735,9 @@ def args(self) -> List[JSHandle]: return self._args @property - def frame(self) -> dict: - """Details from CallFrame (dict) that invoked JS console.""" - return self._jsFrameInfo + def location(self) -> dict: + """dict of url, line, column that invoked JS console.""" + return self._location @property def timestamp(self) -> datetime: @@ -1731,14 +1746,14 @@ def timestamp(self) -> datetime: def toString(self, url_fixer: types.FunctionType = None) -> str: try: - _f = self.frame - if _f and not _f.get('empty', False): - url = _f.get('url', '-') + _l = self.location + if _l and not _l.get('empty', False): + url = _l.get('url', '-') try: if url_fixer: url = url_fixer(url) except: pass - call_str = f"{url}:{_f.get('lineNumber', '-')}:{_f.get('columnNumber', '-')} " + call_str = f"{url}:{_l.get('lineNumber', '-')}:{_l.get('columnNumber', '-')} " else: call_str = "" sb = [] @@ -1757,8 +1772,10 @@ def __repr__(self) -> str: return f" {self.toString()}" - -def _getTopCallFrame(st: dict) -> dict: - sf = st.get('callFrames', None) if st else [] - return (sf[0].copy() if len(sf) > 0 else {'empty': True}) - +def _getLocationFromStacktrace(stackTrace: dict) -> dict: + sf = stackTrace.get('callFrames', None) if stackTrace else None + return {} if not sf or len(sf) < 1 else { + "url": sf[0].get('url', '-'), + "lineNumber": sf[0].get('lineNumber', '-'), + "columnNumber": sf[0].get('columnNumber', '-') + } From 170a0f4f7a9eda770d46f199d7fec3d5861684dd Mon Sep 17 00:00:00 2001 From: JavaScriptDude Date: Fri, 21 Jan 2022 15:24:09 -0500 Subject: [PATCH 07/11] Restore To Original To Prevent Confusion --- pyppeteer/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyppeteer/__init__.py b/pyppeteer/__init__.py index de486b11..4d365853 100644 --- a/pyppeteer/__init__.py +++ b/pyppeteer/__init__.py @@ -24,8 +24,7 @@ __chromium_revision__ = '588429' __base_puppeteer_version__ = 'v1.6.0' -__pyppeteer_home__ = os.environ.get('PYPPETEER_HOME') # type: str -if not __pyppeteer_home__: __pyppeteer_home__ = AppDirs('pyppeteer').user_data_dir +__pyppeteer_home__ = os.environ.get('PYPPETEER_HOME', AppDirs('pyppeteer').user_data_dir) # type: str DEBUG = False from pyppeteer.launcher import connect, executablePath, launch, defaultArgs # noqa: E402; noqa: E402 From 0aab4070866295aeb88079cd78f0e5884ea13d8d Mon Sep 17 00:00:00 2001 From: JavaScriptDude Date: Sat, 22 Jan 2022 02:12:51 -0500 Subject: [PATCH 08/11] Improve Data Processing of Console Messages Included more extensibility of toString() functionality by including a callback for user specified stringifier. --- pyppeteer/page.py | 163 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 127 insertions(+), 36 deletions(-) diff --git a/pyppeteer/page.py b/pyppeteer/page.py index 5b596519..cce952c7 100644 --- a/pyppeteer/page.py +++ b/pyppeteer/page.py @@ -9,6 +9,7 @@ import logging import math import mimetypes +import pprint from types import SimpleNamespace from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, List, Optional, Union, types from datetime import datetime @@ -210,16 +211,19 @@ def _onTargetCrashed(self, *args: Any, **kwargs: Any) -> None: self.emit('error', PageError('Page crashed!')) def _onLogEntryAdded(self, event: Dict) -> None: - entry = event.get('entry', {}) - level = entry.get('level', '') - text = entry.get('text', '') - args = entry.get('args', []) - source = entry.get('source', '') - for arg in args: - helper.releaseObject(self._client, arg) + try: + entry = event.get('entry', {}) + level = entry.get('level', '') + text = entry.get('text', '') + args = entry.get('args', []) + source = entry.get('source', '') + for arg in args: + helper.releaseObject(self._client, arg) - if source != 'worker': - self.emit(Page.Events.Console, ConsoleMessage(level, text, None, _getLocationFromStacktrace(entry.get('stackTrace', None)), event.get('timestamp', -1))) + if source != 'worker': + self.emit(Page.Events.Console, ConsoleMessage(level, text, None, _getLocationFromStacktrace(entry.get('stackTrace', None)), event.get('timestamp', -1))) + except Exception as e: + debugError(logger, e) @property def mainFrame(self) -> Optional['Frame']: @@ -685,29 +689,32 @@ def _handleException(self, exceptionDetails: Dict) -> None: self.emit(Page.Events.PageError, PageError(message)) def _onConsoleAPI(self, event: dict) -> None: - _id = event['executionContextId'] - - if _id == 0: - # DevTools protocol stores the last 1000 console messages. These - # messages are always reported even for removed execution contexts. In - # this case, they are marked with executionContextId = 0 and are - # reported upon enabling Runtime agent. - # - # Ignore these messages since: - # - there's no execution context we can use to operate with message - # arguments - # - these messages are reported before Puppeteer clients can subscribe - # to the 'console' - # page event. - # - # @see https://github.com/puppeteer/puppeteer/issues/3865 - return + try: + _id = event['executionContextId'] + + if _id == 0: + # DevTools protocol stores the last 1000 console messages. These + # messages are always reported even for removed execution contexts. In + # this case, they are marked with executionContextId = 0 and are + # reported upon enabling Runtime agent. + # + # Ignore these messages since: + # - there's no execution context we can use to operate with message + # arguments + # - these messages are reported before Puppeteer clients can subscribe + # to the 'console' + # page event. + # + # @see https://github.com/puppeteer/puppeteer/issues/3865 + return - context = self._frameManager.executionContextById(_id) - values: List[JSHandle] = [] - for arg in event.get('args', []): - values.append(self._frameManager.createJSHandle(context, arg)) - self._addConsoleMessage(event['type'], values, event.get('stackTrace', None), event.get('timestamp', -1)) + context = self._frameManager.executionContextById(_id) + values: List[JSHandle] = [] + for arg in event.get('args', []): + values.append(self._frameManager.createJSHandle(context, arg)) + self._addConsoleMessage(event['type'], values, event.get('stackTrace', None), event.get('timestamp', -1)) + except Exception as e: + debugError(logger, e) def _onBindingCalled(self, event: Dict) -> None: obj = json.loads(event['payload']) @@ -1744,7 +1751,7 @@ def timestamp(self) -> datetime: """Timestamp of console write. Returns None date if empty""" return self._tstamp - def toString(self, url_fixer: types.FunctionType = None) -> str: + def toString(self, url_fixer: types.FunctionType = None, dumper: types.FunctionType = None) -> str: try: _l = self.location if _l and not _l.get('empty', False): @@ -1756,18 +1763,41 @@ def toString(self, url_fixer: types.FunctionType = None) -> str: call_str = f"{url}:{_l.get('lineNumber', '-')}:{_l.get('columnNumber', '-')} " else: call_str = "" - sb = [] + a = [] ts = f"{self.timestamp.strftime('%y%m%d-%H%M%S.%f')[:17]} " if isinstance(self.timestamp, datetime) else "" + for jsh in self.args: - sb.append(f"{ts}{call_str}{jsh._remoteObject.get('value', '-')}") - return '\n'.join(sb) + _ro = jsh._remoteObject + (_type, _subType, _data) = _simplifyRemoteObject(self.type, jsh._remoteObject) + + # Try calling user defined dumper + _msg = self._call_dumper(dumper, _type, _subType, _data) if dumper else None + + # Stringify if None + try: + _msg = _stringifyRemoteObject(_type, _subType, _data) if _msg is None else _msg + except Exception as ex: + _msg = f"_stringifyRemoteObject() Failed on console type: {self.type} remoteObject: {_ro} ex: {ex.args}" + + + a.append(f"{ts}[{self.type}] {call_str}{_msg}") + + return '\n'.join(a) except Exception as ex: - print(f"ConsoleMessage.toString() failed: {ex}") + debugError(logger, "ConsoleMessage.toString() failed") + debugError(logger, ex) return "" + def _call_dumper(self, h, _type, _subType, _data): + try: + return h(self, _type, _subType, _data) + except: + pass + return "-" + def __repr__(self) -> str: return f" {self.toString()}" @@ -1779,3 +1809,64 @@ def _getLocationFromStacktrace(stackTrace: dict) -> dict: "lineNumber": sf[0].get('lineNumber', '-'), "columnNumber": sf[0].get('columnNumber', '-') } + + +# Serializes type and data from remote object as tuple of (_type, _subType, _data) +# For ones it can't figure out, it returns _subType of 'unknown' and remoteObject as _data +def _simplifyRemoteObject(logType: str, remoteObject: dict) -> dict: + _type = remoteObject.get('type', None) + + if logType == 'table': + _subType = remoteObject.get('className', None) + if _subType == 'Array' or _subType == 'Object': + _data = [] if _subType == 'Array' else {} + for _prop in remoteObject.get('preview', {}).get('properties', []): + if _subType == 'Array': + _data.append([_vpp.get('value', None) for _vpp in _prop.get('valuePreview', {}).get('properties', []) if 1]) + elif _subType == 'Object': + _name = _prop.get('name', None) + if not _name is None: + _data[_name] = _prop.get('value', None) + + return ('table', None, _data) + + + if _type == 'undefined': return ('undefined', None, None) + if _type == 'function' or _type == 'symbol': return (_type, None, remoteObject.get('description', "-")) + + if _type == 'object': + _subType = remoteObject.get('subtype', '-') + if _subType == 'date': + return (_type, _subType, remoteObject.get('description', "-")) + + elif _subType == 'null': + return ('null', None, None) + + else: + _className = remoteObject.get('className', '-') + return (_type, _className, remoteObject.get('preview', {}).get('properties', None)) + + else: + _data = remoteObject.get('value', None) + if _data is None: + _data = remoteObject.get('unserializableValue', None) + if not _data is None: return (_type, "unserializable", _data) + + if not _data is None: return (_type, None, _data) + + return (_type, "unknown", remoteObject) + + +def _stringifyRemoteObject(_type: str, _subType: str, _data: Any) -> dict: + if _type == 'table': return f"[console.table]:\n{pprint.PrettyPrinter(indent=2).pformat(_data)}" + if _type == 'function' or _type == 'symbol': return _data + if _type == 'bigint': return f" {_data}" + if _type == 'undefined' or _type == 'null': return f"({_type})" + if _type in ['string', 'number', 'boolean']: return _data + + if _type == 'object': + if _subType == 'date': return f" {_data}" + if _subType == 'Object': return f" ..." + return f" ..." + + return f"??? <{_type} {_subType}> {_data if _data else '-'}" From 193686faff0b76bf5f6788bd4367d3a2e9fbbaf7 Mon Sep 17 00:00:00 2001 From: JavaScriptDude Date: Sat, 22 Jan 2022 02:18:14 -0500 Subject: [PATCH 09/11] Typing Fix and Add Subtype for Edgecase Stringify --- pyppeteer/page.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyppeteer/page.py b/pyppeteer/page.py index cce952c7..bfce9229 100644 --- a/pyppeteer/page.py +++ b/pyppeteer/page.py @@ -10,8 +10,8 @@ import math import mimetypes import pprint -from types import SimpleNamespace -from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, List, Optional, Union, types +from types import SimpleNamespace, FunctionType +from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, List, Optional, Union from datetime import datetime from pyee import EventEmitter @@ -1751,7 +1751,7 @@ def timestamp(self) -> datetime: """Timestamp of console write. Returns None date if empty""" return self._tstamp - def toString(self, url_fixer: types.FunctionType = None, dumper: types.FunctionType = None) -> str: + def toString(self, url_fixer: FunctionType = None, dumper: FunctionType = None) -> str: try: _l = self.location if _l and not _l.get('empty', False): From bcb2e5e67c5c2a7d96c5bc5947cbb6305e40988f Mon Sep 17 00:00:00 2001 From: JavaScriptDude Date: Sun, 23 Jan 2022 11:40:38 -0500 Subject: [PATCH 10/11] Move _simplifyRemoteObject to Method of ConsoleMessage New method is getData(). Done to aid in consuming method from user code and tests. --- pyppeteer/page.py | 94 ++++++++++++++++++++++++----------------------- 1 file changed, 49 insertions(+), 45 deletions(-) diff --git a/pyppeteer/page.py b/pyppeteer/page.py index bfce9229..49e83183 100644 --- a/pyppeteer/page.py +++ b/pyppeteer/page.py @@ -1767,8 +1767,7 @@ def toString(self, url_fixer: FunctionType = None, dumper: FunctionType = None) ts = f"{self.timestamp.strftime('%y%m%d-%H%M%S.%f')[:17]} " if isinstance(self.timestamp, datetime) else "" for jsh in self.args: - _ro = jsh._remoteObject - (_type, _subType, _data) = _simplifyRemoteObject(self.type, jsh._remoteObject) + (_type, _subType, _data) = self.getData(jsh) # Try calling user defined dumper _msg = self._call_dumper(dumper, _type, _subType, _data) if dumper else None @@ -1798,6 +1797,54 @@ def _call_dumper(self, h, _type, _subType, _data): pass return "-" + + # Serializes JSHandle._remoteObject and returns tuple of (_type, _subType, _data) + # For ones it can't figure out, it returns _subType of 'unknown' and remoteObject as _data + def getData(self, jsh: JSHandle) -> dict: + remoteObject: dict = jsh._remoteObject + _type = remoteObject.get('type', None) + + if self.type == 'table': + _subType = remoteObject.get('className', None) + if _subType == 'Array' or _subType == 'Object': + _data = [] if _subType == 'Array' else {} + for _prop in remoteObject.get('preview', {}).get('properties', []): + if _subType == 'Array': + _data.append([_vpp.get('value', None) for _vpp in _prop.get('valuePreview', {}).get('properties', []) if 1]) + elif _subType == 'Object': + _name = _prop.get('name', None) + if not _name is None: + _data[_name] = _prop.get('value', None) + + return ('table', None, _data) + + + if _type == 'undefined': return ('undefined', None, None) + if _type == 'function' or _type == 'symbol': return (_type, None, remoteObject.get('description', "-")) + + if _type == 'object': + _subType = remoteObject.get('subtype', '-') + if _subType == 'date': + return (_type, _subType, remoteObject.get('description', "-")) + + elif _subType == 'null': + return ('null', None, None) + + else: + _className = remoteObject.get('className', '-') + return (_type, _className, remoteObject.get('preview', {}).get('properties', None)) + + else: + _data = remoteObject.get('value', None) + if _data is None: + _data = remoteObject.get('unserializableValue', None) + if not _data is None: return (_type, "unserializable", _data) + + if not _data is None: return (_type, None, _data) + + return (_type, "unknown", remoteObject) + + def __repr__(self) -> str: return f" {self.toString()}" @@ -1811,50 +1858,7 @@ def _getLocationFromStacktrace(stackTrace: dict) -> dict: } -# Serializes type and data from remote object as tuple of (_type, _subType, _data) -# For ones it can't figure out, it returns _subType of 'unknown' and remoteObject as _data -def _simplifyRemoteObject(logType: str, remoteObject: dict) -> dict: - _type = remoteObject.get('type', None) - - if logType == 'table': - _subType = remoteObject.get('className', None) - if _subType == 'Array' or _subType == 'Object': - _data = [] if _subType == 'Array' else {} - for _prop in remoteObject.get('preview', {}).get('properties', []): - if _subType == 'Array': - _data.append([_vpp.get('value', None) for _vpp in _prop.get('valuePreview', {}).get('properties', []) if 1]) - elif _subType == 'Object': - _name = _prop.get('name', None) - if not _name is None: - _data[_name] = _prop.get('value', None) - - return ('table', None, _data) - - - if _type == 'undefined': return ('undefined', None, None) - if _type == 'function' or _type == 'symbol': return (_type, None, remoteObject.get('description', "-")) - - if _type == 'object': - _subType = remoteObject.get('subtype', '-') - if _subType == 'date': - return (_type, _subType, remoteObject.get('description', "-")) - - elif _subType == 'null': - return ('null', None, None) - - else: - _className = remoteObject.get('className', '-') - return (_type, _className, remoteObject.get('preview', {}).get('properties', None)) - - else: - _data = remoteObject.get('value', None) - if _data is None: - _data = remoteObject.get('unserializableValue', None) - if not _data is None: return (_type, "unserializable", _data) - - if not _data is None: return (_type, None, _data) - return (_type, "unknown", remoteObject) def _stringifyRemoteObject(_type: str, _subType: str, _data: Any) -> dict: From 98d1f219da7172d4e5bbc0d0fd0ebe319f548025 Mon Sep 17 00:00:00 2001 From: JavaScriptDude Date: Mon, 24 Jan 2022 13:13:21 -0500 Subject: [PATCH 11/11] Cleanup --- pyppeteer/page.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/pyppeteer/page.py b/pyppeteer/page.py index 49e83183..6aca448c 100644 --- a/pyppeteer/page.py +++ b/pyppeteer/page.py @@ -1770,13 +1770,17 @@ def toString(self, url_fixer: FunctionType = None, dumper: FunctionType = None) (_type, _subType, _data) = self.getData(jsh) # Try calling user defined dumper - _msg = self._call_dumper(dumper, _type, _subType, _data) if dumper else None + if dumper: + try: + _msg = dumper(self, _type, _subType, _data) + except: + _msg = "-" - # Stringify if None + # Stringify if _msg is None try: _msg = _stringifyRemoteObject(_type, _subType, _data) if _msg is None else _msg except Exception as ex: - _msg = f"_stringifyRemoteObject() Failed on console type: {self.type} remoteObject: {_ro} ex: {ex.args}" + _msg = f"_stringifyRemoteObject() Failed on console type: {self.type} remoteObject: {jsh._remoteObject} ex: {ex.args}" a.append(f"{ts}[{self.type}] {call_str}{_msg}") @@ -1790,14 +1794,6 @@ def toString(self, url_fixer: FunctionType = None, dumper: FunctionType = None) return "" - def _call_dumper(self, h, _type, _subType, _data): - try: - return h(self, _type, _subType, _data) - except: - pass - return "-" - - # Serializes JSHandle._remoteObject and returns tuple of (_type, _subType, _data) # For ones it can't figure out, it returns _subType of 'unknown' and remoteObject as _data def getData(self, jsh: JSHandle) -> dict: @@ -1819,6 +1815,7 @@ def getData(self, jsh: JSHandle) -> dict: return ('table', None, _data) + if _type == 'undefined': return ('undefined', None, None) if _type == 'function' or _type == 'symbol': return (_type, None, remoteObject.get('description', "-")) @@ -1835,12 +1832,11 @@ def getData(self, jsh: JSHandle) -> dict: return (_type, _className, remoteObject.get('preview', {}).get('properties', None)) else: - _data = remoteObject.get('value', None) - if _data is None: - _data = remoteObject.get('unserializableValue', None) - if not _data is None: return (_type, "unserializable", _data) - - if not _data is None: return (_type, None, _data) + try: + _data = helper.valueFromRemoteObject(remoteObject) + return (_type, None, _data) + except helper.ElementHandleError as e: + return (_type, "unserializable", remoteObject.get('unserializableValue')) return (_type, "unknown", remoteObject)