python html selenium innerhtml outerhtml

How to get the html in selenium of current page

I try to get HTML code of the page, but driver.page_source always return HTML + JavaScript.

I know it is many ways described earlier, but in my case they don't work.

The page is - https://qe.com.qa/en/companymoreinformationsearch?CompanyCode=QNBK

Solution

To extract the HTML along with the JavaScript you need to induce WebDriverWait for the visibility_of_all_elements_located() with id as moreinfo-heading and you can use the following Locator Strategy:

Code Block:

Using driver.page_source:

driver.get("https://qe.com.qa/en/companymoreinformationsearch?CompanyCode=QNBK")
WebDriverWait(driver, 20).until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, "#moreinfo-heading")))
print(driver.page_source)

As an alternative, you can also use document.documentElement.outerHTML and document.documentElement.innerHTML as follows:

Using document.documentElement.outerHTML:

driver.get("https://qe.com.qa/en/companymoreinformationsearch?CompanyCode=QNBK")
print(driver.execute_script("return document.documentElement.outerHTML"))

Using document.documentElement.innerHTML:

driver.get("https://qe.com.qa/en/companymoreinformationsearch?CompanyCode=QNBK")
print(driver.execute_script("return document.documentElement.innerHTML"))

Console Output:

getSessionId: function() {
                return '';
            },
            getSiteAdminURL: function() {
                return 'https://qe.com.qa/group/guest/~/control_panel/manage?p_p_lifecycle=0&p_p_state=maximized&p_p_mode=view';
            },
            getSiteGroupId: function() {
                return '20181';
            },
            getURLControlPanel: function() {
                return '/en/group/control_panel?refererPlid=1002057';
            },
            getURLHome: function() {
                return 'https\x3a\x2f\x2fqe\x2ecom\x2eqa\x2fweb\x2fguest';
            },
            getUserEmailAddress: function() {
                return '';
            },
            getUserId: function() {
                return '20158';
            },
            getUserName: function() {
                return '';
            },
            isAddSessionIdToURL: function() {
                return false;
            },
            isImpersonated: function() {
                return false;
            },
            isSignedIn: function() {
                return false;
            },
            isStateExclusive: function() {
                return false;
            },
            isStateMaximized: function() {
                return false;
            },
            isStatePopUp: function() {
                return false;
            }
        };

        var themeDisplay = Liferay.ThemeDisplay;

        Liferay.AUI = {



            getAvailableLangPath: function() {
                return 'available_languages.jsp?browserId=other&themeId=mainsitetheme_WAR_mainsitetheme&colorSchemeId=01&minifierType=js&languageId=en_US&b=7210&t=1635431572526';
            },
            getCombine: function() {
                return true;
            },
            getComboPath: function() {
                return '/combo/?browserId=other&minifierType=&languageId=en_US&b=7210&t=1620065756342&';
            },
            getDateFormat: function() {
                return '%m/%d/%Y';
            },
            getEditorCKEditorPath: function() {
                return '/o/frontend-editor-ckeditor-web';
            },
            getFilter: function() {
                var filter = 'raw';



                        filter = 'min';



                return filter;
            },
            getFilterConfig: function() {
                var instance = this;

                var filterConfig = null;

                if (!instance.getCombine()) {
                    filterConfig = {
                        replaceStr: '.js' + instance.getStaticResourceURLParams(),
                        searchExp: '\\.js$'
                    };
                }

                return filterConfig;
            },
            getJavaScriptRootPath: function() {
                return '/o/frontend-js-web';
            },
            getLangPath: function() {
                return 'aui_lang.jsp?browserId=other&themeId=mainsitetheme_WAR_mainsitetheme&colorSchemeId=01&minifierType=js&languageId=en_US&b=7210&t=1620065756342';
            },
            getPortletRootPath: function() {
                return '/html/portlet';
            },
            getStaticResourceURLParams: function() {
                return '?browserId=other&minifierType=&languageId=en_US&b=7210&t=1620065756342';
            }
        };

        Liferay.authToken = '8LGXsu8b';



        Liferay.currentURL = '\x2fen\x2fcompanymoreinformationsearch\x3fCompanyCode\x3dQNBK';
        Liferay.currentURLEncoded = '\x252Fen\x252Fcompanymoreinformationsearch\x253FCompanyCode\x253DQNBK';
    // ]]>
</script>

<script src="/o/js_loader_config?t=1635431593465" type="text/javascript"></script>
<script data-senna-track="permanent" src="/combo?browserId=other&amp;minifierType=js&amp;languageId=en_US&amp;b=7210&amp;t=1620065756342&amp;/o/frontend-js-web/loader/config.js&amp;/o/frontend-js-web/loader/loader.js&amp;/o/frontend-js-web/aui/aui/aui.js&amp;/o/frontend-js-web/aui/aui-base-html5-shiv/aui-base-html5-shiv.js&amp;/o/frontend-js-web/liferay/browser_selectors.js&amp;/o/frontend-js-web/liferay/modules.js&amp;/o/frontend-js-web/liferay/aui_sandbox.js&amp;/o/frontend-js-web/misc/svg4everybody.js&amp;/o/frontend-js-web/aui/arraylist-add/arraylist-add.js&amp;/o/frontend-js-web/aui/arraylist-filter/arraylist-filter.js&amp;/o/frontend-js-web/aui/arraylist/arraylist.js&amp;/o/frontend-js-web/aui/array-extras/array-extras.js&amp;/o/frontend-js-web/aui/array-invoke/array-invoke.js&amp;/o/frontend-js-web/aui/attribute-base/attribute-base.js&amp;/o/frontend-js-web/aui/attribute-complex/attribute-complex.js&amp;/o/frontend-js-web/aui/attribute-core/attribute-core.js&amp;/o/frontend-js-web/aui/attribute-observable/attribute-observable.js&amp;/o/frontend-js-web/aui/attribute-extras/attribute-extras.js&amp;/o/frontend-js-web/aui/base-base/base-base.js&amp;/o/frontend-js-web/aui/base-pluginhost/base-pluginhost.js&amp;/o/frontend-js-web/aui/classnamemanager/classnamemanager.js&amp;/o/frontend-js-web/aui/datatype-xml-format/datatype-xml-format.js&amp;/o/frontend-js-web/aui/datatype-xml-parse/datatype-xml-parse.js&amp;/o/frontend-js-web/aui/dom-base/dom-base.js&amp;/o/frontend-js-web/aui/dom-core/dom-core.js&amp;/o/frontend-js-web/aui/dom-screen/dom-screen.js&amp;/o/frontend-js-web/aui/dom-style/dom-style.js&amp;/o/frontend-js-web/aui/event-base/event-base.js&amp;/o/frontend-js-web/aui/event-custom-base/event-custom-base.js&amp;/o/frontend-js-web/aui/event-custom-complex/event-custom-complex.js&amp;/o/frontend-js-web/aui/event-delegate/event-delegate.js&amp;/o/frontend-js-web/aui/event-focus/event-focus.js&amp;/o/frontend-js-web/aui/event-hover/event-hover.js&amp;/o/frontend-js-web/aui/event-key/event-key.js&amp;/o/frontend-js-web/aui/event-mouseenter/event-mouseenter.js&amp;/o/frontend-js-web/aui/event-mousewheel/event-mousewheel.js" type="text/javascript"></script>
<script data-senna-track="permanent" src="/combo?browserId=other&amp;minifierType=js&amp;languageId=en_US&amp;b=7210&amp;t=1620065756342&amp;/o/frontend-js-web/aui/event-outside/event-outside.js&amp;/o/frontend-js-web/aui/event-resize/event-resize.js&amp;/o/frontend-js-web/aui/event-simulate/event-simulate.js&amp;/o/frontend-js-web/aui/event-synthetic/event-synthetic.js&amp;/o/frontend-js-web/aui/intl/intl.js&amp;/o/frontend-js-web/aui/io-base/io-base.js&amp;/o/frontend-js-web/aui/io-form/io-form.js&amp;/o/frontend-js-web/aui/io-queue/io-queue.js&amp;/o/frontend-js-web/aui/io-upload-iframe/io-upload-iframe.js&amp;/o/frontend-js-web/aui/io-xdr/io-xdr.js&amp;/o/frontend-js-web/aui/json-parse/json-parse.js&amp;/o/frontend-js-web/aui/json-stringify/json-stringify.js&amp;/o/frontend-js-web/aui/node-base/node-base.js&amp;/o/frontend-js-web/aui/node-core/node-core.js&amp;/o/frontend-js-web/aui/node-event-delegate/node-event-delegate.js&amp;/o/frontend-js-web/aui/node-event-simulate/node-event-simulate.js&amp;/o/frontend-js-web/aui/node-focusmanager/node-focusmanager.js&amp;/o/frontend-js-web/aui/node-pluginhost/node-pluginhost.js&amp;/o/frontend-js-web/aui/node-screen/node-screen.js&amp;/o/frontend-js-web/aui/node-style/node-style.js&amp;/o/frontend-js-web/aui/oop/oop.js&amp;/o/frontend-js-web/aui/plugin/plugin.js&amp;/o/frontend-js-web/aui/pluginhost-base/pluginhost-base.js&amp;/o/frontend-js-web/aui/pluginhost-config/pluginhost-config.js&amp;/o/frontend-js-web/aui/querystring-stringify-simple/querystring-stringify-simple.js&amp;/o/frontend-js-web/aui/queue-promote/queue-promote.js&amp;/o/frontend-js-web/aui/selector-css2/selector-css2.js&amp;/o/frontend-js-web/aui/selector-css3/selector-css3.js&amp;/o/frontend-js-web/aui/selector-native/selector-native.js&amp;/o/frontend-js-web/aui/selector/selector.js&amp;/o/frontend-js-web/aui/widget-base/widget-base.js&amp;/o/frontend-js-web/aui/widget-htmlparser/widget-htmlparser.js&amp;/o/frontend-js-web/aui/widget-skin/widget-skin.js&amp;/o/frontend-js-web/aui/widget-uievents/widget-uievents.js&amp;/o/frontend-js-web/aui/yui-throttle/yui-throttle.js&amp;/o/frontend-js-web/aui/aui-base-core/aui-base-core.js" type="text/javascript"></script>
<script data-senna-track="permanent" src="/combo?browserId=other&amp;minifierType=js&amp;languageId=en_US&amp;b=7210&amp;t=1620065756342&amp;/o/frontend-js-web/aui/aui-base-lang/aui-base-lang.js&amp;/o/frontend-js-web/aui/aui-classnamemanager/aui-classnamemanager.js&amp;/o/frontend-js-web/aui/aui-component/aui-component.js&amp;/o/frontend-js-web/aui/aui-debounce/aui-debounce.js&amp;/o/frontend-js-web/aui/aui-delayed-task-deprecated/aui-delayed-task-deprecated.js&amp;/o/frontend-js-web/aui/aui-event-base/aui-event-base.js&amp;/o/frontend-js-web/aui/aui-event-input/aui-event-input.js&amp;/o/frontend-js-web/aui/aui-form-validator/aui-form-validator.js&amp;/o/frontend-js-web/aui/aui-node-base/aui-node-base.js&amp;/o/frontend-js-web/aui/aui-node-html5/aui-node-html5.js&amp;/o/frontend-js-web/aui/aui-selector/aui-selector.js&amp;/o/frontend-js-web/aui/aui-timer/aui-timer.js&amp;/o/frontend-js-web/liferay/dependency.js&amp;/o/frontend-js-web/liferay/dom_task_runner.js&amp;/o/frontend-js-web/liferay/events.js&amp;/o/frontend-js-web/liferay/language.js&amp;/o/frontend-js-web/liferay/lazy_load.js&amp;/o/frontend-js-web/liferay/liferay.js&amp;/o/frontend-js-web/liferay/util.js&amp;/o/frontend-js-web/liferay/global.bundle.js&amp;/o/frontend-js-web/liferay/portal.js&amp;/o/frontend-js-web/liferay/portlet.js&amp;/o/frontend-js-web/liferay/workflow.js&amp;/o/frontend-js-web/liferay/form.js&amp;/o/frontend-js-web/liferay/form_placeholders.js&amp;/o/frontend-js-web/liferay/icon.js&amp;/o/frontend-js-web/liferay/menu.js&amp;/o/frontend-js-web/liferay/notice.js&amp;/o/frontend-js-web/liferay/poller.js" type="text/javascript"></script>






    <script data-senna-track="temporary" src="/o/js_bundle_config?t=1635431612919" type="text/javascript"></script>


<script data-senna-track="temporary" type="text/javascript">
    // <![CDATA[








    // ]]>
</script>
...