[FIXED] PySide: QWebView: why a lot of elements are null?

Issue

Why a lot of QWebElements in a QWebView are Null? Though they seem perfectly valid in html.

Here's a demonstration using PySide:

from PySide.QtGui import QWidget, QMainWindow, QApplication, QVBoxLayout,\
  QLineEdit, QPainter, QPen
from PySide.QtCore import QUrl, Qt
from PySide.QtWebKit import QWebView, QWebElement
import sys

class Window(QMainWindow):
  def __init__(self, parent=None):
    super(Window, self).__init__(parent)
    layout = QVBoxLayout()
    self.lineedit_search = QLineEdit()
    self.lineedit_search.editingFinished.connect(self.loadBrowser)
    self.lineedit_search.setText("http://www.imdb.com/chart/top")
    self.browser = MouseSensitiveWebView()
    self.browser.setUrl(QUrl("http://www.imdb.com/chart/top"))
    layout.addWidget(self.lineedit_search)
    layout.addWidget(self.browser)
    layout_widget = QWidget()
    layout_widget.setLayout(layout)
    self.setCentralWidget(layout_widget)

  def loadBrowser(self):
    txt = self.lineedit_search.text()
    self.browser.load(QUrl(txt))


class MouseSensitiveWebView(QWebView):
  def __init__(self, parent=None):
    super(MouseSensitiveWebView, self).__init__(parent)
    self.setMouseTracking(True)
    self._hover_rect = None
    self._hit_elem = None

  def mouseMoveEvent(self, mme):
    mouse_pos = mme.pos()
    hit_test_result = self.page().mainFrame().hitTestContent(mouse_pos)
    if hit_test_result:
      new_rect = hit_test_result.boundingRect()
      if new_rect == self._hover_rect: # don't repaint unless the hovered element changed
        return
      hit_elem = hit_test_result.element()
      if hit_elem.isNull():
        print("{0} is Null".format(hit_elem))
      else:
        print("{0} is not Null".format(hit_elem))
      self._hover_rect = new_rect  
      self._hit_elem = hit_elem
      self.repaint()
    else:
      self._hover_rect = None

  def paintEvent(self, pe):
    if not self._hover_rect:
      super(MouseSensitiveWebView, self).paintEvent(pe)
    else:
      super(MouseSensitiveWebView, self).paintEvent(pe)
      # get the current horizontal and vertical scroll bar positions
      main_frame = self.page().mainFrame()
      hor_pos, ver_pos = main_frame.scrollBarValue(Qt.Horizontal), main_frame.scrollBarValue(Qt.Vertical)
      # move the hover rect by these values
      hover_rect = self._hover_rect
      hover_rect.moveTop(hover_rect.top() - ver_pos)
      hover_rect.moveLeft(hover_rect.left() - hor_pos)
      painter = QPainter(self)
      pen = QPen(Qt.red)
      pen.setWidth(2)
      painter.setPen(pen)
      painter.drawRect(hover_rect)

if __name__ == "__main__":
  app = QApplication(sys.argv)
  main = Window()
  main.setGeometry(50, 50, 800, 600)
  main.show()
  sys.exit(app.exec_())

Just keep hovering your mouse and a red square will be drawn around the element you are hovering. If it's null or not, it'll logged to the console.

You'll notice that most of the visible elements are actually Null. Which makes the QWebView much less useful when operating on the page elements. This also applies to PyQt4 which seems similar in that respect or event worse and getting more Nulls.

Solution

First, the issue of null elements seems to have been filed as a bug recently.

I've made a little workaround that'll allow only links (<a> elements), from the inline family, to be selected. I get their text and search within the enclosing parent for an element with that text:

def mouseMoveEvent(self, mme):
    mouse_pos = mme.pos()
    hit_test_result = self.page().mainFrame().hitTestContent(mouse_pos)
    if hit_test_result:
      new_rect = hit_test_result.boundingRect()
      if new_rect == self._hover_rect: # don't repaint unless the hovered element changed
        return
      hit_elem = hit_test_result.element()
      if hit_elem.isNull():
        enclosing = hit_test_result.enclosingBlockElement()
        elem_text = hit_elem.linkText() # works only for <a> elements
        for subelement in enclosing.findAll('*'):
          if subelement.toPlainText() == elem_text:
            print "Found Link {}".format(subelement)
            break
        else:
          print("Element is null and not a link")
      else:
        print("{0} is not Null".format(hit_elem))
      self._hover_rect = new_rect  
      self._hit_elem = hit_elem
      self.repaint()
    else:
      self._hover_rect = None

Answered By - MadeOfAir