"Drawing" over html text

Question

I need to do the following: having a normal webpage, being able to use a mouse, stylus or finger (in case of a tablet or smartphone) to "draw shapes" over the text, and then using the information of the shapes, determine certain "actions".

Let me give an example. Imagine this question on stackoverflow, and I "draw" on an ipad a circle around this paragraph. The circle (or the attempt of a circle, manual drawing is always very imperfect), "capture" the paragraph, so I know that I will make an action "over" the paragraph, like, for example, select all the inside text.

My question is: Is this feasible? there is any library that does this? If not, which technologies could be the ones to investigate?

I have been making my own research, but nothing has given me all that I need. For example, canvas draws exactly as I need, but a canvas cannot be used over an html page. It is another element on the page, not one who "wraps" the other.

Answer 1

So to answer the initial question: is this feasible?

Yes

Firstly, you need the ability to draw on the page. The HTML5 <canvas> tag is useful for this as it allows us to pretty easily draw on the page.

Second, you need to be able to identify what elements on the page have been circled. The javascript function .getBoundingClientRect() can be useful for this, as it tells us the x and y positions as well as the width and height. And because these values are based on their position relative to the document's bounding box (rather than the parent element), we can compare this with coordinates of our drawn circle.

So with those two components, we can throw together something like this... (I'll admit this is a little clunky, but it can be optimized based on your specific use-case):

var canvas, ctx, flag, coords, cntnr, dot_flag;
    
    function _InitDraw() {
      flag = false;
      coords = {
        pX: 0,
        cX: 0,
        pY: 0,
        cY: 0
      };
      cntnr = {
        minX: document.body.offsetWidth,
        minY: document.body.offsetHeight,
        maxX: 0,
        maxY: 0
      };
      dot_flag = false;

      canvas = document.createElement('canvas');
      canvas.id = "cnvTest";
      canvas.style.position = "fixed";
      canvas.style.top = 0;
      canvas.style.left = 0;
      document.body.appendChild(canvas);
      canvas.style.width = "100%";
      canvas.style.height = "100%";
      canvas.style.zIndex = 9001;
      ctx = canvas.getContext("2d");
      ctx.canvas.width = window.innerWidth;
      ctx.canvas.height = window.innerHeight;
  
      canvas.addEventListener("mousemove", function (e) { _Draw('move', e); }, false);
      canvas.addEventListener("mousedown", function (e) { _Draw('down', e); }, false);
      canvas.addEventListener("mouseup", function (e) { _Draw('up', e); }, false);
      //canvas.addEventListener("mouseout", function (e) { _Draw('out', e); }, false);
    }
    
    function _Draw(action, e) {
      if(action == 'down') {
        coords.pX = coords.cX;
        coords.pY = coords.cY;
        coords.cX = e.pageX;
        coords.cY = e.pageY;
        
        if(coords.cX < cntnr.minX) cntnr.minX = coords.cX;
        if(coords.cY < cntnr.minY) cntnr.minY = coords.cY;
        if(coords.cX > cntnr.maxX) cntnr.maxX = coords.cX;
        if(coords.cY > cntnr.maxY) cntnr.maxY = coords.cY;

        flag = true;
        dot_flag = true;
        if(dot_flag) {
          ctx.beginPath();
          ctx.fillStyle = "black";
          ctx.lineCap = 'round';
          ctx.fillRect(coords.cX, coords.cY, 2, 2);
          ctx.closePath();
          dot_flag = false;
        }
      }
      if(action == 'up' || action == "out") {
        flag = false;
        _GetText();
      }
      if(action == 'move') {
        if(flag) {
          coords.pX = coords.cX;
          coords.pY = coords.cY;
          coords.cX = e.pageX;
          coords.cY = e.pageY;

          if(coords.cX < cntnr.minX) cntnr.minX = coords.cX;
          if(coords.cY < cntnr.minY) cntnr.minY = coords.cY;
          if(coords.cX > cntnr.maxX) cntnr.maxX = coords.cX;
          if(coords.cY > cntnr.maxY) cntnr.maxY = coords.cY;
          
          ctx.beginPath();
          ctx.moveTo(coords.pX, coords.pY);
          ctx.lineTo(coords.cX, coords.cY);
          ctx.strokeStyle = "black";
          ctx.lineCap = 'round';
          ctx.lineWidth = 4;
          ctx.stroke();
          ctx.closePath();
        }
      }
    }

    function _GetText() {
      // Find position and get closest element
      document.body.removeChild(canvas);

      var allEls = document.body.getElementsByTagName("*");
      for(var i = 0; i < allEls.length; i++) {
        var elBox = allEls[i].getBoundingClientRect();
        if(elBox.x > cntnr.minX && elBox.x+elBox.width < cntnr.maxX && elBox.y > cntnr.minY && elBox.y+elBox.height < cntnr.maxY) {
          // Do whatever you want with the element here.
          // I am adding an outline for confirmation of the circled elements.
          allEls[i].style.outline = "2px solid rgba(255, 255, 0, 0.8)";
          if(allEls[i].innerText !== null && allEls[i].innerText !== "undefined") {
            // Do something with the innerText here if you need the text from the element.
            console.log(allEls[i].innerText);
          }
        }
      }
    }

Just for a bit of context, you need to call the _InitDraw() function first. This will create a <canvas> element on the page that should cover the entire page.

Once you release the mouse it ends the drawing mode and uses the minimum and maximum position values and loops through all elements on the page to see if any elements are within those bounds.

For example purposes, this script then outlines those objects and logs any text inside to the console.

*If you want to use this for mobile the event listeners need to change (touchstart, touchmove, touchend).

**Also note that certain elements (such as any 'block' element) can have a bounding box much larger than the text you visually see. So this script will not 'capture' those elements. Granted, you could modify the script to include any elements the drawing passed over, but that could lead to a messy result.