Linux Evdev Poll Lag

Question

I'm on a laptop with 2 connected keyboards (built-in and USB). I'm obtaining these connected keyboards with libudev and using epoll to poll them for input via the evdev interface:

// Compile with $(gcc udev.c -ludev)

#include <stdbool.h>
#include <stdio.h>
#include <string.h>

#include <sys/epoll.h>
#include <sys/poll.h>
#include <sys/types.h>
#include <fcntl.h>
#include <unistd.h>
#include <linux/input.h>
#include <time.h>
#include <libudev.h>

#define BILLION 1000000000L

long timespec_diff(struct timespec *start, struct timespec *end)
{
  return (BILLION * (end->tv_sec - start->tv_sec)) +
         (end->tv_nsec - start->tv_nsec);
}

bool want_to_run = true;

int
main(int argc, char *argv[])
{
  int epoll_fd = epoll_create1(0);

  struct udev *udev_obj = udev_new();
  struct udev_enumerate *udev_enum = udev_enumerate_new(udev_obj);
  udev_enumerate_add_match_subsystem(udev_enum, "input");
  udev_enumerate_scan_devices(udev_enum);

  struct udev_list_entry *udev_entries = udev_enumerate_get_list_entry(udev_enum);
  struct udev_list_entry *udev_entry = NULL;
  udev_list_entry_foreach(udev_entry, udev_entries)
  {
    char const *udev_entry_syspath = udev_list_entry_get_name(udev_entry);
    struct udev_device *device = udev_device_new_from_syspath(udev_obj, 
                                                              udev_entry_syspath);

    char const *dev_prop = \
      udev_device_get_property_value(device, "ID_INPUT_KEYBOARD");
    if (dev_prop != NULL && strcmp(dev_prop, "1") == 0) 
    {
      const char *dev_path = udev_device_get_devnode(device);
      if (dev_path != NULL)
      {
        int dev_fd = open(dev_path, O_RDWR | O_NONBLOCK);

        struct epoll_event event = {};
        event.events = EPOLLIN;
        event.data.fd = dev_fd;
        epoll_ctl(epoll_fd, EPOLL_CTL_ADD, dev_fd, &event);
      }
    }
    udev_device_unref(device);
  }
  udev_enumerate_unref(udev_enum);

  struct timespec prev_timespec = {};
  clock_gettime(CLOCK_MONOTONIC_RAW, &prev_timespec);
  while (want_to_run)
  {
    struct epoll_event epoll_events[5] = {0};
    int num_epoll_events = epoll_wait(epoll_fd, epoll_events, 5, 0);
    for (int epoll_event_i = 0; epoll_event_i < num_epoll_events; ++epoll_event_i)
    {
      int dev_fd = epoll_events[epoll_event_i].data.fd;

      struct input_event dev_events[4] = {0};
      int dev_event_bytes_read = read(dev_fd, dev_events, sizeof(dev_events));

      int num_dev_events = dev_event_bytes_read / sizeof(dev_events[0]); 
      for (int dev_event_i = 0; dev_event_i < num_dev_events; ++dev_event_i)
      {
        int dev_event_type = dev_events[dev_event_i].type;
        int dev_event_code = dev_events[dev_event_i].code;
        int dev_event_value = dev_events[dev_event_i].value;

        bool is_released = (dev_event_type == EV_KEY ? dev_event_value == 0 : false);
        bool is_down = (dev_event_type == EV_KEY ? dev_event_value == 1 : false);
        bool was_down = (dev_event_type == EV_KEY ? dev_event_value == 2 : false);

        bool w = (dev_event_code == KEY_W);
        bool a = (dev_event_code == KEY_A);
        bool s = (dev_event_code == KEY_S);
        bool d = (dev_event_code == KEY_D);
        bool q = (dev_event_code == KEY_Q);
        bool e = (dev_event_code == KEY_E);
        bool up = (dev_event_code == KEY_UP);
        bool down = (dev_event_code == KEY_DOWN);
        bool left = (dev_event_code == KEY_LEFT);
        bool right = (dev_event_code == KEY_RIGHT);
        bool escape = (dev_event_code == KEY_ESC);
        bool space = (dev_event_code == KEY_SPACE);
        bool enter = (dev_event_code == KEY_ENTER);
        bool ctrl = (dev_event_code == KEY_LEFTCTRL);
        if (q) want_to_run = false;
      }
    }

    struct timespec end_timespec = {};
    clock_gettime(CLOCK_MONOTONIC_RAW, &end_timespec);
    printf("ns per frame: %lu\n", timespec_diff(&prev_timespec, &end_timespec)); 
    prev_timespec = end_timespec;
  }

  return 0;
}

Experimenting by entering keys on each keyboard I experience some serious lag/stall in the following circumstances (I encourage you to compile and try yourself):

1. If I start entering keys on one keyboard and then switch to the other, the program stalls briefly.
2. If I simultaneously enter keys on each keyboard the program stalls indefinitely until I stop entering keys.

I have tested with different keyboards and experience the same result. What is going on here?

UPDATE

I think it may be specific to my host environment (Ubuntu 20.04) as when I spam keys on each keyboard inside another program, e.g. gnome terminal, firefox, etc. the same stalling and delay happens. If I wail on the keys it's some serious stall (the cpu fan goes off and everything) It only happens for keyboards it seems as moving an external mouse and trackpad together causes no issues.

Running under Intel VTune hotspots analysis shows that epoll_wait() is the source of the stall (no surprises there)

Answer 1

I have tested your program in my ubuntu desktop (20.04), same isusse happend. But when i enter CLI mode (CTRL + ALT + F3), run the program again, there's no problem.

then I go back to GUI mode, append current timestamp to printf("ns per frame...), rebuild and run, simultaneously enter keys on two keyboards, the program stop output, but if i stop typing, after a short time of lag, logs with timestamp during the lag time will gushing out. So it seems that there's no problem with the program, maybe a BUG of Xorg affected all desktop softwares.

I found this post: https://askubuntu.com/questions/1044985/using-2-keyboards-at-the-same-time-create-annoying-input-lag