Unix domain sockets 100x slower on Solaris 10 than on Linux?

Question

I am benchmarking local socket performance on Linux and Solaris for a project. For some reason I cannot find out, performance on Solaris is roughly 100x worse than on Linux. In Linux, opening a socket, exchanging one very short (2 char) message each way and closing it takes about 10us elapsed time. On Solaris, the same thing takes about 1000us.

Set-up is Solaris 10 developer vm in Virtual Box and Linux both in the same Virtual Box and directly on the same hardware (makes no difference).

Is this a known issue with Solaris? Any ways to work around it? I cannot use a local network connection instead for reasons I cannot go into here.

Code for client and server below. Compile with "cc -fast -m64 -lrt -lsocket -lnsl -o server server.c" and the equivalent for the client. Gcc 3.4.3 as delivered with Solaris 10 gives comparable results. This code has been cut down, for example timeouts have been removed end error handling is minimal.

server.c:

#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <ctype.h>
#include <sys/select.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/un.h>
#include <time.h>
#include <string.h>
#include <unistd.h>

#define DIRECTORY "sub/"
#define FULL_PATH "sub/c_socket"
#define MAX_COMMAND_LEN 8192
#define PERMISSIONS 0700

void on_error(int err, char * msg) {  // simple convenient error handler
  if (err == -1) {                    // Tests whether 'err' is -1 and
    perror(msg);                      // prints error and msg if so.
    exit(-1);
  }
}

int main() {
  struct sockaddr_un addr;
  int srv_fd, inst_fd;
  int inst_adr_size;
  char c;
  int ret;
  char readbuf[MAX_COMMAND_LEN];
  int num_read;
  fd_set rfds;
  int fail;
  int i;

  // make address
  memset(&addr, 0, sizeof(addr));  // clear out addr
  addr.sun_family = AF_UNIX;
  strncpy(addr.sun_path, FULL_PATH, sizeof(addr.sun_path));

  // Remove old pseudo file if present
  ret = unlink(FULL_PATH);
  if (ret == -1 && errno != ENOENT) {
    on_error(ret,"\nRemoving old socket file\n");
  }
  // Remove old directory if present
  ret = rmdir(DIRECTORY);
  if (ret == -1 && errno != ENOENT) {
    on_error(ret, "\nRemoving old socket directory\n");
  }

  // Re-create new directory with appropriate permissonsm
  ret = mkdir(DIRECTORY, PERMISSIONS);
  on_error(ret,"\nCreating directoroy for socket file\n");

  // create server listening socket
  srv_fd = socket(AF_UNIX, SOCK_STREAM, 0);
  on_error(srv_fd, "\nSocket creation:\n");

  // bind server listening socket to address
  ret = bind(srv_fd, (struct sockaddr *) &addr, sizeof(addr));
  on_error(ret, "\nSocket binding:\n");

  // set file permissions for socket file (somewhat redundant)
  ret = chmod(FULL_PATH, PERMISSIONS);
  on_error(ret, "\nSetting socket file permissions\n");

  // set socket listening and queue length
  ret = listen(srv_fd, 10);
  on_error(ret, "\nSet socket to listen:\n");
  while(1) {
    // accept requests
    inst_fd = accept(srv_fd, NULL, NULL);
    on_error(inst_fd, "\n accepting connection:\n");

    // prepare to use select on inst_fd
    FD_ZERO(&rfds);
    FD_SET(inst_fd, &rfds);

    // now interact with the client on the instance socket.
    while(1) {
      num_read = 0;

      while (1) {
        // read a line terminated by '\n'
        ret = select(inst_fd + 1, &rfds, NULL, NULL, NULL);
        on_error(ret, "\nSelect on socket\n");

        if (ret == 1) {
          // we can read something
          ret = recv(inst_fd, readbuf+num_read, MAX_COMMAND_LEN-num_read, 0
          on_error(ret, "\nrecv:\n");

          if (ret == 0) {
            break; // we have EOF
          }

          num_read += ret;
          if (readbuf[num_read - 1] == '\n') {
            break;
          }
        }
      } /* reading one input line done */

      if (num_read == 0) break; // EOF propagated

      // process command: Just send 2 chars back
      ret = send(inst_fd, "n\n", 2, 0);
    }
    close(inst_fd);               // clean up
  }
  // runs forever...
}

client.c:

#include <stdio.h>
#include <time.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#define RCVBUFSIZE 8192   /* Size of receive buffer */
#define FULL_PATH "sub/c_socket"
#define CYCLES 100000

void on_error(int err, char * msg) {  // more convenient error output
  if (err == -1) {                    // Tests whether 'err' is -1 and
    perror(msg);                      // prints error and msg if so.
    exit(-1);
  }
}

int main(int argc, char *argv[]) {
  int client_fd;
  struct sockaddr_un addr;
  char readbuf[RCVBUFSIZE+1];
  int num_read;
  int ret;
  int count;
  fd_set rfds;

  char * msg = "N\n";

  // make address
  memset(&addr, 0, sizeof(addr));  // clear out addr
  addr.sun_family = AF_UNIX;
  strncpy(addr.sun_path, FULL_PATH, sizeof(addr.sun_path));

  for(count = 0; count < CYCLES; count++) {
    // create socket
    client_fd = socket(PF_UNIX, SOCK_STREAM, 0);
    on_error(client_fd, "socket() failed");

    // prepare to use select on inst_fd
    FD_ZERO(&rfds);
    FD_SET(client_fd, &rfds);

    // connect
    ret = connect(client_fd, (struct sockaddr *) &addr, sizeof(addr));
    on_error(ret, "connect() failed");

    // send msg to server
    ret = send(client_fd, msg, 2, 0);
    if (ret != 2) {
      on_error(-1, "\nnot all bytes sent\n");
    }

    num_read = 0;
    // read until we have a '\n'
    while (1) {
      ret = select(client_fd + 1, &rfds, NULL, NULL, NULL);
      on_error(ret, "\nSelect on socket\n");

      if (ret == 1) {
        // we can read something
        ret = recv(client_fd, readbuf + num_read, RCVBUFSIZE - num_read, 0)
        on_error(ret, "\nrecv:\n");
        num_read += ret;
        if (readbuf[num_read - 1] == '\n') break;
      }
    }
    if (num_read == 0) break;
    close(client_fd);
  }
  return(0);
}

Answer 1

I had a similar issue when I was studying sockets and tried writing an ftp server: because of a bug in the conversion to ascii I ended up writing files one byte at a time, but on linux it was ok, while on windows I ended up with something like 100KB/s on the loop interface... if that is the case, increasing the number of bytes should lessen the difference a lot. It seems that under linux the act of requesting a system call is simply faster.

PS I don't know much about the internals of an operating system, so if anyone can share some pointers to understand the issue (like http://yarchive.net/comp/linux/linux_speed.html) I'd be grateful.