Docker nginx proxy websocket getting 404

Question

My app is made up of angular front end, nginx proxy server, with a nestjs node application. The node application exposes an api on port 3000 and sockets on 3003. Running on my dev machine it all works, but in the docker configuration the websockets don't work. The api calls works. The mongodb works as well.

I set up a docker compose with the mongodb and server, and running the angular app in ng serve, and the websockets work, but they don't work through the nginx proxy. I'm not sure what I am missing.

Here is the server Dockerfile

FROM node

ENV HOME=/usr/src/app
RUN mkdir -p $HOME
WORKDIR $HOME

RUN npm -g install @angular/[email protected]

EXPOSE 3000
EXPOSE 3334
EXPOSE 3003

USER 1000

Here is the nginx configuration

version: '3.6'
services:
  mongodb:
    image: mongo:latest
    container_name: mongodb
    restart: always
    secrets:
      - mongodb_rootusername
      - mongodb_rootuserpwd
      - mongodb_username
      - mongodb_userpwd
    environment:
      MONGO_INITDB_ROOT_USERNAME: /run/secrets/mongodb_rootusername
      MONGO_INITDB_ROOT_PASSWORD: /run/secrets/mongodb_rootuserpwd
      MONGO_INITDB_DATABASE: admin
      MONGO_USERNAME: /run/secrets/mongodb_username
      MONGO_USERPWD: /run/secrets/mongodb_userpwd
    ports:
      - 27017
    volumes:
      - ./mongo-init.js:/docker-entrypoint-initdb.d/mongo-init.sh:ro
  oauth2:
    container_name: oauth2
    image: 97842411e57c
    ports:
      - '4180:4180'
    command:
      - --provider=google
      - --cookie-secure=false
      - --cookie-refresh=1h
      - --cookie-expire=168h
      - --upstream=http://upstream:80
      - --http-address=0.0.0.0:4180
      - --email-domain=<mydomain>
      - --set-xauthrequest=true
      - --set-authorization-header=true
      - --request-logging=false
      - --proxy-websockets=true
    secrets:
      - OAUTH2_PROXY_CLIENT_ID
      - OAUTH2_PROXY_CLIENT_SECRET
      - OAUTH2_PROXY_COOKIE_NAME
      - OAUTH2_PROXY_COOKIE_SECRET
      - OAUTH2_PROXY_REDIRECT_URL

  server:
    container_name: server
    build:
      context: .
      dockerfile: Dockerfile.server.dev
    ports:
      - "3000:3000"
      - "3334:3334"
      - "3003:3003"
    volumes:
      - .:/usr/src/app
    command: ng serve api
    depends_on:
      - mongodb
  scripts:
    container_name: scripts
    build:
      context: .
      dockerfile: Dockerfile.scripts.dev
    ports:
      - "3333:3333"
    volumes:
      - .:/usr/src/app
    command: ng serve scripts
    depends_on:
      - mongodb
  angular:
    container_name: angular
    build:
      context: .
      dockerfile: Dockerfile.angular
    ports:
      - "4200"
    volumes:
      - .:/usr/src/app
    command: ng serve --aot --host 0.0.0.0
  web:
    container_name: web
    build: .
    volumes:
      - ./nginx.dev.conf:/etc/nginx/nginx.conf
    ports:
      - "80:80"
    depends_on:
      - angular
      - oauth2

secrets:
  mongodb_rootusername:
    file: ../../serverdata/dev/mongodb_rootusername
  mongodb_rootuserpwd:
    file: ../../serverdata/dev/mongodb_rootuserpwd
  mongodb_username:
    file: ../../serverdata/dev/mongodb_username
  mongodb_userpwd:
    file: ../../serverdata/dev/mongodb_userpwd
  OAUTH2_PROXY_CLIENT_ID:
    file: ../../serverdata/dev/oauth2_clientid
  OAUTH2_PROXY_CLIENT_SECRET:
    file: ../../serverdata/dev/oauth2_clientsecret
  OAUTH2_PROXY_COOKIE_NAME:
    file: ../../serverdata/dev/oauth2_cookiename
  OAUTH2_PROXY_COOKIE_SECRET:
    file: ../../serverdata/dev/oauth2_cookiesecret
  OAUTH2_PROXY_REDIRECT_URL:
    file: ../../serverdata/dev/oauth2_redirecturl

Here is the nginx configuration

events {}
http {
        upstream node_server {
            server server:3000;
        }

        upstream node_server_websockets {
            server server:3003;
        }

        upstream angular_cli {
            server angular:4200;
        }

        upstream oauth2 {
            server oauth2:4180;


     }
server {
        listen 80 default_server;
        listen [::]:80 default_server;

        server_name localhost;



    location / {
        auth_request /oauth2/auth;
        error_page 401 = /oauth2/sign_in;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection $http_connection;
        proxy_set_header Host $host;
        proxy_pass http://angular_cli;
    }

    location /oauth2/ {
        proxy_pass       http://oauth2;
        proxy_set_header Host                    $host;
        proxy_set_header X-Real-IP               $remote_addr;
        proxy_set_header X-Scheme                $scheme;
        proxy_set_header X-Auth-Request-Redirect $request_uri;
        # or, if you are handling multiple domains:
        # proxy_set_header X-Auth-Request-Redirect $scheme://$host$request_uri;
      }
      location = /oauth2/auth {
        proxy_pass       http://oauth2;
        proxy_set_header Host             $host;
        proxy_set_header X-Real-IP        $remote_addr;
        proxy_set_header X-Scheme         $scheme;
        # nginx auth_request includes headers but not body
        proxy_set_header Content-Length   "";
        proxy_pass_request_body           off;
      }


     location /tapi/ {
        auth_request /oauth2/auth;


    # pass information via X-User and X-Email headers to backend,
        # requires running with --set-xauthrequest flag
        auth_request_set $user   $upstream_http_x_auth_request_user;
        auth_request_set $email  $upstream_http_x_auth_request_email;
        proxy_set_header X-User  $user;
        proxy_set_header X-Email $email;

        # if you enabled --pass-access-token, this will pass the token to the backend
        auth_request_set $token  $upstream_http_x_auth_request_access_token;
        proxy_set_header X-Access-Token $token;

        # if you enabled --cookie-refresh, this is needed for it to work with auth_request
        auth_request_set $auth_cookie $upstream_http_set_cookie;
        add_header Set-Cookie $auth_cookie;

        # When using the --set-authorization-header flag, some provider's cookies can exceed the 4kb
        # limit and so the OAuth2 Proxy splits these into multiple parts.
        # Nginx normally only copies the first `Set-Cookie` header from the auth_request to the response,
        # so if your cookies are larger than 4kb, you will need to extract additional cookies manually.
        auth_request_set $auth_cookie_name_upstream_1 $upstream_cookie_auth_cookie_name_1;

        # Extract the Cookie attributes from the first Set-Cookie header and append them
        # to the second part ($upstream_cookie_* variables only contain the raw cookie content)
        if ($auth_cookie ~* "(; .*)") {
            set $auth_cookie_name_0 $auth_cookie;
            set $auth_cookie_name_1 "auth_cookie_name_1=$auth_cookie_name_upstream_1$1";
        }

        # Send both Set-Cookie headers now if there was a second part
        if ($auth_cookie_name_upstream_1) {
            add_header Set-Cookie $auth_cookie_name_0;
            add_header Set-Cookie $auth_cookie_name_1;
        }



        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection 'upgrade';
        proxy_set_header Host $host;
        proxy_cache_bypass $http_upgrade;
        proxy_pass http://node_server;

    }
    location /socket.io {
        # auth_request /oauth2/auth;

        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection 'upgrade';
        # proxy_set_header Host $host;
        proxy_pass http://server:3003/;
    }

}
}

The node server nestjs web socket controller:

@WebSocketGateway(3003, {})
export class ActionsGateway implements OnGatewayConnection, OnGatewayDisconnect, OnGatewayInit {
    @WebSocketServer() wss;

    private logger = new Logger('AppGateway');

    handleConnection(client) {
        this.logger.log('New client connected');
        client.emit('connection', 'Successfully connected to server');
    }

    handleDisconnect(client) {
        this.logger.log('Client disconnected');
    }

    afterInit(server): any {
        this.logger.log('actions gateway initialized');
        // this.wss.
    }
}

And in angular:

this.socket = io(environment.socket.baseUrl); 

where environment.socket is:

socket: {
        baseUrl: 'ws://localhost/socket.io/',
        config: {}
    }

The error from nginx logs:

web        | 172.18.0.1 - - [31/Oct/2019:21:56:47 +0000] "GET /socket.io/?EIO=3&transport=polling&t=MuZqZ14 HTTP/1.1" 404 5 "http://localhost/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36"

Answer 1

I also faced similar problem few days ago. My problem was with nginx. I could solve my problem by adding below code to nginx configuration.

location / {
  proxy_pass http://localhost:8080;
  proxy_http_version 1.1;
  proxy_set_header Upgrade $http_upgrade;
  proxy_set_header Connection "upgrade";
  proxy_set_header Host $host;
}

For more information refer this GitHub issue and this article.

Answer 2

nginx configure,is the 'map' parameter missing ?

map $http_upgrade $connection_upgrade {
default upgrade;
''      close;
}

server {
   listen       80;
   server_name  _;

   location / {
        auth_request /oauth2/auth;
        error_page 401 = /oauth2/sign_in;
        proxy_pass http://angular_cli;
        proxy_set_header Host $host;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection $connection_upgrade;
   }
}

take a look at the official documents of nginx