How to fix 'failed to find any PEM data in key input' error?

Question

I'm setting up Traefik with provided certificates for HTTPS using docker Swarm and it doesn't load them failing with failed to find any PEM data in key input

I've tried to set it up with relative and absolute paths (see https://github.com/containous/traefik/issues/2001 ) but it doesn't seem to solve the issue.

The certificates I'm using are self signed but they're perfectly working with Nginx.

Traefik configuration in compose:

version: "3.6"

services:

  traefik:
    image: traefik
    command:
      - "--defaultentrypoints=http,https"
      - "--docker"
      - "--docker.swarmMode"
      - "--docker.exposedByDefault=false"
      - "--docker.domain=sdb.it"
      - "--docker.watch"
      - "--entryPoints='Name:http Address::80 Redirect.EntryPoint:https'"
      - "--entryPoints='Name:https Address::443 TLS:/etc/ssl/certs/sonarqube.crt,/etc/ssl/certs/sonarqube.key'"
      - "--loglevel=DEBUG"
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
    ports:
      - 80:80
      - 443:443
    networks:
      - traefik
    secrets:
      - source: sdbit-sonarqube-docker.sdb.it.crt
        target: /etc/ssl/certs/sonarqube.crt
        mode: 644
      - source: sdbit-sonarqube-docker.sdb.it.key
        target: /etc/ssl/certs/sonarqube.key
        mode: 644
    deploy:
      placement:
        constraints:
          - node.role == manager

volumes:
  certificates:
    external: true
networks:
  traefik:
    external: true
secrets:
  sdbit-sonarqube-docker.sdb.it.crt:
    external: true
  sdbit-sonarqube-docker.sdb.it.key:
    external: true

And this is the Traefik log:

time="2019-02-15T17:57:51Z" level=info msg="No tls.defaultCertificate given for : using the first item in tls.certificates as a fallback.",
time="2019-02-15T17:57:51Z" level=info msg="Traefik version v1.7.9 built on 2019-02-11_11:36:32AM",
time="2019-02-15T17:57:51Z" level=debug msg="Global configuration loaded {\"LifeCycle\":{\"RequestAcceptGraceTimeout\":0,\"GraceTimeOut\":10000000000},\"GraceTimeOut\":0,\"Debug\":false,\"CheckNewVersion\":true,\"SendAnonymousUsage\":false,\"AccessLogsFile\":\"\",\"AccessLog\":null,\"TraefikLogsFile\":\"\",\"TraefikLog\":null,\"Tracing\":null,\"LogLevel\":\"DEBUG\",\"EntryPoints\":{\"\":{\"Address\":\":443\",\"TLS\":{\"MinVersion\":\"\",\"CipherSuites\":null,\"Certificates\":[{\"CertFile\":\"certs/sonarqube.crt\",\"KeyFile\":\"certs/sonarqube.key'\"}],\"ClientCAFiles\":null,\"ClientCA\":{\"Files\":null,\"Optional\":false},\"DefaultCertificate\":{\"CertFile\":\"certs/sonarqube.crt\",\"KeyFile\":\"certs/sonarqube.key'\"},\"SniStrict\":false},\"Redirect\":null,\"Auth\":null,\"WhitelistSourceRange\":null,\"WhiteList\":null,\"Compress\":false,\"ProxyProtocol\":null,\"ForwardedHeaders\":{\"Insecure\":true,\"TrustedIPs\":null}}},\"Cluster\":null,\"Constraints\":[],\"ACME\":null,\"DefaultEntryPoints\":[\"http\",\"https\"],\"ProvidersThrottleDuration\":2000000000,\"MaxIdleConnsPerHost\":200,\"IdleTimeout\":0,\"InsecureSkipVerify\":false,\"RootCAs\":null,\"Retry\":null,\"HealthCheck\":{\"Interval\":30000000000},\"RespondingTimeouts\":null,\"ForwardingTimeouts\":null,\"AllowMinWeightZero\":false,\"KeepTrailingSlash\":false,\"Web\":null,\"Docker\":{\"Watch\":true,\"Filename\":\"\",\"Constraints\":null,\"Trace\":false,\"TemplateVersion\":2,\"DebugLogGeneratedTemplate\":false,\"Endpoint\":\"unix:///var/run/docker.sock\",\"Domain\":\"sdb.it\",\"TLS\":null,\"ExposedByDefault\":false,\"UseBindPortIP\":false,\"SwarmMode\":true,\"Network\":\"\",\"SwarmModeRefreshSeconds\":15},\"File\":null,\"Marathon\":null,\"Consul\":null,\"ConsulCatalog\":null,\"Etcd\":null,\"Zookeeper\":null,\"Boltdb\":null,\"Kubernetes\":null,\"Mesos\":null,\"Eureka\":null,\"ECS\":null,\"Rancher\":null,\"DynamoDB\":null,\"ServiceFabric\":null,\"Rest\":null,\"API\":null,\"Metrics\":null,\"Ping\":null,\"HostResolver\":null}",
time="2019-02-15T17:57:51Z" level=info msg="\nStats collection is disabled.\nHelp us improve Traefik by turning this feature on :)\nMore details on: https://docs.traefik.io/basics/#collected-data\n",
time="2019-02-15T17:57:51Z" level=error msg="failed to load X509 key pair: tls: failed to find any PEM data in certificate input",
time="2019-02-15T17:57:51Z" level=info msg="Preparing server  &{Address::443 TLS:0xc000283290 Redirect:<nil> Auth:<nil> WhitelistSourceRange:[] WhiteList:<nil> Compress:false ProxyProtocol:<nil> ForwardedHeaders:0xc000512540} with readTimeout=0s writeTimeout=0s idleTimeout=3m0s",
time="2019-02-15T17:57:51Z" level=error msg="Unable to add a certificate to the entryPoint \"\" : unable to generate TLS certificate : tls: failed to find any PEM data in certificate input",
time="2019-02-15T17:57:51Z" level=info msg="Starting provider configuration.ProviderAggregator {}",
time="2019-02-15T17:57:51Z" level=info msg="Starting server on :443",
time="2019-02-15T17:57:51Z" level=info msg="Starting provider *docker.Provider {\"Watch\":true,\"Filename\":\"\",\"Constraints\":null,\"Trace\":false,\"TemplateVersion\":2,\"DebugLogGeneratedTemplate\":false,\"Endpoint\":\"unix:///var/run/docker.sock\",\"Domain\":\"sdb.it\",\"TLS\":null,\"ExposedByDefault\":false,\"UseBindPortIP\":false,\"SwarmMode\":true,\"Network\":\"\",\"SwarmModeRefreshSeconds\":15}",
time="2019-02-15T17:57:51Z" level=debug msg="Provider connection established with docker 18.09.0 (API 1.39)",
time="2019-02-15T17:57:51Z" level=debug msg="Filtering disabled container prometheus_alertmanager.1",
time="2019-02-15T17:57:51Z" level=debug msg="Filtering disabled container portainer_portainer.1",
time="2019-02-15T17:57:51Z" level=debug msg="Filtering disabled container prometheus_cadvisor.02f9e4aqq9h8p5wxtvebrpdmi",
time="2019-02-15T17:57:51Z" level=debug msg="Filtering disabled container prometheus_cadvisor.3wjdodinomlez4o034htgxq4f",
time="2019-02-15T17:57:51Z" level=debug msg="Filtering disabled container prometheus_cadvisor.6qextrzc6c3mli99sl5qs8sj7",
time="2019-02-15T17:57:51Z" level=debug msg="Filtering disabled container prometheus_cadvisor.epwzjchzyldg35bp7zh83h2l8",
time="2019-02-15T17:57:51Z" level=debug msg="Filtering disabled container prometheus_cadvisor.fex6ncwmfhrs4mp8g3iwk2yxb",
time="2019-02-15T17:57:51Z" level=debug msg="Filtering disabled container prometheus_prometheus.1",
time="2019-02-15T17:57:51Z" level=debug msg="Filtering disabled container sonarqube-glf-dev_sonarqube.1",
time="2019-02-15T17:57:51Z" level=debug msg="Filtering disabled container sonarqube-glf-dev_db.1",
time="2019-02-15T17:57:51Z" level=debug msg="Filtering disabled container portainer_agent.dm14e8f833zvl3iov8c7ejlui",
time="2019-02-15T17:57:51Z" level=debug msg="Filtering disabled container portainer_agent.f61gqjypxiepukygmba1kjwi1",
time="2019-02-15T17:57:51Z" level=debug msg="Filtering disabled container portainer_agent.iei6yqpdqfqm6okwmp54pbdt8",
time="2019-02-15T17:57:51Z" level=debug msg="Filtering disabled container portainer_agent.oej5oojf7vhp17hi0h0notgjd",
time="2019-02-15T17:57:51Z" level=debug msg="Filtering disabled container portainer_agent.oxa7l6ahqpo4mu5j0zoh4puf9",
time="2019-02-15T17:57:51Z" level=debug msg="Filtering disabled container prometheus_node-exporter.hzarmo2gu75r0mrmwtfeitbok",
time="2019-02-15T17:57:51Z" level=debug msg="Filtering disabled container prometheus_node-exporter.igb6gb1yb313gky7j3t9idc8k",
time="2019-02-15T17:57:51Z" level=debug msg="Filtering disabled container prometheus_node-exporter.oyr1umf2pp7bdkvuez7nz8m54",
time="2019-02-15T17:57:51Z" level=debug msg="Filtering disabled container prometheus_node-exporter.v7q6iugofokx59254h537tvnz",
time="2019-02-15T17:57:51Z" level=debug msg="Filtering disabled container prometheus_node-exporter.v9d4wnwgvlcfytgk4de1ys1k6",
time="2019-02-15T17:57:51Z" level=debug msg="Filtering disabled container prometheus_grafana.1",
time="2019-02-15T17:57:51Z" level=debug msg="Filtering disabled container gitlab-runner_gitlab-runner.1",
time="2019-02-15T17:57:51Z" level=debug msg="Configuration received from provider docker: {}",
time="2019-02-15T17:57:51Z" level=error msg="failed to load X509 key pair: tls: failed to find any PEM data in certificate input",
time="2019-02-15T17:57:51Z" level=info msg="Server configuration reloaded on :443",

Answer 1

Replying to my own question: it was the way I was passing the command line arguments in the above Compose file.

This way Traefik does not accept the certificates:

      - "--defaultentrypoints=http,https"
      - "--docker"
      - "--docker.swarmMode"
      - "--docker.exposedByDefault=false"
      - "--docker.domain=sdb.it"
      - "--docker.watch"
      - "--entryPoints='Name:http Address::80 Redirect.EntryPoint:https'"
      - "--entryPoints='Name:https Address::443 TLS:/etc/ssl/certs/sonarqube.crt,/etc/ssl/certs/sonarqube.key'"
      - "--loglevel=DEBUG"

But if you remove the quotes I mistakenly put in entryPoints parameter it does:

    command:
      - --defaultentrypoints=http,https
      - --docker
      - --docker.swarmMode
      - --docker.exposedByDefault=false
      - --docker.domain=sdb.it
      - --docker.watch
      - --entryPoints=Name:http Address::80 Redirect.EntryPoint:https
      - --entryPoints=Name:https Address::443 TLS:/etc/ssl/certs/sonarqube.crt,/etc/ssl/certs/sonarqube.key