App Engine Java 11 could not find or load main class on live server

Question

tl;dr: Why does this work locally but not when I deploy to my live App Engine project?

I'm trying to create a barebones servlet-based web app using the Java 11 version of App Engine. I'm updating a few projects from Java 8 to Java 11 following this guide. I'm also using this guide and this example. My goal is to use Jetty to run a very simple web app that serves a single static HTML file and a single servlet file in App Engine.

My web app works fine when I run locally:

mvn clean install
mvn exec:java -Dexec.args="target/app-engine-hello-world-1.war"

When I run these commands, both my index.html and my servlet URL work fine.

But when I deploy to my live site:

mvn package appengine:deploy

...the command succeeds, but when I navigate to my live URL, I get this error for both the HTML file and the servlet URL: "Error: Server Error. The server encountered an error and could not complete your request. Please try again in 30 seconds." If I look in the logs in the Cloud console, I see this error:

Error: Could not find or load main class io.happycoding.Main
Caused by: java.lang.ClassNotFoundException: io.happycoding.Main

Something is off with my setup, but I don't see anything obviously wrong.

Here are the files in my project:

pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>

  <groupId>io.happycoding</groupId>
  <artifactId>app-engine-hello-world</artifactId>
  <version>1</version>
  <packaging>war</packaging>

  <properties>
    <!-- App Engine currently supports Java 11 -->
    <maven.compiler.source>11</maven.compiler.source>
    <maven.compiler.target>11</maven.compiler.target>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <failOnMissingWebXml>false</failOnMissingWebXml>
  </properties>

  <dependencies>
    <dependency>
      <groupId>javax.servlet</groupId>
      <artifactId>javax.servlet-api</artifactId>
      <version>4.0.1</version>
    </dependency>

    <dependency>
      <groupId>org.eclipse.jetty</groupId>
      <artifactId>jetty-server</artifactId>
      <version>9.4.31.v20200723</version>
    </dependency>
    <dependency>
      <groupId>org.eclipse.jetty</groupId>
      <artifactId>jetty-webapp</artifactId>
      <version>9.4.31.v20200723</version>
      <type>jar</type>
    </dependency>
    <dependency>
      <groupId>org.eclipse.jetty</groupId>
      <artifactId>jetty-util</artifactId>
      <version>9.4.31.v20200723</version>
    </dependency>
    <dependency>
      <groupId>org.eclipse.jetty</groupId>
      <artifactId>jetty-annotations</artifactId>
      <version>9.4.31.v20200723</version>
      <type>jar</type>
    </dependency>

  </dependencies>

  <build>
    <plugins>

      <plugin>
        <groupId>org.codehaus.mojo</groupId>
        <artifactId>exec-maven-plugin</artifactId>
        <version>3.0.0</version>
        <executions>
          <execution>
            <goals>
              <goal>java</goal>
            </goals>
          </execution>
        </executions>
        <configuration>
          <mainClass>io.happycoding.Main</mainClass>
        </configuration>
      </plugin>

      <plugin>
        <groupId>com.google.cloud.tools</groupId>
        <artifactId>appengine-maven-plugin</artifactId>
        <version>2.2.0</version>
        <configuration>
          <projectId>happy-coding-gcloud</projectId>
          <version>1</version>
        </configuration>
      </plugin>
    </plugins>
  </build>
</project>

src/main/appengine/app.yaml

runtime: java11
entrypoint: 'java -cp "*" io.happycoding.Main app-engine-hello-world-1.war'

src/main/java/io/happycoding/Main.java

package io.happycoding;

import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.webapp.Configuration.ClassList;
import org.eclipse.jetty.webapp.WebAppContext;
import io.happycoding.servlets.HelloWorldServlet;

/** Simple Jetty Main that can execute a WAR file when passed as an argument. */
public class Main {

  public static void main(String[] args) throws Exception {
    if (args.length != 1) {
      System.err.println("Usage: need a relative path to the war file to execute");
      System.exit(1);
    }
    System.setProperty("org.eclipse.jetty.util.log.class", "org.eclipse.jetty.util.log.StrErrLog");
    System.setProperty("org.eclipse.jetty.LEVEL", "INFO");

    Server server = new Server(8080);

    WebAppContext webapp = new WebAppContext();
    webapp.setContextPath("/");
    webapp.setWar(args[0]);
    ClassList classlist = ClassList.setServerDefault(server);

    // Enable Annotation Scanning.
    classlist.addBefore(
        "org.eclipse.jetty.webapp.JettyWebXmlConfiguration",
        "org.eclipse.jetty.annotations.AnnotationConfiguration");

    server.setHandler(webapp);
    server.join();
  }
}

src/main/webapp/index.html

<!DOCTYPE html>
<html>
  <head>
    <meta charset="UTF-8">
    <title>Google Cloud Hello World</title>
  </head>
  <body>
    <h1>Google Cloud Hello World</h1>
    <p>This is a sample HTML file. Click <a href="/hello">here</a> to see content served from a servlet.</p>
    <p>Learn more at <a href="https://happycoding.io">HappyCoding.io</a>.</p>
  </body>
</html>

src/main/java/io/happycoding/servlets/HelloWorldServlet.java

package io.happycoding.servlets;

import java.io.IOException;

import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

@WebServlet("/hello")
public class HelloWorldServlet extends HttpServlet {

  @Override
  public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException {
    response.setContentType("text/html;");
    response.getOutputStream().println("<h1>Hello world!</h1>");
  }
}

I'm guessing something is off with how I'm setting the classpath of the live site, but I don't see anything obviously wrong.

With the packaging property in pom.xml set to war, I get a .war file with these contents:

index.html
META-INF/MANIFEST.MF
META-INF/maven/io.happycoding/app-engine-hello-world/pom.properties
META-INF/maven/io.happycoding/app-engine-hello-world/pom.xml
WEB-INF/classes/io/happycoding/Main.class
WEB-INF/classes/io/happycoding/servlets/HelloWorldServlet.class
WEB-INF/classes/lib/asm-7.3.1.jar
WEB-INF/classes/lib/asm-analysis-7.3.1.jar
WEB-INF/classes/lib/asm-commons-7.3.1.jar
WEB-INF/classes/lib/asm-tree-7.3.1.jar
WEB-INF/classes/lib/javax.annotation-api-1.3.jar
WEB-INF/classes/lib/javax.servlet-api-4.0.1.jar
WEB-INF/classes/lib/jetty-annotations-9.4.31.v20200723.jar
WEB-INF/classes/lib/jetty-http-9.4.31.v20200723.jar
WEB-INF/classes/lib/jetty-io-9.4.31.v20200723.jar
WEB-INF/classes/lib/jetty-jndi-9.4.31.v20200723.jar
WEB-INF/classes/lib/jetty-plus-9.4.31.v20200723.jar
WEB-INF/classes/lib/jetty-security-9.4.31.v20200723.jar
WEB-INF/classes/lib/jetty-server-9.4.31.v20200723.jar
WEB-INF/classes/lib/jetty-servlet-9.4.31.v20200723.jar
WEB-INF/classes/lib/jetty-util-9.4.31.v20200723.jar
WEB-INF/classes/lib/jetty-webapp-9.4.31.v20200723.jar
WEB-INF/classes/lib/jetty-xml-9.4.31.v20200723.jar

If I change the packaging property in pom.xml to jar, then I get a .jar file with these contents:

io/happycoding/Main.class
io/happycoding/servlets/HelloWorldServlet.class
META-INF/MANIFEST.MF
META-INF/maven/io.happycoding/app-engine-hello-world/pom.properties
META-INF/maven/io.happycoding/app-engine-hello-world/pom.xml

And I get this error in the logs for the live site instead:

Error: Unable to initialize main class io.happycoding.Main 
Caused by: java.lang.NoClassDefFoundError: org/eclipse/jetty/server/Handler 

That feels like progress, but then I also get 404 errors in my live server, so I feel pretty stuck.

What do I need to change about my above setup to make it work both locally and on my live server?

Edit: I can see the following files in the App Engine debugger:

I tried adding this to my pom.xml file:

<plugin>
  <groupId>org.apache.maven.plugins</groupId>
  <artifactId>maven-dependency-plugin</artifactId>
  <version>3.1.2</version>
  <executions>
    <execution>
      <id>copy</id>
      <phase>prepare-package</phase>
      <goals>
        <goal>copy-dependencies</goal>
      </goals>
      <configuration>
        <outputDirectory>
          ${project.build.directory}/appengine-staging
        </outputDirectory>
      </configuration>
    </execution>
  </executions>
</plugin>

Then I see these file in the App Engine debugger:

But I still get the same error.

I believe that the problem is caused by my Main class being inside a .war file which has no effect on the classpath, which is why it can't be found.

How do I package my project up so it works locally and on my live server?

Answer 1

I think your problem is that you are including the Main class in the war itself, and App Engine is unable to find it.

As you can see in the GCP migration guide, the Main class is defined in an external dependency named simple-jetty-main.

With the execution of the maven-dependency-plugin this dependency is copied to the appengine-staging directory, making it accessible from the Java classpath.

This is the reason why the Main class can be found in the example proposed in the guide when executing the command from the app.yaml entrypoint:

entrypoint: 'java -cp "*" com.example.appengine.demo.jettymain.Main helloworld.war'

Therefore, the solution will be to include your Main class in another library, independent from the war file that you need to deploy.

Maybe you can create a library - as Google does with simple-jetty-main - that can reuse in your GCP projects for this task.

Just for testing, in order to confirm this point, you can use the simple-jetty-main library itself (you can clone the required code from https://github.com/GoogleCloudPlatform/java-docs-samples/tree/master/appengine-java11/appengine-simple-jetty-main). Install it, include the dependency in your pom.xml, include also the maven-dependency-plugin, and define your entrypoint as follows:

entrypoint: 'java -cp "*" com.example.appengine.demo.jettymain.Main app-engine-hello-world-1.war'

For your comments, you will prefer not to have the separation between the Main class and the rest of the code.

To meet that requirement we must first change the Main class so that Jetty can serve HelloWorldSevlet and the static content. The code is actually very similar to the one you provided. Please excuse the simplicity of the setup, it is based on web.xml file; if necessary, further development can be done to deal with annotations or whatever is deemed appropriate:

package io.happycoding;

import java.net.URL;

import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.webapp.WebAppContext;

public class Main {

  public static final String WEBAPP_RESOURCES_LOCATION = "META-INF/resources";

  public static void main(String[] args) throws Exception {
    System.setProperty("org.eclipse.jetty.util.log.class", "org.eclipse.jetty.util.log.StrErrLog");
    System.setProperty("org.eclipse.jetty.LEVEL", "INFO");

    Server server = new Server(8080);

    URL webAppDir = Thread.currentThread().getContextClassLoader().getResource(WEBAPP_RESOURCES_LOCATION);
    if (webAppDir == null) {
      throw new RuntimeException(String.format("Unable to find %s directory into the JAR file", WEBAPP_RESOURCES_LOCATION));
    }

    WebAppContext webAppContext = new WebAppContext();
    webAppContext.setContextPath("/");
    webAppContext.setDescriptor(WEBAPP_RESOURCES_LOCATION + "/WEB-INF/web.xml");
    webAppContext.setResourceBase(webAppDir.toURI().toString());
    webAppContext.setParentLoaderPriority(true);

    server.setHandler(webAppContext);

    server.start();

    server.join();
  }
}

The static resources can be loaded from a directory of your choice (it will be parameterized in the pom.xml).

For instance, I have created the src/main/webapp folder to store the static content.

In this folder, you also need to define - in this case, due to the way we setup Jetty - a WEB-INF directory with this web.xml file inside:

<?xml version="1.0" encoding="UTF-8"?>
<web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xmlns="http://java.sun.com/xml/ns/javaee" xmlns:web="http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd"
         xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd"
         version="2.5">

    <servlet>
        <servlet-name>HelloWorldServlet</servlet-name>
        <servlet-class>io.happycoding.servlets.HelloWorldServlet</servlet-class>
    </servlet>

    <servlet-mapping>
        <servlet-name>HelloWorldServlet</servlet-name>
        <url-pattern>/hello</url-pattern>
    </servlet-mapping>

    <welcome-file-list>
        <welcome-file>index.html</welcome-file>
    </welcome-file-list>

</web-app>

This is a tree of my source code setup:

The pom.xml file is very similar to the one you provided. I only included the maven-resources-plugin to copy the web app static content to the jar file, and the maven-shade-plugin to generate an UberJar:

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>io.happycoding</groupId>
    <artifactId>app-engine-hello-world</artifactId>
    <version>1</version>
    <packaging>jar</packaging>

    <properties>
        <!-- App Engine currently supports Java 11 -->
        <maven.compiler.source>11</maven.compiler.source>
        <maven.compiler.target>11</maven.compiler.target>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <failOnMissingWebXml>false</failOnMissingWebXml>
        <!-- Directory where static content resides -->
        <webapp.dir>./src/main/webapp</webapp.dir>
    </properties>

    <dependencies>
        <dependency>
            <groupId>javax.servlet</groupId>
            <artifactId>javax.servlet-api</artifactId>
            <version>4.0.1</version>
        </dependency>

        <dependency>
            <groupId>org.eclipse.jetty</groupId>
            <artifactId>jetty-server</artifactId>
            <version>9.4.31.v20200723</version>
        </dependency>
        <dependency>
            <groupId>org.eclipse.jetty</groupId>
            <artifactId>jetty-webapp</artifactId>
            <version>9.4.31.v20200723</version>
            <type>jar</type>
        </dependency>
        <dependency>
            <groupId>org.eclipse.jetty</groupId>
            <artifactId>jetty-util</artifactId>
            <version>9.4.31.v20200723</version>
        </dependency>
        <dependency>
            <groupId>org.eclipse.jetty</groupId>
            <artifactId>jetty-annotations</artifactId>
            <version>9.4.31.v20200723</version>
            <type>jar</type>
        </dependency>

    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.codehaus.mojo</groupId>
                <artifactId>exec-maven-plugin</artifactId>
                <version>3.0.0</version>
                <executions>
                    <execution>
                        <goals>
                            <goal>java</goal>
                        </goals>
                    </execution>
                </executions>
                <configuration>
                    <mainClass>io.happycoding.Main</mainClass>
                </configuration>
            </plugin>

            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-resources-plugin</artifactId>
                <version>2.7</version>
                <executions>
                    <execution>
                        <id>copy-web-resources</id>
                        <phase>compile</phase>
                        <goals>
                            <goal>copy-resources</goal>
                        </goals>
                        <configuration>
                            <outputDirectory>${project.build.directory}/classes/META-INF/resources</outputDirectory>
                            <resources>
                                <resource>
                                    <directory>${webapp.dir}</directory>
                                </resource>
                            </resources>
                        </configuration>
                    </execution>
                </executions>
            </plugin>

            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-shade-plugin</artifactId>
                <version>3.2.4</version>
                <executions>
                    <execution>
                        <phase>package</phase>
                        <goals>
                            <goal>shade</goal>
                        </goals>
                        <configuration>
                            <transformers>
                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                                    <mainClass>io.happycoding.Main</mainClass>
                                </transformer>
                            </transformers>
                        </configuration>
                    </execution>
                </executions>
            </plugin>

            <plugin>
              <groupId>com.google.cloud.tools</groupId>
              <artifactId>appengine-maven-plugin</artifactId>
              <version>2.2.0</version>
              <configuration>
                <projectId>happy-coding-gcloud</projectId>
                <version>1</version>
              </configuration>
            </plugin>
        </plugins>
    </build>


</project>

With this setup, you can run the application locally by executing the following command:

mvn exec:java

You can also run the program locally right from the java tool:

java -jar appengine-deploy-sample-1.jar

Sorry, I cannot test the setup in GCP but I think, that according to the migration guide, you can try to deploy the application without indicating the entrypoint in your app.yaml.

If it does not work, you can try to run the app by configuring an entrypoint similar to the following:

entrypoint: 'java -jar appengine-deploy-sample-1.jar'

Or maybe:

entrypoint: 'java -cp "*" -jar appengine-deploy-sample-1.jar'