How to use Stackdriver logging on Cloud Run

Question

I'm trying to get stackdriver logging working for a simple Go app running in Google Cloud Run (fully managed), but don't see stackdriver entries in CloudRun logs.

I've created simplest possible demo app based on "official" stackdriver golang example

Cloud Run docs states that no additional actions should be performed to write stackdriver logs

My service uses default service account

I'm using Go 1.13 to compile code (Dockerfile is copied from Cloud Run example "as-is")

I've tried to deploy it to different regions, with no success

When running container locally, using service account credentials, stackdriver log message does not appear in local terminal or stackdriver console

No matter what, on app start I see only "Before stackdriver logging" followed by "After stackdriver logging" with no other messages\errors in the middle

Here's part of logging code (use link above to get full source, Dockerfile and instructions to build and run the app):

import (
    "context"
    "log"
    "os"
    ...

    "cloud.google.com/go/compute/metadata"
    "cloud.google.com/go/logging"
)

func main() {
    loggingClient, err := stackdriverClient()
    ...
    log.Println("Before stackdriver logging")
    logger.StandardLogger(logging.Info).Println("Stackdriver log")
    if err = logger.Flush(); err != nil {
        log.Fatalf("Failed to flush client: %v", err)
    }
    if err = loggingClient.Close(); err != nil {
        log.Fatalf("Failed to close client: %v", err)
    }
    log.Println("After stackdriver logging")
    ...
}

func stackdriverClient() (client *logging.Client, err error) {
    var projectID string
    if projectID, err = metadata.ProjectID(); err == nil {
        client, err = logging.NewClient(context.Background(), projectID)
    }
    return
}

Answer 1

It turns out that log entries are written successfully
But default Cloud Run filter in logviewer web UI does not include them
Filter expression below worked for me to get all the logs:

resource.type = "project" OR resource.type = "cloud_run_revision"

(service name, location, severity omitted)

"stdout\stderr" log entries match resource.type="cloud_run_revision", while stackdriver log entries match resource.type="project"

Update: I've created a ticket for this in google tracker

Answer 2

Since Cloud Run is already integrated with Cloud Logging, there is no need to use the Go Client Library. We run all our gRPC services on Cloud Run and use the following to ensure the logs are properly formatted in Cloud Logging:

package main

import (
    "github.com/rs/zerolog"
    "github.com/rs/zerolog/log"
    "os"
)

// initializeLogging sets up the logging configuration for the service.
// Invoke this method in your Init() method.
func initializeLogging() {
    // Set logging options for production development
    if os.Getenv("ENV") != "DEV" {
        // change the level field name to ensure these are parsed correctly in Stackdriver
        zerolog.LevelFieldName = "severity"
        // UNIX Time is faster and smaller than most timestamps
        zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
    } else {
        // Set logging options for local development
        log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
        zerolog.SetGlobalLevel(zerolog.DebugLevel)
    }
    
    // Example log
    log.Info().Msg("This is how you log at Info level")
}

The logs are then nicely displayed for local development.

If you don't want to use any 3rd party logging libraries, a simple way is to construct your own Entry objects.

package main

import (
    "context"
    "encoding/json"
    "fmt"
    "google.golang.org/grpc"
    "google.golang.org/grpc/metadata"
    "log"
    "os"
    "strings"
)

// LogSeverity is used to map the logging levels consistent with Google Cloud Logging.
type LogSeverity string

const (
    // LogSeverity_DEBUG Debug or trace information.
    LogSeverity_DEBUG LogSeverity = "DEBUG"
    // LogSeverity_INFO Routine information, such as ongoing status or performance.
    LogSeverity_INFO LogSeverity = "INFO"
    // LogSeverity_NOTICE Normal but significant events, such as start up, shut down, or
    // a configuration change.
    LogSeverity_NOTICE LogSeverity = "NOTICE"
    // LogSeverity_WARNING Warning events might cause problems.
    LogSeverity_WARNING LogSeverity = "WARNING"
    // LogSeverity_ERROR Error events are likely to cause problems.
    LogSeverity_ERROR LogSeverity = "ERROR"
    // LogSeverity_CRITICAL Critical events cause more severe problems or outages.
    LogSeverity_CRITICAL LogSeverity = "CRITICAL"
    // LogSeverity_ALERT A person must take an action immediately.
    LogSeverity_ALERT LogSeverity = "ALERT"
    // LogSeverity_EMERGENCY One or more systems are unusable.
    LogSeverity_EMERGENCY LogSeverity = "EMERGENCY"
)

// Entry defines a log entry.
// If logs are provided in this format, Google Cloud Logging automatically
// parses the attributes into their LogEntry format as per
// https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry which then automatically
// makes the logs available in Google Cloud Logging and Tracing.
type Entry struct {
    Message  string      `json:"message"`
    Severity LogSeverity `json:"severity,omitempty"`
    Trace    string      `json:"logging.googleapis.com/trace,omitempty"`
    // To extend details sent to the logs, you may add the attributes here.
    //MyAttr1 string `json:"component,omitempty"`
}

// String renders an entry structure to the JSON format expected by Cloud Logging.
func (e Entry) String() string {

    // Defaults to INFO level.
    if e.Severity == "" {
        e.Severity = LogSeverity_INFO
    }

    // if Development is local then print out all logs
    if os.Getenv("ENV") == "LOCAL" {
        var prefix string
        switch e.Severity {
        case LogSeverity_DEBUG:
            prefix = colorize("DBG:      ", 90)
        case LogSeverity_INFO:
            prefix = colorize("INFO:     ", 32)
        case LogSeverity_NOTICE:
            prefix = colorize("NOTICE:   ", 34)
        case LogSeverity_WARNING:
            prefix = colorize("WARNING:  ", 33)
        case LogSeverity_ERROR:
            prefix = colorize("ERROR:    ", 31)
        case LogSeverity_ALERT:
            prefix = colorize("ALERT:    ", 91)
        case LogSeverity_CRITICAL:
            prefix = colorize("CRITICAL: ", 41)
        case LogSeverity_EMERGENCY:
            prefix = colorize("EMERGENCY:", 101)
        }
        return prefix + " " + e.Message
    } else {
        out, err := json.Marshal(e)
        if err != nil {
            log.Printf("json.Marshal: %v", err)
        }
        return string(out)
    }
}

// colorize returns the string s wrapped in ANSI code c
// Codes available at https://en.wikipedia.org/wiki/ANSI_escape_code#Colors
func colorize(s interface{}, c int) string {
    return fmt.Sprintf("\x1b[%dm%v\x1b[0m", c, s)
}

Using Google Cloud's Special Fields allows tighter integration with their Cloud Logging product.